diff --git a/.github/workflows/actions/build-and-package/action.yaml b/.github/workflows/actions/build-and-package/action.yaml new file mode 100644 index 000000000..0f36d0947 --- /dev/null +++ b/.github/workflows/actions/build-and-package/action.yaml @@ -0,0 +1,143 @@ +name: Check Build and Package +description: Run checks, build and package VSIX, sign it, and run security scans (Ubuntu only) +inputs: + SEGMENT_KEY: + description: Segment analytics key + required: true + ARTIFACTORY_HOST: + description: Artifactory host for signing + required: true + ARTIFACTORY_PASSWORD: + description: Artifactory password for signing + required: true + ARTIFACTORY_USERNAME: + description: Artifactory username for signing + required: true + GARASIGN_PASSWORD: + description: Garasign password for signing + required: true + GARASIGN_USERNAME: + description: Garasign username for signing + required: true + SNYK_TOKEN: + description: Snyk token for security scanning + required: true + JIRA_API_TOKEN: + description: Jira API token for vulnerability tickets + required: true + +runs: + using: "composite" + steps: + - name: Install Deps Ubuntu + run: sudo apt-get update -y && sudo apt-get -y install libkrb5-dev libsecret-1-dev net-tools libstdc++6 gnome-keyring + shell: bash + + # Default Python (3.12) doesn't have support for distutils because of + # which the dep install fails constantly on macos + # https://github.com/nodejs/node-gyp/issues/2869 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run node-gyp bug workaround script + run: | + curl -sSfLO https://raw.githubusercontent.com/mongodb-js/compass/42e6142ae08be6fec944b80ff6289e6bcd11badf/.evergreen/node-gyp-bug-workaround.sh && bash node-gyp-bug-workaround.sh + shell: bash + + - name: Set SEGMENT_KEY + env: + SEGMENT_KEY: ${{ inputs.SEGMENT_KEY }} + run: | + echo "SEGMENT_KEY=${SEGMENT_KEY}" >> $GITHUB_ENV + shell: bash + + - name: Validate SEGMENT_KEY + run: | + if [ -z "${SEGMENT_KEY}" ]; then + echo "SEGMENT_KEY is not set or is empty" + exit 1 + fi + shell: bash + + - name: Install Dependencies + run: npm ci --omit=optional + shell: bash + + - name: Compile + run: npm run compile + shell: bash + + - name: Run Checks + run: npm run check + shell: bash + + - name: Build .vsix + env: + NODE_OPTIONS: "--require ./scripts/no-npm-list-fail.js --max_old_space_size=4096" + # NOTE: --githubBranch is "The GitHub branch used to infer relative links in README.md." + run: | + npx vsce package --githubBranch main + shell: bash + + - name: Check .vsix filesize + run: npm run check-vsix-size + shell: bash + + - name: Sign .vsix + env: + ARTIFACTORY_PASSWORD: ${{ inputs.ARTIFACTORY_PASSWORD }} + ARTIFACTORY_USERNAME: ${{ inputs.ARTIFACTORY_USERNAME }} + GARASIGN_PASSWORD: ${{ inputs.GARASIGN_PASSWORD }} + GARASIGN_USERNAME: ${{ inputs.GARASIGN_USERNAME }} + run: | + set -e + FILE_TO_SIGN=$(find . -maxdepth 1 -name '*.vsix' -print -quit) + if [ -z "$FILE_TO_SIGN" ]; then + echo "Error: No .vsix file found in the current directory." >&2 + exit 1 + fi + node scripts/sign-vsix.js "${FILE_TO_SIGN}" + ls *.vsix.sig + shell: bash + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: VSIX Package + path: | + *.vsix + *.vsix.sig + + - name: Run Snyk Test + shell: bash + env: + SNYK_TOKEN: ${{ inputs.SNYK_TOKEN }} + run: | + npm run snyk-test + + - name: Create Jira Tickets + if: > + ( + github.event_name == 'push' && github.ref == 'refs/heads/main' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + ) + shell: bash + env: + JIRA_API_TOKEN: ${{ inputs.JIRA_API_TOKEN }} + JIRA_BASE_URL: "https://jira.mongodb.org" + JIRA_PROJECT: "VSCODE" + JIRA_VULNERABILITY_BUILD_INFO: "- [GitHub Run|https://github.com/mongodb-js/vscode/actions/runs/${{github.run_id}}/jobs/${{github.job}}]" + run: | + npm run create-vulnerability-tickets > /dev/null + + - name: Generate Vulnerability Report (Fail on >= High) + continue-on-error: ${{ github.event_name == 'pull_request' }} + shell: bash + run: | + # The standard output is suppressed since Github Actions logs are + # available for everyone with read access to the repo, which is everyone that is + # logged in for public repos. + # This command is only here to fail on failures for `main` and tags. + npm run generate-vulnerability-report > /dev/null diff --git a/.github/workflows/actions/run-tests/action.yaml b/.github/workflows/actions/run-tests/action.yaml new file mode 100644 index 000000000..df1cf5ece --- /dev/null +++ b/.github/workflows/actions/run-tests/action.yaml @@ -0,0 +1,80 @@ +name: Run Tests +description: Run checks, tests, and install tests on the VSIX package +inputs: + SEGMENT_KEY: + description: Segment analytics key + required: true + +runs: + using: "composite" + steps: + - name: Install Deps Ubuntu + if: ${{ runner.os == 'Linux' }} + run: sudo apt-get update -y && sudo apt-get -y install libkrb5-dev libsecret-1-dev net-tools libstdc++6 gnome-keyring + shell: bash + + # Default Python (3.12) doesn't have support for distutils because of + # which the dep install fails constantly on macos + # https://github.com/nodejs/node-gyp/issues/2869 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run node-gyp bug workaround script + run: | + curl -sSfLO https://raw.githubusercontent.com/mongodb-js/compass/42e6142ae08be6fec944b80ff6289e6bcd11badf/.evergreen/node-gyp-bug-workaround.sh && bash node-gyp-bug-workaround.sh + shell: bash + + - name: Set SEGMENT_KEY + env: + SEGMENT_KEY: ${{ inputs.SEGMENT_KEY }} + run: | + echo "SEGMENT_KEY=${SEGMENT_KEY}" >> $GITHUB_ENV + shell: bash + + - name: Validate SEGMENT_KEY + run: | + if [ -z "${SEGMENT_KEY}" ]; then + echo "SEGMENT_KEY is not set or is empty" + exit 1 + fi + shell: bash + + - name: Install Dependencies + shell: bash + run: | + npm ci --omit=optional + + - name: Download VSIX artifact + uses: actions/download-artifact@v4 + with: + name: VSIX Package + + - name: Run Tests + env: + NODE_OPTIONS: "--max_old_space_size=4096" + run: | + npm run test + shell: bash + + - name: Install VSIX and Test + shell: bash + run: | + # Find the VSIX file + VSIX_FILE=$(find . -maxdepth 1 -name '*.vsix' -print -quit) + if [ -z "$VSIX_FILE" ]; then + echo "Error: No .vsix file found" >&2 + exit 1 + fi + + echo "Found VSIX file: $VSIX_FILE" + + # For now, just verify the file exists and is readable + # Next, this will include actual VS Code installation tests + if [ ! -r "$VSIX_FILE" ]; then + echo "Error: VSIX file is not readable" >&2 + exit 1 + fi + + echo "VSIX file validation passed" + ls -la "$VSIX_FILE" diff --git a/.github/workflows/draft-release.yaml b/.github/workflows/draft-release.yaml index 70c290198..b64d31a28 100644 --- a/.github/workflows/draft-release.yaml +++ b/.github/workflows/draft-release.yaml @@ -1,34 +1,35 @@ +# Run manually to prepare a draft release for the next version of the extension. +# The workflow will create a draft github release where the .vsix can be +# downloaded and manually tested before publishing. To release the version, +# publish the draft release, which will trigger the publish-release workflow. name: Draft release on: workflow_dispatch: inputs: versionBump: - description: 'Version bump' + description: "Version bump" type: choice required: true - default: 'patch' + default: "patch" options: - - patch - - minor - - major - - exact-version + - patch + - minor + - major + - exact-version exactVersion: description: 'Exact version: (Only effective selecting "exact-version" as version bump)' required: false -description: | - Run manually to prepare a draft release for the next version of the extension. The workflow will create a draft - github release where the .vsix can be downloaded and manually tested before publishing. To release the version, - publish the draft release, which will trigger the publish-release workflow. - permissions: contents: write jobs: prepare-release: runs-on: ubuntu-latest + outputs: + release-tag: ${{ steps.set-tag.outputs.release-tag }} steps: - name: Checkout uses: actions/checkout@v4 @@ -85,8 +86,40 @@ jobs: exit 1 fi - - name: Run tests and build - uses: ./.github/workflows/actions/test-and-build + - name: Set release tag output + id: set-tag + run: echo "release-tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT + + - name: Upload updated package.json + uses: actions/upload-artifact@v4 + with: + name: updated-package-json + path: package.json + + build-and-package: + name: Check, Build and Package + runs-on: ubuntu-latest + needs: prepare-release + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download updated package.json + uses: actions/download-artifact@v4 + with: + name: updated-package-json + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: 22.15.1 + cache: npm + + - name: Check, build and package + uses: ./.github/workflows/actions/build-and-package with: SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} ARTIFACTORY_HOST: ${{ secrets.ARTIFACTORY_HOST }} @@ -97,9 +130,57 @@ jobs: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + test: + name: Test + needs: build-and-package + + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download updated package.json + uses: actions/download-artifact@v4 + with: + name: updated-package-json + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: 22.15.1 + cache: npm + + - name: Run tests + uses: ./.github/workflows/actions/run-tests + with: + SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} + + create-draft-release: + name: Create Draft Release + runs-on: ubuntu-latest + needs: [prepare-release, test] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download VSIX artifact + uses: actions/download-artifact@v4 + with: + name: VSIX Package + - name: Create Draft Release run: | set -e + RELEASE_TAG="${{ needs.prepare-release.outputs.release-tag }}" echo Creating draft release for: "${RELEASE_TAG}" ls *.vsix ls *.vsix.sig @@ -114,4 +195,3 @@ jobs: shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/test-and-build-from-fork.yaml b/.github/workflows/test-and-build-from-fork.yaml index 91bb7345b..bbb1d792a 100644 --- a/.github/workflows/test-and-build-from-fork.yaml +++ b/.github/workflows/test-and-build-from-fork.yaml @@ -8,8 +8,40 @@ permissions: pull-requests: write jobs: - test-and-build: - name: Test and Build + build-and-package: + name: Check, Build and Package + runs-on: ubuntu-latest + if: github.event.pull_request.user.login == 'dependabot[bot]' || github.event.pull_request.head.repo.full_name != github.repository + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: 22.15.1 + cache: npm + + - name: Check, build and package + uses: ./.github/workflows/actions/build-and-package + with: + SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_DEV }} + ARTIFACTORY_HOST: ${{ secrets.ARTIFACTORY_HOST }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + GARASIGN_PASSWORD: ${{ secrets.GARASIGN_PASSWORD }} + GARASIGN_USERNAME: ${{ secrets.GARASIGN_USERNAME }} + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + + test: + name: Test + needs: build-and-package if: github.event.pull_request.user.login == 'dependabot[bot]' || github.event.pull_request.head.repo.full_name != github.repository strategy: @@ -33,25 +65,16 @@ jobs: node-version: 22.15.1 cache: npm - - name: Install Dependencies - run: npm ci --omit=optional - - - name: Run Checks - run: npm run check - # the glob here just fails - if: ${{ runner.os != 'Windows' }} - - - name: Run Tests - env: - NODE_OPTIONS: "--max_old_space_size=4096" + - name: Run tests + uses: ./.github/workflows/actions/run-tests + with: SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_DEV }} - run: npm run test merge-dependabot-pr: name: Merge Dependabot PR runs-on: ubuntu-latest needs: - - test-and-build + - test if: github.event.pull_request.user.login == 'dependabot[bot]' steps: - name: Enable auto-merge for Dependabot PRs diff --git a/.github/workflows/test-and-build.yaml b/.github/workflows/test-and-build.yaml index 5399eef5f..d8a3ee9c4 100644 --- a/.github/workflows/test-and-build.yaml +++ b/.github/workflows/test-and-build.yaml @@ -14,17 +14,11 @@ permissions: contents: read jobs: - test-and-build: - name: Test and Build + build-and-package: + name: Check, Build and Package + runs-on: ubuntu-latest if: github.event.pull_request.user.login != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - fail-fast: false - - runs-on: ${{ matrix.os }} - steps: - name: Checkout uses: actions/checkout@v4 @@ -37,8 +31,8 @@ jobs: node-version: 22.15.1 cache: npm - - name: Run tests and build - uses: ./.github/workflows/actions/test-and-build + - name: Check, build and package + uses: ./.github/workflows/actions/build-and-package with: SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} ARTIFACTORY_HOST: ${{ secrets.ARTIFACTORY_HOST }} @@ -48,3 +42,32 @@ jobs: GARASIGN_USERNAME: ${{ secrets.GARASIGN_USERNAME }} SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + + test: + name: Test + needs: build-and-package + if: github.event.pull_request.user.login != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository + + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: 22.15.1 + cache: npm + + - name: Run tests + uses: ./.github/workflows/actions/run-tests + with: + SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} diff --git a/package.json b/package.json index f9a7c2308..ad28554d1 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,9 @@ "check-vsix-size": "ts-node ./scripts/check-vsix-size.ts", "release-draft": "node ./scripts/release-draft.js", "reformat": "prettier --write .", + "presnyk-test": "echo \"Creating backup for package-lock.json.\"; cp package-lock.json original-package-lock.json", "snyk-test": "node scripts/snyk-test.js", + "postsnyk-test": "echo \"Restoring original package-lock.json.\"; mv original-package-lock.json package-lock.json", "generate-icon-font": "ts-node ./scripts/generate-icon-font.ts", "generate-vulnerability-report": "mongodb-sbom-tools generate-vulnerability-report --snyk-reports=.sbom/snyk-test-result.json --dependencies=.sbom/dependencies.json --fail-on=high", "create-vulnerability-tickets": "mongodb-sbom-tools generate-vulnerability-report --snyk-reports=.sbom/snyk-test-result.json --dependencies=.sbom/dependencies.json --create-jira-issues", diff --git a/scripts/snyk-test.js b/scripts/snyk-test.js index 688ce15d6..47743cd63 100644 --- a/scripts/snyk-test.js +++ b/scripts/snyk-test.js @@ -6,6 +6,56 @@ const { glob } = require('glob'); const { promisify } = require('util'); const execFile = promisify(childProcess.execFile); +const PACKAGE_LOCK_PATH = path.join(__dirname, '..', 'package-lock.json'); + +/** + * "node_modules/@vscode/vsce-sign" package which is a dev dependency used for + * publishing extension declares platform specific optionalDependencies, namely + * the following: + * - "@vscode/vsce-sign-alpine-arm64" + * - "@vscode/vsce-sign-alpine-x64" + * - "@vscode/vsce-sign-darwin-arm64" + * - "@vscode/vsce-sign-darwin-x64" + * - "@vscode/vsce-sign-linux-arm" + * - "@vscode/vsce-sign-linux-arm64" + * - "@vscode/vsce-sign-linux-x64" + * - "@vscode/vsce-sign-win32-arm64" + * - "@vscode/vsce-sign-win32-x64" + * + * Snyk requires what is declared in package-lock.json to be also present in + * installed node_modules but this will never happen because for any platform, + * other platform specific deps will always be missing which means Snyk will + * always fail in this case. + * + * Because we always install with `npm ci --omit=optional`, with this method we + * try to remove these identified problematic optionalDependencies before + * running the Snyk tests and once the tests are finished, we restore the + * original state back using npm hooks. + */ +async function removeProblematicOptionalDepsFromPackageLock() { + const packageLockContent = JSON.parse( + await fs.readFile(PACKAGE_LOCK_PATH, 'utf-8'), + ); + + const vsceSignPackage = + packageLockContent.packages?.['node_modules/@vscode/vsce-sign']; + + if (!vsceSignPackage || !vsceSignPackage.optionalDependencies) { + console.info('No problematic optional dependencies to fix'); + return; + } + + // Temporarily remove the optional dependencies + vsceSignPackage['optionalDependencies'] = {}; + + // We write the actual package-lock path but restoring of the original file is + // handled by npm hooks. + await fs.writeFile( + PACKAGE_LOCK_PATH, + JSON.stringify(packageLockContent, null, 2), + ); +} + async function snykTest(cwd) { const tmpPath = path.join(os.tmpdir(), 'tempfile-' + Date.now()); @@ -17,9 +67,8 @@ async function snykTest(cwd) { await execFile( 'npx', [ - 'snyk', + 'snyk@latest', 'test', - '--all-projects', '--severity-threshold=low', '--dev', `--json-file-output=${tmpPath}`, @@ -47,6 +96,7 @@ async function snykTest(cwd) { async function main() { const rootPath = path.resolve(__dirname, '..'); await fs.mkdir(path.join(rootPath, `.sbom`), { recursive: true }); + await removeProblematicOptionalDepsFromPackageLock(); const results = await snykTest(rootPath); await fs.writeFile(