feat(lsp): older and delisted versions of lsp are automatically removed (#6409)

## Problem
- Installed LSP artifacts remain forever. There is no cleanup effort
made.

## Solution
Implement the following heuristic:
- Delete all currently installed delisted versions. 
- Delete all versions that remain, except the most recent 2. 

Included in this change is a new utility function `partition`.
`partition` is like `filter`, but it produces both the positive and
negative result in two separate sublists. See the tests for a simple
example.

---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

Author: Hweinstock

packages/amazonq/src/lsp/lspInstaller.ts

@@ -12,6 +12,7 @@ import {
     fs,
     LspResolution,
     getNodeExecutableName,
+    cleanLspDownloads,
 } from 'aws-core-vscode/shared'
 import path from 'path'
 
@@ -46,7 +47,7 @@ export class AmazonQLSPResolver implements LspResolver {
         const nodePath = path.join(installationResult.assetDirectory, `servers/${getNodeExecutableName()}`)
         await fs.chmod(nodePath, 0o755)
 
-        // TODO Cleanup old versions of language servers
+        await cleanLspDownloads(manifest.versions, path.dirname(installationResult.assetDirectory))
         return {
             ...installationResult,
             resourcePaths: {

packages/core/src/amazonq/lsp/workspaceInstaller.ts

@@ -10,6 +10,7 @@ import { LanguageServerResolver } from '../../shared/lsp/lspResolver'
 import { Range } from 'semver'
 import { getNodeExecutableName } from '../../shared/lsp/utils/platform'
 import { fs } from '../../shared/fs/fs'
+import { cleanLspDownloads } from '../../shared'
 
 const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json'
 // this LSP client in Q extension is only going to work with these LSP server versions
@@ -30,7 +31,7 @@ export class WorkspaceLSPResolver implements LspResolver {
         const nodePath = path.join(installationResult.assetDirectory, nodeName)
         await fs.chmod(nodePath, 0o755)
 
-        // TODO Cleanup old versions of language servers
+        await cleanLspDownloads(manifest.versions, path.basename(installationResult.assetDirectory))
         return {
             ...installationResult,
             resourcePaths: {

packages/core/src/shared/index.ts

@@ -63,6 +63,7 @@ export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants'
 export * from './lsp/manifestResolver'
 export * from './lsp/lspResolver'
 export * from './lsp/types'
+export * from './lsp/utils/cleanup'
 export { default as request } from './request'
 export * from './lsp/utils/platform'
 export * as processUtils from './utilities/processUtils'

packages/core/src/shared/lsp/lspResolver.ts

@@ -338,9 +338,13 @@ export class LanguageServerResolver {
         return version.targets.find((x) => x.arch === arch && x.platform === platform)
     }
 
+    // lazy calls to `getApplicationSupportFolder()` to avoid failure on windows.
+    public static get defaultDir() {
+        return path.join(getApplicationSupportFolder(), `aws/toolkits/language-servers`)
+    }
+
     defaultDownloadFolder() {
-        const applicationSupportFolder = getApplicationSupportFolder()
-        return path.join(applicationSupportFolder, `aws/toolkits/language-servers/${this.lsName}`)
+        return path.join(LanguageServerResolver.defaultDir, `${this.lsName}`)
     }
 
     private getDownloadDirectory(version: string) {

packages/core/src/shared/lsp/utils/cleanup.ts

@@ -0,0 +1,41 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import path from 'path'
+import { LspVersion } from '../types'
+import { fs } from '../../../shared/fs/fs'
+import { partition } from '../../../shared/utilities/tsUtils'
+import { sort } from 'semver'
+
+async function getDownloadedVersions(installLocation: string) {
+    return (await fs.readdir(installLocation)).map(([f, _], __) => f)
+}
+
+function isDelisted(manifestVersions: LspVersion[], targetVersion: string): boolean {
+    return manifestVersions.find((v) => v.serverVersion === targetVersion)?.isDelisted ?? false
+}
+
+/**
+ * Delete all delisted versions and keep the two newest versions that remain
+ * @param manifest
+ * @param downloadDirectory
+ */
+export async function cleanLspDownloads(manifestVersions: LspVersion[], downloadDirectory: string): Promise<void> {
+    const downloadedVersions = await getDownloadedVersions(downloadDirectory)
+    const [delistedVersions, remainingVersions] = partition(downloadedVersions, (v: string) =>
+        isDelisted(manifestVersions, v)
+    )
+    for (const v of delistedVersions) {
+        await fs.delete(path.join(downloadDirectory, v), { force: true, recursive: true })
+    }
+
+    if (remainingVersions.length <= 2) {
+        return
+    }
+
+    for (const v of sort(remainingVersions).slice(0, -2)) {
+        await fs.delete(path.join(downloadDirectory, v), { force: true, recursive: true })
+    }
+}

packages/core/src/shared/utilities/tsUtils.ts

@@ -94,6 +94,21 @@ export function createFactoryFunction<T extends new (...args: any[]) => any>(cto
     return (...args) => new ctor(...args)
 }
 
+/**
+ * Split a list into two sublists based on the result of a predicate.
+ * @param lst list to split
+ * @param pred predicate to apply to each element
+ * @returns two nested lists, where for all items x in the left sublist, pred(x) returns true. The remaining elements are in the right sublist.
+ */
+export function partition<T>(lst: T[], pred: (arg: T) => boolean): [T[], T[]] {
+    return lst.reduce(
+        ([leftAcc, rightAcc], item) => {
+            return pred(item) ? [[...leftAcc, item], rightAcc] : [leftAcc, [...rightAcc, item]]
+        },
+        [[], []] as [T[], T[]]
+    )
+}
+
 type NoSymbols<T> = { [Property in keyof T]: Property extends symbol ? never : Property }[keyof T]
 export type InterfaceNoSymbol<T> = Pick<T, NoSymbols<T>>
 /**

packages/core/src/test/shared/lsp/utils/cleanup.test.ts

@@ -0,0 +1,93 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { Uri } from 'vscode'
+import { cleanLspDownloads, fs } from '../../../../shared'
+import { createTestWorkspaceFolder } from '../../../testUtil'
+import path from 'path'
+import assert from 'assert'
+
+async function fakeInstallVersion(version: string, installationDir: string): Promise<void> {
+    const versionDir = path.join(installationDir, version)
+    await fs.mkdir(versionDir)
+    await fs.writeFile(path.join(versionDir, 'file.txt'), 'content')
+}
+
+async function fakeInstallVersions(versions: string[], installationDir: string): Promise<void> {
+    for (const v of versions) {
+        await fakeInstallVersion(v, installationDir)
+    }
+}
+
+describe('cleanLSPDownloads', function () {
+    let installationDir: Uri
+
+    before(async function () {
+        installationDir = (await createTestWorkspaceFolder()).uri
+    })
+
+    afterEach(async function () {
+        const files = await fs.readdir(installationDir.fsPath)
+        for (const [name, _type] of files) {
+            await fs.delete(path.join(installationDir.fsPath, name), { force: true, recursive: true })
+        }
+    })
+
+    after(async function () {
+        await fs.delete(installationDir, { force: true, recursive: true })
+    })
+
+    it('keeps two newest versions', async function () {
+        await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath)
+        await cleanLspDownloads([], installationDir.fsPath)
+
+        const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename)
+        assert.strictEqual(result.length, 2)
+        assert.ok(result.includes('2.1.1'))
+        assert.ok(result.includes('1.1.1'))
+    })
+
+    it('deletes delisted versions', async function () {
+        await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath)
+        await cleanLspDownloads([{ serverVersion: '1.1.1', isDelisted: true, targets: [] }], installationDir.fsPath)
+
+        const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename)
+        assert.strictEqual(result.length, 2)
+        assert.ok(result.includes('2.1.1'))
+        assert.ok(result.includes('1.0.1'))
+    })
+
+    it('handles case where less than 2 versions are not delisted', async function () {
+        await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath)
+        await cleanLspDownloads(
+            [
+                { serverVersion: '1.1.1', isDelisted: true, targets: [] },
+                { serverVersion: '2.1.1', isDelisted: true, targets: [] },
+                { serverVersion: '1.0.0', isDelisted: true, targets: [] },
+            ],
+            installationDir.fsPath
+        )
+
+        const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename)
+        assert.strictEqual(result.length, 1)
+        assert.ok(result.includes('1.0.1'))
+    })
+
+    it('handles case where less than 2 versions exist', async function () {
+        await fakeInstallVersions(['1.0.0'], installationDir.fsPath)
+        await cleanLspDownloads([], installationDir.fsPath)
+
+        const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename)
+        assert.strictEqual(result.length, 1)
+    })
+
+    it('does not install delisted version when no other option exists', async function () {
+        await fakeInstallVersions(['1.0.0'], installationDir.fsPath)
+        await cleanLspDownloads([{ serverVersion: '1.0.0', isDelisted: true, targets: [] }], installationDir.fsPath)
+
+        const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename)
+        assert.strictEqual(result.length, 0)
+    })
+})

packages/core/src/test/shared/utilities/tsUtils.test.ts

@@ -0,0 +1,16 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { partition } from '../../../shared/utilities/tsUtils'
+import assert from 'assert'
+
+describe('partition', function () {
+    it('should split the list according to predicate', function () {
+        const items = [1, 2, 3, 4, 5, 6, 7, 8]
+        const [even, odd] = partition(items, (i) => i % 2 === 0)
+        assert.deepStrictEqual(even, [2, 4, 6, 8])
+        assert.deepStrictEqual(odd, [1, 3, 5, 7])
+    })
+})