feat(amazonq): Auto update language servers when new versions are available

Problem:
- Only one version is installed right now

Solution:
- Automatically update when a new version is available in the manifest

Author: jpinkney-aws

packages/amazonq/src/lsp/activation.ts

@@ -11,6 +11,11 @@ import { startLanguageServer } from './client'
 export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
     const serverPath = ctx.asAbsolutePath('resources/qdeveloperserver')
     const clientPath = ctx.asAbsolutePath('resources/qdeveloperclient')
-    await new AmazonQLSPDownloader(serverPath, clientPath).tryInstallLsp()
-    await startLanguageServer(ctx, path.join(serverPath, 'aws-lsp-codewhisperer.js'))
+    const installedAndReady = await new AmazonQLSPDownloader(serverPath, clientPath).tryInstallLsp()
+    if (installedAndReady) {
+        await startLanguageServer(
+            ctx,
+            process.env.AWS_LANGUAGE_SERVER_OVERRIDE ?? path.join(serverPath, 'aws-lsp-codewhisperer.js')
+        )
+    }
 }

packages/amazonq/src/lsp/download.ts

@@ -10,6 +10,7 @@ import {
     tryRemoveFolder,
     fs,
     Manifest,
+    globals,
 } from 'aws-core-vscode/shared'
 
 const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json'
@@ -19,7 +20,7 @@ export class AmazonQLSPDownloader extends LspDownloader {
         private readonly serverPath: string,
         private readonly clientPath: string
     ) {
-        super(manifestURL)
+        super(manifestURL, 'codewhisperer')
     }
 
     async isLspInstalled(): Promise<boolean> {
@@ -46,26 +47,38 @@ export class AmazonQLSPDownloader extends LspDownloader {
             return false
         }
 
+        const current = globals.globalState.tryGet('aws.toolkit.lsp.versions', Object, {})
+        current[this.lsName] = server.serverVersion
+        globals.globalState.tryUpdate('aws.toolkit.lsp.versions', current)
+
         let tempFolder = undefined
 
         try {
             tempFolder = await makeTemporaryToolkitFolder()
 
             // download and extract the business logic
-            await this.downloadAndExtractServer({
+            const downloadedServer = await this.downloadAndExtractServer({
                 content: server,
                 installLocation: this.serverPath,
                 name: 'qdeveloperserver',
                 tempFolder,
             })
+            if (!downloadedServer) {
+                getLogger('lsp').error(`Failed to download and extract server`)
+                return false
+            }
 
             // download and extract mynah ui
-            await this.downloadAndExtractServer({
+            const downloadedClient = await this.downloadAndExtractServer({
                 content: clients,
                 installLocation: this.clientPath,
                 name: 'qdeveloperclient',
                 tempFolder,
             })
+            if (!downloadedClient) {
+                getLogger('lsp').error(`Failed to download and extract client`)
+                return false
+            }
         } finally {
             if (tempFolder) {
                 await tryRemoveFolder(tempFolder)

packages/core/src/amazonq/lsp/lspController.ts

@@ -60,7 +60,7 @@ export class LspController extends LspDownloader {
     }
 
     constructor() {
-        super(manifestUrl, supportedLspServerVersions)
+        super(manifestUrl, 'qcontextserver', supportedLspServerVersions)
         this.serverPath = globals.context.asAbsolutePath(path.join('resources', 'qserver'))
         this.nodePath = globals.context.asAbsolutePath(path.join('resources', nodeBinName))
     }
@@ -194,6 +194,10 @@ export class LspController extends LspDownloader {
             return false
         }
 
+        const current = globals.globalState.tryGet('aws.toolkit.lsp.versions', Object, {})
+        current[this.lsName] = server.serverVersion
+        globals.globalState.tryUpdate('aws.toolkit.lsp.versions', current)
+
         let tempFolder = undefined
 
         try {

packages/core/src/shared/fetchLsp.ts

@@ -7,14 +7,16 @@ import path from 'path'
 import * as crypto from 'crypto'
 import fs from './fs/fs'
 import { getLogger } from './logger/logger'
-import request from './request'
 import { getUserAgent } from './telemetry/util'
 import { ToolkitError } from './errors'
 import fetch from 'node-fetch'
 // TODO remove
 // eslint-disable-next-line no-restricted-imports
 import { createWriteStream } from 'fs'
 import AdmZip from 'adm-zip'
+import { RetryableResourceFetcher } from './resourcefetcher/httpResourceFetcher'
+import { Timeout } from './utilities/timeoutUtils'
+import globals from './extensionGlobals'
 
 export interface Content {
     filename: string
@@ -42,25 +44,58 @@ export interface Manifest {
     }[]
 }
 
+export const logger = getLogger('lsp')
+
+interface StorageManifest {
+    etag: string
+    content: string
+}
+
 export abstract class LspDownloader {
     constructor(
         private readonly manifestURL: string,
+        protected readonly lsName: string,
         private readonly supportedLspServerVersions?: string[]
     ) {}
 
-    async fetchManifest() {
+    /**
+     * Finds the latest available manifest. If it fails to download then fallback to the local
+     * manifest version
+     */
+    async downloadManifest() {
         try {
-            const resp = await request.fetch('GET', this.manifestURL, {
-                headers: {
-                    'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }),
+            const resourceFetcher = new RetryableResourceFetcher({
+                resource: this.manifestURL,
+                params: {
+                    timeout: new Timeout(15000),
                 },
-            }).response
-            if (!resp.ok) {
-                throw new ToolkitError(`Failed to fetch manifest. Error: ${resp.statusText}`)
+            })
+            const etag = (globals.globalState.tryGet('aws.toolkit.lsp.manifest', Object) as StorageManifest)?.etag
+            const resp = await resourceFetcher.getNewETagContent(etag)
+            if (resp.content === undefined) {
+                throw new ToolkitError('Content was not downloaded')
+            }
+
+            const manifest = JSON.parse(resp.content) as Manifest
+            if (manifest.isManifestDeprecated) {
+                logger.info('This LSP manifest is deprecated. No future updates will be available.')
             }
-            return resp.json()
+            globals.globalState.tryUpdate('aws.toolkit.lsp.manifest', {
+                etag: resp.eTag,
+                content: resp.content,
+            } as StorageManifest)
+            return manifest
         } catch (e: any) {
-            throw new ToolkitError(`Failed to fetch manifest. Error: ${JSON.stringify(e)}`)
+            logger.info('Failed to download latest LSP manifest. Falling back to local manifest.')
+            const manifest = globals.globalState.tryGet('aws.toolkit.lsp.manifest', Object)
+            if (!manifest) {
+                throw new ToolkitError('Failed to download LSP manifest and no local manifest found.')
+            }
+
+            if (manifest?.isManifestDeprecated) {
+                logger.info('This LSP manifest is deprecated. No future updates will be available.')
+            }
+            return manifest
         }
     }
 
@@ -95,7 +130,7 @@ export abstract class LspDownloader {
     private async hashMatch(filePath: string, content: Content) {
         const sha384 = await this.getFileSha384(filePath)
         if ('sha384:' + sha384 !== content.hashes[0]) {
-            getLogger('lsp').error(`Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.`)
+            logger.error(`Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.`)
             await fs.delete(filePath)
             return false
         }
@@ -104,11 +139,7 @@ export abstract class LspDownloader {
 
     async downloadAndCheckHash(filePath: string, content: Content) {
         await this._download(filePath, content.url)
-        const match = await this.hashMatch(filePath, content)
-        if (!match) {
-            return false
-        }
-        return true
+        return await this.hashMatch(filePath, content)
     }
 
     getDependency(manifest: Manifest, name: string): Content | undefined {
@@ -189,26 +220,117 @@ export abstract class LspDownloader {
      */
     abstract install(manifest: Manifest): Promise<boolean>
 
+    /**
+     * Get the currently installed version of the LSP on disk
+     */
+    async latestInstalledVersion(): Promise<string | undefined> {
+        const latestVersion: Record<string, string> = globals.globalState.tryGet('aws.toolkit.lsp.versions', Object)
+        if (!latestVersion || !latestVersion[this.lsName]) {
+            return undefined
+        }
+        return latestVersion[this.lsName]
+    }
+
     async tryInstallLsp(): Promise<boolean> {
         try {
-            if (await this.isLspInstalled()) {
-                getLogger('lsp').info(`LSP already installed`)
+            if (process.env.AWS_LANGUAGE_SERVER_OVERRIDE) {
+                logger.info(`LSP override location: ${process.env.AWS_LANGUAGE_SERVER_OVERRIDE}`)
                 return true
             }
 
-            const clean = await this.cleanup()
-            if (!clean) {
-                getLogger('lsp').error(`Failed to clean up old LSPs`)
-                return false
-            }
+            const manifest: Manifest = await this.downloadManifest()
 
-            // fetch download url for server and runtime
-            const manifest: Manifest = (await this.fetchManifest()) as Manifest
+            // if a compatible version was found check what's installed locally
+            if (this.latestCompatibleVersion(manifest)) {
+                return await this.checkInstalledLS(manifest)
+            }
 
-            return await this.install(manifest)
-        } catch (e) {
-            getLogger().error(`LspController: Failed to setup LSP server ${e}`)
+            // we found no latest compatible version in the manifest; try to fallback to a local version
+            return this.fallbackToLocalVersion(manifest)
+        } catch (err) {
+            const e = err as ToolkitError
+            logger.info(`Failed to setup LSP server: ${e.message}`)
             return false
         }
     }
+
+    /**
+     * Attempts to fall back to a local version if one is available
+     */
+    async fallbackToLocalVersion(manifest?: Manifest): Promise<boolean> {
+        // was language server previously downloaded?
+        const installed = await this.isLspInstalled()
+
+        // yes
+        if (installed) {
+            if (!manifest) {
+                // we want to launch if the manifest can't be found
+                return true
+            }
+
+            // the manifest is found; check that the current version is not delisted
+            const currentVersion = await this.latestInstalledVersion()
+            const v = manifest.versions.find((v) => v.serverVersion === currentVersion)
+            if (v?.isDelisted) {
+                throw new ToolkitError('Local LSP version is delisted. Please update to a newer version.')
+            }
+
+            // current version is not delisted, we should launch
+            return true
+        }
+
+        // it was not installed before
+        throw new ToolkitError('No compatible local LSP version found', { code: 'LSPNotInstalled' })
+    }
+
+    /**
+     * Check to see if we can re-use the previously downloaded language server.
+     * If it wasn't previously downloaded then download it and store it
+     * If it was then check the current installed language version
+     *  If there is an error, download the latest version and store it
+     *  If there wasn't an error, compare the current and latest versions
+     *    If they mismatch then download the latest language server and store it
+     *    If they are the same then launch the language server
+     */
+    async checkInstalledLS(manifest: Manifest): Promise<boolean> {
+        // was ls previously downloaded
+        const installed = await this.isLspInstalled()
+
+        // yes
+        if (installed) {
+            try {
+                const currentVersion = await this.latestInstalledVersion()
+                if (currentVersion !== this.latestCompatibleVersion(manifest)) {
+                    // download and install latest version
+                    return this._install(manifest)
+                }
+                return true
+            } catch (e) {
+                logger.info('Failed to query language server for installed version')
+
+                // error found! download the latest version and store it
+                return this._install(manifest)
+            }
+        }
+
+        // no; install and store it
+        return this._install(manifest)
+    }
+
+    private _install(manifest: Manifest): Promise<boolean> {
+        return this.install(manifest).catch((_) => this.fallbackToLocalVersion(manifest))
+    }
+
+    private latestCompatibleVersion(manifest: Manifest) {
+        for (const version of manifest.versions) {
+            if (version.isDelisted) {
+                continue
+            }
+            if (this.supportedLspServerVersions && !this.supportedLspServerVersions.includes(version.serverVersion)) {
+                continue
+            }
+            return version.serverVersion
+        }
+        return undefined
+    }
 }

packages/core/src/shared/globalState.ts

@@ -46,6 +46,8 @@ export type globalKey =
     | 'aws.amazonq.workspaceIndexToggleOn'
     | 'aws.toolkit.separationPromptCommand'
     | 'aws.toolkit.separationPromptDismissed'
+    | 'aws.toolkit.lsp.versions'
+    | 'aws.toolkit.lsp.manifest'
     // Deprecated/legacy names. New keys should start with "aws.".
     | '#sessionCreationDates' // Legacy name from `ssoAccessTokenProvider.ts`.
     | 'CODECATALYST_RECONNECT'

packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts

@@ -16,6 +16,7 @@ import { ResourceFetcher } from './resourcefetcher'
 import { Timeout, CancellationError, CancelEvent } from '../utilities/timeoutUtils'
 import { isCloud9 } from '../extensionUtilities'
 import { Headers } from 'got/dist/source/core'
+import { withRetries } from '../utilities/functionUtils'
 
 // XXX: patched Got module for compatability with older VS Code versions (e.g. Cloud9)
 // `got` has also deprecated `urlToOptions`
@@ -201,6 +202,50 @@ export class HttpResourceFetcher implements ResourceFetcher {
     }
 }
 
+export class RetryableResourceFetcher extends HttpResourceFetcher {
+    private readonly retryNumber: number
+    private readonly retryIntervalMs: number
+    private readonly resource: string
+
+    constructor({
+        resource,
+        params: { retryNumber = 5, retryIntervalMs = 3000, showUrl = true, timeout = new Timeout(5000) },
+    }: {
+        resource: string
+        params: {
+            retryNumber?: number
+            retryIntervalMs?: number
+            showUrl?: boolean
+            timeout?: Timeout
+        }
+    }) {
+        super(resource, {
+            showUrl,
+            timeout,
+        })
+        this.retryNumber = retryNumber
+        this.retryIntervalMs = retryIntervalMs
+        this.resource = resource
+    }
+
+    fetch(versionTag?: string) {
+        return withRetries(
+            async () => {
+                try {
+                    return await this.getNewETagContent(versionTag)
+                } catch (err) {
+                    getLogger('lsp').error('Failed to fetch at endpoint: %s, err: %s', this.resource, err)
+                    throw err
+                }
+            },
+            {
+                maxRetries: this.retryNumber,
+                delay: this.retryIntervalMs,
+            }
+        )
+    }
+}
+
 /**
  * Retrieves JSON property value from a remote resource
  * @param property property to retrieve