feat(utils): Add some util functions for upcoming feature (aws#5657)

This adds some util functions that are needed for an upcoming feature.

Each commit is its own isolated util, so review them individually.

**A significant amount of the changes come from minor refactors in
files**

---

<!--- REMINDER: Ensure that your PR meets the guidelines in
CONTRIBUTING.md -->

License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Signed-off-by: nkomonen-amazon <[email protected]>

Author: nkomonen-amazon

package-lock.json

@@ -4103,6 +4103,7 @@
         "@types/node": "^16.18.95",
         "@types/node-fetch": "^2.6.8",
         "@types/prismjs": "^1.26.0",
+        "@types/proper-lockfile": "^4.1.4",
         "@types/readline-sync": "^1.4.3",
         "@types/semver": "^7.5.0",
         "@types/sinon": "^10.0.5",
@@ -4182,6 +4183,8 @@
         "mime-types": "^2.1.32",
         "node-fetch": "^2.7.0",
         "portfinder": "^1.0.32",
+        "proper-lockfile": "^4.1.2",
+        "ps-list": "^8.1.1",
         "semver": "^7.5.4",
         "stream-buffers": "^3.0.2",
         "strip-ansi": "^5.2.0",

packages/core/package.json

@@ -41,6 +41,10 @@ export function productName() {
     return isAmazonQ() ? 'Amazon Q' : `${getIdeProperties().company} Toolkit`
 }
 
+export const getExtensionId = () => {
+    return isAmazonQ() ? VSCODE_EXTENSION_ID.amazonq : VSCODE_EXTENSION_ID.awstoolkit
+}
+
 /** Gets the "AWS" or "Amazon Q" prefix (in package.json: `commands.category`). */
 export function commandsPrefix(): string {
     return isAmazonQ() ? 'Amazon Q' : getIdeProperties().company

packages/core/src/shared/extensionUtilities.ts

@@ -23,6 +23,7 @@ import crypto from 'crypto'
 import { waitUntil } from '../utilities/timeoutUtils'
 import { telemetry } from '../telemetry/telemetry'
 import { getLogger } from '../logger/logger'
+import { toUri } from '../utilities/uriUtils'
 
 const vfs = vscode.workspace.fs
 type Uri = vscode.Uri
@@ -88,7 +89,7 @@ export class FileSystem {
 
     /** Creates the directory as well as missing parent directories. */
     async mkdir(path: Uri | string): Promise<void> {
-        const uri = this.#toUri(path)
+        const uri = toUri(path)
         const errHandler = createPermissionsErrorHandler(this.isWeb, vscode.Uri.joinPath(uri, '..'), '*wx')
 
         // Certain URIs are not supported with vscode.workspace.fs in Cloud9
@@ -105,7 +106,7 @@ export class FileSystem {
 
     // TODO: rename to readFileBytes()?
     async readFile(path: Uri | string): Promise<Uint8Array> {
-        const uri = this.#toUri(path)
+        const uri = toUri(path)
         const errHandler = createPermissionsErrorHandler(this.isWeb, uri, 'r**')
 
         if (isCloud9()) {
@@ -117,7 +118,7 @@ export class FileSystem {
 
     // TODO: rename to readFile()?
     async readFileAsString(path: Uri | string, decoder: TextDecoder = FileSystem.#decoder): Promise<string> {
-        const uri = this.#toUri(path)
+        const uri = toUri(path)
         const bytes = await this.readFile(uri)
         return decoder.decode(bytes)
     }
@@ -127,7 +128,7 @@ export class FileSystem {
      * so we must do it ourselves (this implementation is inefficient).
      */
     async appendFile(path: Uri | string, content: Uint8Array | string): Promise<void> {
-        path = this.#toUri(path)
+        path = toUri(path)
 
         const currentContent: Uint8Array = (await this.existsFile(path)) ? await this.readFile(path) : new Uint8Array(0)
         const currentLength = currentContent.length
@@ -146,7 +147,7 @@ export class FileSystem {
         if (path === undefined || path === '') {
             return false
         }
-        const uri = this.#toUri(path)
+        const uri = toUri(path)
         if (uri.fsPath === undefined || uri.fsPath === '') {
             return false
         }
@@ -212,7 +213,7 @@ export class FileSystem {
         data: string | Uint8Array,
         opts?: WriteFileOptions & { atomic?: boolean }
     ): Promise<void> {
-        const uri = this.#toUri(path)
+        const uri = toUri(path)
         const errHandler = createPermissionsErrorHandler(this.isWeb, uri, '*w*')
         const content = this.#toBytes(data)
 
@@ -249,7 +250,7 @@ export class FileSystem {
             // 3. Finally, do a regular file write, but may result in invalid file content
             //
             // For telemetry, we will only report failures as to not overload with succeed events.
-            const tempFile = this.#toUri(`${uri.fsPath}.${crypto.randomBytes(8).toString('hex')}.tmp`)
+            const tempFile = toUri(`${uri.fsPath}.${crypto.randomBytes(8).toString('hex')}.tmp`)
             try {
                 await write(tempFile)
                 await fs.rename(tempFile, uri)
@@ -290,8 +291,8 @@ export class FileSystem {
     }
 
     async rename(oldPath: vscode.Uri | string, newPath: vscode.Uri | string) {
-        const oldUri = this.#toUri(oldPath)
-        const newUri = this.#toUri(newPath)
+        const oldUri = toUri(oldPath)
+        const newUri = toUri(newPath)
         const errHandler = createPermissionsErrorHandler(this.isWeb, oldUri, 'rw*')
 
         if (isCloud9()) {
@@ -352,7 +353,7 @@ export class FileSystem {
      * The stat of the file,  throws if the file does not exist or on any other error.
      */
     async stat(uri: vscode.Uri | string): Promise<vscode.FileStat> {
-        const path = this.#toUri(uri)
+        const path = toUri(uri)
         return await vfs.stat(path)
     }
 
@@ -370,7 +371,7 @@ export class FileSystem {
     async delete(fileOrDir: string | vscode.Uri, opt_?: { recursive?: boolean; force?: boolean }): Promise<void> {
         const opt = { ...opt_, recursive: !!opt_?.recursive }
         opt.force = opt.force === false ? opt.force : !!(opt.force || opt.recursive)
-        const uri = this.#toUri(fileOrDir)
+        const uri = toUri(fileOrDir)
         const parent = vscode.Uri.joinPath(uri, '..')
         const errHandler = createPermissionsErrorHandler(this.isWeb, parent, '*wx')
 
@@ -427,7 +428,7 @@ export class FileSystem {
     }
 
     async readdir(uri: vscode.Uri | string): Promise<[string, vscode.FileType][]> {
-        const path = this.#toUri(uri)
+        const path = toUri(uri)
 
         // readdir is not a supported vscode API in Cloud9
         if (isCloud9()) {
@@ -441,8 +442,8 @@ export class FileSystem {
     }
 
     async copy(source: vscode.Uri | string, target: vscode.Uri | string): Promise<void> {
-        const sourcePath = this.#toUri(source)
-        const targetPath = this.#toUri(target)
+        const sourcePath = toUri(source)
+        const targetPath = toUri(target)
         return await vfs.copy(sourcePath, targetPath, { overwrite: true })
     }
 
@@ -454,7 +455,7 @@ export class FileSystem {
     async checkPerms(file: string | vscode.Uri, perms: PermissionsTriplet): Promise<void> {
         // TODO: implement checkExactPerms() by checking the file mode.
         // public static async checkExactPerms(file: string | vscode.Uri, perms: `${PermissionsTriplet}${PermissionsTriplet}${PermissionsTriplet}`)
-        const uri = this.#toUri(file)
+        const uri = toUri(file)
         const errHandler = createPermissionsErrorHandler(this.isWeb, uri, perms)
         const flags = Array.from(perms) as (keyof typeof this.modeMap)[]
         const mode = flags.reduce((m, f) => m | this.modeMap[f], nodeConstants.F_OK)
@@ -634,19 +635,6 @@ export class FileSystem {
         return data
     }
 
-    /**
-     * Retrieve the Uri of the file.
-     *
-     * @param path The file path for which to retrieve metadata.
-     * @return The Uri about the file.
-     */
-    #toUri(path: string | vscode.Uri): vscode.Uri {
-        if (path instanceof vscode.Uri) {
-            return path
-        }
-        return vscode.Uri.file(path)
-    }
-
     private get modeMap() {
         return {
             '*': 0,

packages/core/src/shared/fs/fs.ts

@@ -0,0 +1,76 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import vscode from 'vscode'
+import _path from 'path'
+import _fsNode from 'fs/promises'
+import lockFile, { LockOptions } from 'proper-lockfile'
+import { ToolkitError } from '../../errors'
+import { toUri } from '../../utilities/uriUtils'
+
+/**
+ * FileSystem related methods that only work in Node.js
+ */
+export class NodeFileSystem {
+    protected constructor() {}
+    static #instance: NodeFileSystem
+    static get instance(): NodeFileSystem {
+        return (this.#instance ??= new NodeFileSystem())
+    }
+
+    /**
+     * Acquires a lock on the given file, then runs the given callback, releasing the lock when done.
+     *
+     * The reason this is node specific:
+     *   - The proper-lockfile module uses the mkdir() method for lockfiles as it behaves as an atomic exists + make lockfile
+     *   - But the VSC Filesystem implementation does not throw an error when the lock file (directory) already exists,
+     *     so we need to use the node fs module, which does not work in web.
+     *
+     * @param uri The uri of the file to lock
+     * @param lockId Some way to identify who acquired the lock
+     * @param callback The callback to run once the lock is acquired
+     */
+    public async lock(uri: vscode.Uri | string, callback: () => Promise<void>): Promise<void> {
+        let release = undefined
+        try {
+            try {
+                const path = toUri(uri).fsPath
+                release = await lockFile.lock(path, this.lockOptions)
+            } catch (err) {
+                if (!(err instanceof Error)) {
+                    throw err
+                }
+                throw ToolkitError.chain(err, `Failed in lock: ${uri}`, { code: 'NodeLockError' })
+            }
+
+            try {
+                return await callback()
+            } catch (err) {
+                if (!(err instanceof Error)) {
+                    throw err
+                }
+                throw ToolkitError.chain(err, `Failed in callback of lock: ${uri}`, { code: 'NodeLockError' })
+            }
+        } finally {
+            await release?.()
+        }
+    }
+
+    protected get lockOptions() {
+        const options: LockOptions = {
+            stale: 5000, // lockfile becomes stale after this many millis
+            update: 1000, // update lockfile mtime every this many millis, useful for a long running callback that exceeds the time
+            retries: {
+                maxRetryTime: 10_000, // How long to try to acquire the lock before giving up
+                minTimeout: 100, // How long to wait between each retrying, but changes with exponential backoff
+                factor: 2, // Exponential backoff (doubles each retry)
+            },
+        }
+        return options
+    }
+}
+
+export const fsNode = NodeFileSystem.instance
+export default NodeFileSystem.instance

packages/core/src/shared/fs/node/fs.ts

@@ -58,6 +58,7 @@ type globalKey =
     | 'isExtensionFirstUse'
     | 'lastExtensionVersion'
     | 'lastSelectedRegion'
+    | 'lastOsStartTime'
     | 'recentCredentials'
     // List of regions enabled in AWS Explorer.
     | 'region'