refactor(fs): prefer our fs module, eliminate fs-extra #5689

## Problem
We want to avoid fs-extra, and eventually move it to the banned import
list. As a first step, we remove fs-extra with as much of our own
`fs.ts` work, and `fs` when not possible.

## Solution
- Replace `ensureDir` with `mkDir` as suggested here:
https://github.com/jprichardson/node-fs-extra/blob/HEAD/docs/ensureDir.md
- Also `ensureDir` with permissions arg can be `mkDir` with `chmod`
following.
- Replace `move` with `rename` since we aren't using virtual file
system, or crossing disk partitions:
https://stackoverflow.com/questions/8579055/how-do-i-move-files-in-node-js

Author: Hweinstock

packages/core/src/amazonq/commons/diff.ts

@@ -3,26 +3,26 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { existsSync } from 'fs'
 import * as vscode from 'vscode'
 import { featureDevScheme } from '../../amazonqFeatureDev/constants'
+import { fs } from '../../shared'
 
 export async function openDiff(leftPath: string, rightPath: string, tabId: string) {
-    const { left, right } = getFileDiffUris(leftPath, rightPath, tabId)
+    const { left, right } = await getFileDiffUris(leftPath, rightPath, tabId)
     await vscode.commands.executeCommand('vscode.diff', left, right)
 }
 
 export async function openDeletedDiff(filePath: string, name: string, tabId: string) {
-    const fileUri = getOriginalFileUri(filePath, tabId)
+    const fileUri = await getOriginalFileUri(filePath, tabId)
     await vscode.commands.executeCommand('vscode.open', fileUri, {}, `${name} (Deleted)`)
 }
 
-export function getOriginalFileUri(fullPath: string, tabId: string) {
-    return existsSync(fullPath) ? vscode.Uri.file(fullPath) : createAmazonQUri('empty', tabId)
+export async function getOriginalFileUri(fullPath: string, tabId: string) {
+    return (await fs.exists(fullPath)) ? vscode.Uri.file(fullPath) : createAmazonQUri('empty', tabId)
 }
 
-export function getFileDiffUris(leftPath: string, rightPath: string, tabId: string) {
-    const left = getOriginalFileUri(leftPath, tabId)
+export async function getFileDiffUris(leftPath: string, rightPath: string, tabId: string) {
+    const left = await getOriginalFileUri(leftPath, tabId)
     const right = createAmazonQUri(rightPath, tabId)
 
     return { left, right }

packages/core/src/amazonq/lsp/lspController.ts

@@ -5,7 +5,7 @@
 
 import * as vscode from 'vscode'
 import * as path from 'path'
-import * as fs from 'fs-extra'
+import { createWriteStream } from 'fs'
 import * as crypto from 'crypto'
 import { getLogger } from '../../shared/logger/logger'
 import { CurrentWsFolders, collectFilesForIndex } from '../../shared/utilities/workspaceUtils'
@@ -19,12 +19,11 @@ import { CodeWhispererSettings } from '../../codewhisperer/util/codewhispererSet
 import { activate as activateLsp } from './lspClient'
 import { telemetry } from '../../shared/telemetry'
 import { isCloud9 } from '../../shared/extensionUtilities'
-import { globals, ToolkitError } from '../../shared'
+import { fs, globals, ToolkitError } from '../../shared'
 import { AuthUtil } from '../../codewhisperer'
 import { isWeb } from '../../shared/extensionGlobals'
 import { getUserAgent } from '../../shared/telemetry/util'
 import { isAmazonInternalOs } from '../../shared/vscode/env'
-import { fs as fs2 } from '../../shared/fs/fs'
 
 function getProjectPaths() {
     const workspaceFolders = vscode.workspace.workspaceFolders
@@ -106,7 +105,7 @@ export class LspController {
             throw new ToolkitError(`Failed to download. Error: ${JSON.stringify(res)}`)
         }
         return new Promise((resolve, reject) => {
-            const file = fs.createWriteStream(localFile)
+            const file = createWriteStream(localFile)
             res.body.pipe(file)
             res.body.on('error', (err) => {
                 reject(err)
@@ -134,16 +133,16 @@ export class LspController {
     }
 
     async getFileSha384(filePath: string): Promise<string> {
-        const fileBuffer = await fs.promises.readFile(filePath)
+        const fileBuffer = await fs.readFile(filePath)
         const hash = crypto.createHash('sha384')
         hash.update(fileBuffer)
         return hash.digest('hex')
     }
 
-    isLspInstalled(context: vscode.ExtensionContext) {
+    async isLspInstalled(context: vscode.ExtensionContext) {
         const localQServer = context.asAbsolutePath(path.join('resources', 'qserver'))
         const localNodeRuntime = context.asAbsolutePath(path.join('resources', nodeBinName))
-        return fs.existsSync(localQServer) && fs.existsSync(localNodeRuntime)
+        return (await fs.exists(localQServer)) && (await fs.exists(localNodeRuntime))
     }
 
     getQserverFromManifest(manifest: Manifest): Content | undefined {
@@ -208,7 +207,7 @@ export class LspController {
             getLogger().error(
                 `LspController: Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.`
             )
-            fs.removeSync(filePath)
+            await fs.delete(filePath)
             return false
         }
         return true
@@ -226,19 +225,19 @@ export class LspController {
     async tryInstallLsp(context: vscode.ExtensionContext): Promise<boolean> {
         let tempFolder = undefined
         try {
-            if (this.isLspInstalled(context)) {
+            if (await this.isLspInstalled(context)) {
                 getLogger().info(`LspController: LSP already installed`)
                 return true
             }
             // clean up previous downloaded LSP
             const qserverPath = context.asAbsolutePath(path.join('resources', 'qserver'))
-            if (fs.existsSync(qserverPath)) {
+            if (await fs.exists(qserverPath)) {
                 await tryRemoveFolder(qserverPath)
             }
             // clean up previous downloaded node runtime
             const nodeRuntimePath = context.asAbsolutePath(path.join('resources', nodeBinName))
-            if (fs.existsSync(nodeRuntimePath)) {
-                fs.rmSync(nodeRuntimePath)
+            if (await fs.exists(nodeRuntimePath)) {
+                await fs.delete(nodeRuntimePath)
             }
             // fetch download url for qserver and node runtime
             const manifest: Manifest = (await this.fetchManifest()) as Manifest
@@ -259,16 +258,16 @@ export class LspController {
             }
             const zip = new AdmZip(qserverZipTempPath)
             zip.extractAllTo(tempFolder)
-            fs.moveSync(path.join(tempFolder, 'qserver'), qserverPath)
+            await fs.rename(path.join(tempFolder, 'qserver'), qserverPath)
 
             // download node runtime to temp folder
             const nodeRuntimeTempPath = path.join(tempFolder, nodeBinName)
             const downloadNodeOk = await this.downloadAndCheckHash(nodeRuntimeTempPath, nodeRuntimeContent)
             if (!downloadNodeOk) {
                 return false
             }
-            await fs2.chmod(nodeRuntimeTempPath, 0o755)
-            fs.moveSync(nodeRuntimeTempPath, nodeRuntimePath)
+            await fs.chmod(nodeRuntimeTempPath, 0o755)
+            await fs.rename(nodeRuntimeTempPath, nodeRuntimePath)
             return true
         } catch (e) {
             getLogger().error(`LspController: Failed to setup LSP server ${e}`)

packages/core/src/awsService/iot/commands/createCert.ts

@@ -4,14 +4,14 @@
  */
 
 import * as vscode from 'vscode'
-import * as fs from 'fs-extra'
 import * as path from 'path'
 import { getLogger } from '../../../shared/logger'
 import { localize } from '../../../shared/utilities/vsCodeUtils'
 import { showViewLogsMessage } from '../../../shared/utilities/messages'
 import { IotCertsFolderNode } from '../explorer/iotCertFolderNode'
 import { fileExists } from '../../../shared/filesystemUtilities'
 import { Iot } from 'aws-sdk'
+import { fs } from '../../../shared'
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const MODE_RW_R_R = 0o644 //File permission 0644 rw-r--r-- for PEM files.

packages/core/src/awsService/iot/commands/createPolicy.ts

@@ -4,11 +4,11 @@
  */
 
 import * as vscode from 'vscode'
-import * as fs from 'fs-extra'
 import { getLogger } from '../../../shared/logger'
 import { localize } from '../../../shared/utilities/vsCodeUtils'
 import { showViewLogsMessage } from '../../../shared/utilities/messages'
 import { IotPolicyFolderNode } from '../explorer/iotPolicyFolderNode'
+import { fs } from '../../../shared'
 
 /**
  * Creates a policy from a policy document.
@@ -49,7 +49,7 @@ export async function createPolicyCommand(node: IotPolicyFolderNode, getPolicyDo
     await node.refreshNode()
 }
 
-export async function getPolicyDocument(): Promise<Buffer | undefined> {
+export async function getPolicyDocument(): Promise<Uint8Array | undefined> {
     const fileLocation = await vscode.window.showOpenDialog({
         canSelectFolders: false,
         canSelectFiles: true,
@@ -64,7 +64,7 @@ export async function getPolicyDocument(): Promise<Buffer | undefined> {
 
     const policyLocation = fileLocation[0]
 
-    let data: Buffer
+    let data: Uint8Array
     try {
         data = await fs.readFile(policyLocation.fsPath)
     } catch (e) {

packages/core/src/codecatalyst/model.ts

@@ -18,7 +18,6 @@ import { DevEnvClient } from '../shared/clients/devenvClient'
 import { getLogger } from '../shared/logger'
 import { AsyncCollection, toCollection } from '../shared/utilities/asyncCollection'
 import { getCodeCatalystSpaceName, getCodeCatalystProjectName, getCodeCatalystDevEnvId } from '../shared/vscode/env'
-import { writeFile } from 'fs-extra'
 import { sshAgentSocketVariable, startSshAgent, startVscodeRemote } from '../shared/extensions/ssh'
 import { ChildProcess } from '../shared/utilities/processUtils'
 import { isDevenvVscode } from './utils'
@@ -31,6 +30,7 @@ import { ToolkitError } from '../shared/errors'
 import { Result } from '../shared/utilities/result'
 import { VscodeRemoteConnection, ensureDependencies } from '../shared/remoteSession'
 import { SshConfig, sshLogFileLocation } from '../shared/sshConfig'
+import { fs } from '../shared'
 
 export type DevEnvironmentId = Pick<DevEnvironment, 'id' | 'org' | 'project'>
 export const connectScriptPrefix = 'codecatalyst_connect'
@@ -134,7 +134,7 @@ export function createBoundProcess(envProvider: EnvProvider): typeof ChildProces
 }
 
 export async function cacheBearerToken(bearerToken: string, devenvId: string): Promise<void> {
-    await writeFile(bearerTokenCacheLocation(devenvId), `${bearerToken}`, 'utf8')
+    await fs.writeFile(bearerTokenCacheLocation(devenvId), `${bearerToken}`, 'utf8')
 }
 
 export function bearerTokenCacheLocation(devenvId: string): string {

packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import * as vscode from 'vscode'
-import * as fs from 'fs'
+import * as nodefs from 'fs'
 import * as path from 'path'
 import * as os from 'os'
 import * as codeWhisperer from '../../client/codewhisperer'
@@ -44,7 +44,7 @@ import { AuthUtil } from '../../util/authUtil'
 import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient'
 import { downloadExportResultArchive } from '../../../shared/utilities/download'
 import { ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming'
-import fs2 from '../../../shared/fs/fs'
+import fs from '../../../shared/fs/fs'
 import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession'
 import { convertToTimeString, encodeHTML } from '../../../shared/utilities/textUtilities'
 
@@ -109,7 +109,7 @@ export async function uploadArtifactToS3(
 ) {
     throwIfCancelled()
     try {
-        const uploadFileByteSize = (await fs.promises.stat(fileName)).size
+        const uploadFileByteSize = (await nodefs.promises.stat(fileName)).size
         getLogger().info(
             `Uploading project artifact at %s with checksum %s using uploadId: %s and size %s kB`,
             fileName,
@@ -177,7 +177,7 @@ export async function stopJob(jobId: string) {
 }
 
 export async function uploadPayload(payloadFileName: string, uploadContext?: UploadContext) {
-    const buffer = fs.readFileSync(payloadFileName)
+    const buffer = Buffer.from(await fs.readFile(payloadFileName))
     const sha256 = getSha256(buffer)
 
     throwIfCancelled()
@@ -249,7 +249,7 @@ function isExcludedDependencyFile(path: string): boolean {
  * getFilesRecursively on the source code folder.
  */
 function getFilesRecursively(dir: string, isDependenciesFolder: boolean): string[] {
-    const entries = fs.readdirSync(dir, { withFileTypes: true })
+    const entries = nodefs.readdirSync(dir, { withFileTypes: true })
     const files = entries.flatMap((entry) => {
         const res = path.resolve(dir, entry.name)
         // exclude 'target' directory from ZIP (except if zipping dependencies) due to issues in backend
@@ -301,22 +301,22 @@ export async function zipCode({ dependenciesFolder, humanInTheLoopFlag, modulePa
             const sourceFiles = getFilesRecursively(modulePath, false)
             let sourceFilesSize = 0
             for (const file of sourceFiles) {
-                if (fs.statSync(file).isDirectory()) {
+                if (nodefs.statSync(file).isDirectory()) {
                     getLogger().info('CodeTransformation: Skipping directory, likely a symlink')
                     continue
                 }
                 const relativePath = path.relative(modulePath, file)
                 const paddedPath = path.join('sources', relativePath)
                 zip.addLocalFile(file, path.dirname(paddedPath))
-                sourceFilesSize += (await fs.promises.stat(file)).size
+                sourceFilesSize += (await nodefs.promises.stat(file)).size
             }
             getLogger().info(`CodeTransformation: source code files size = ${sourceFilesSize}`)
         }
 
         throwIfCancelled()
 
         let dependencyFiles: string[] = []
-        if (fs.existsSync(dependenciesFolder.path)) {
+        if (await fs.exists(dependenciesFolder.path)) {
             dependencyFiles = getFilesRecursively(dependenciesFolder.path, true)
         }
 
@@ -330,7 +330,7 @@ export async function zipCode({ dependenciesFolder, humanInTheLoopFlag, modulePa
                 // const paddedPath = path.join(`dependencies/${dependenciesFolder.name}`, relativePath)
                 const paddedPath = path.join(`dependencies/`, relativePath)
                 zip.addLocalFile(file, path.dirname(paddedPath))
-                dependencyFilesSize += (await fs.promises.stat(file)).size
+                dependencyFilesSize += (await nodefs.promises.stat(file)).size
             }
             getLogger().info(`CodeTransformation: dependency files size = ${dependencyFilesSize}`)
             dependenciesCopied = true
@@ -353,19 +353,19 @@ export async function zipCode({ dependenciesFolder, humanInTheLoopFlag, modulePa
         }
 
         tempFilePath = path.join(os.tmpdir(), 'zipped-code.zip')
-        fs.writeFileSync(tempFilePath, zip.toBuffer())
-        if (fs.existsSync(dependenciesFolder.path)) {
-            fs.rmSync(dependenciesFolder.path, { recursive: true, force: true })
+        await fs.writeFile(tempFilePath, zip.toBuffer())
+        if (await fs.exists(dependenciesFolder.path)) {
+            await fs.delete(dependenciesFolder.path, { recursive: true, force: true })
         }
     } catch (e: any) {
         throw Error('Failed to zip project')
     } finally {
         if (logFilePath) {
-            fs.rmSync(logFilePath)
+            await fs.delete(logFilePath)
         }
     }
 
-    const zipSize = (await fs.promises.stat(tempFilePath)).size
+    const zipSize = (await nodefs.promises.stat(tempFilePath)).size
 
     const exceedsLimit = zipSize > CodeWhispererConstants.uploadZipSizeLimitInBytes
 
@@ -409,7 +409,7 @@ export async function startJob(uploadId: string) {
 }
 
 export function getImageAsBase64(filePath: string) {
-    const fileContents = fs.readFileSync(filePath, { encoding: 'base64' })
+    const fileContents = nodefs.readFileSync(filePath, { encoding: 'base64' })
     return `data:image/svg+xml;base64,${fileContents}`
 }
 
@@ -725,9 +725,9 @@ export async function downloadAndExtractResultArchive(
     pathToArchiveDir: string,
     downloadArtifactType: TransformationDownloadArtifactType
 ) {
-    const archivePathExists = await fs2.existsDir(pathToArchiveDir)
+    const archivePathExists = await fs.existsDir(pathToArchiveDir)
     if (!archivePathExists) {
-        await fs2.mkdir(pathToArchiveDir)
+        await fs.mkdir(pathToArchiveDir)
     }
 
     const pathToArchive = path.join(pathToArchiveDir, 'ExportResultsArchive.zip')

packages/core/src/dev/codecatalyst.ts

@@ -4,7 +4,6 @@
  */
 
 import { glob } from 'glob'
-import * as fs from 'fs-extra'
 import * as path from 'path'
 import * as vscode from 'vscode'
 import * as manifest from '../../package.json'
@@ -21,6 +20,7 @@ import { startVscodeRemote } from '../shared/extensions/ssh'
 import { isValidResponse } from '../shared/wizards/wizard'
 import { createQuickPick } from '../shared/ui/pickerPrompter'
 import { createCommonButtons } from '../shared/ui/buttons'
+import { fs } from '../shared'
 
 type LazyProgress<T> = vscode.Progress<T> & vscode.Disposable & { getToken(): Timeout }
 
@@ -217,7 +217,7 @@ async function installVsix(
     if (path.extname(resp) !== '.vsix') {
         progress.report({ message: 'Copying extension...' })
 
-        const packageData = await fs.readFile(path.join(resp, 'package.json'), 'utf-8')
+        const packageData = await fs.readFileAsString(path.join(resp, 'package.json'))
         const targetManfiest: typeof manifest = JSON.parse(packageData)
         const destName = `${extPath}/${extId}-${targetManfiest.version}`
         const source = `${resp}${path.sep}`

packages/core/src/dynamicResources/awsResourceManager.ts

@@ -4,7 +4,6 @@
  */
 
 import { writeFileSync } from 'fs'
-import { remove } from 'fs-extra'
 import * as path from 'path'
 import * as vscode from 'vscode'
 import { CloudFormationClient } from '../shared/clients/cloudFormationClient'
@@ -20,6 +19,7 @@ import { ResourceNode } from './explorer/nodes/resourceNode'
 import { ResourceTypeNode } from './explorer/nodes/resourceTypeNode'
 import { isCloud9 } from '../shared/extensionUtilities'
 import globals from '../shared/extensionGlobals'
+import { fs } from '../shared'
 
 export const resourceFileGlobPattern = '**/*.awsResource.json'
 
@@ -97,7 +97,7 @@ export class AwsResourceManager {
             }
 
             if (uri.scheme === 'file') {
-                await remove(uri.fsPath)
+                await fs.delete(uri.fsPath)
 
                 globals.schemaService.registerMapping({
                     uri,