perf(sam): skip files excluded by user settings

Problem:
"Scanning CloudFormation templates..." takes a long time on big
workspaces, and there is no workaround. #3510

Solution:
When scanning for CFN/SAM templates, exclude all directories/files
specified in _all_ of the vscode settings `files.exclude`,
`search.exclude`, or `files.watcherExclude`.

Author: justinmk3

.changes/next-release/Feature-f2e37981-fba6-4a29-bdf3-4e06fa4dfa2f.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "SAM template detection now skips directories specified in [user settings](https://code.visualstudio.com/docs/getstarted/settings) `files.exclude`, `search.exclude`, or `files.watcherExclude`. This improves performance on big workspaces and avoids the \"Scanning CloudFormation templates...\" message. [#3510](https://github.com/aws/aws-toolkit-vscode/issues/3510)"
+}

src/shared/fs/watchedFiles.ts

@@ -7,14 +7,27 @@ import * as vscode from 'vscode'
 import { getLogger } from '../logger/logger'
 import * as pathutils from '../utilities/pathUtils'
 import * as path from 'path'
-import { isUntitledScheme, normalizeVSCodeUri } from '../utilities/vsCodeUtils'
+import { globDirs, isUntitledScheme, normalizeVSCodeUri } from '../utilities/vsCodeUtils'
+import { Settings } from '../settings'
+import { once } from '../utilities/functionUtils'
 
 /**
  * Prevent `findFiles()` from recursing into these directories.
  *
  * `findFiles()` defaults to the vscode `files.exclude` setting, which by default does not exclude "node_modules/".
  */
-const alwaysExclude = '**/{.aws-sam,.git,.svn,.hg,.rvm,.gem,.project,node_modules,venv,bower_components}/'
+const alwaysExclude = {
+    '.aws-sam': true,
+    '.git': true,
+    '.svn': true,
+    '.hg': true,
+    '.rvm': true,
+    '.gem': true,
+    '.project': true,
+    node_modules: true,
+    venv: true,
+    bower_components: true,
+}
 
 export interface WatchedItem<T> {
     /**
@@ -27,6 +40,21 @@ export interface WatchedItem<T> {
     item: T
 }
 
+/** Builds an exclude pattern based on vscode global settings and the `alwaysExclude` default. */
+export function getExcludePattern() {
+    const vscodeFilesExclude = Settings.instance.get<object>('files.exclude', Object, {})
+    const vscodeSearchExclude = Settings.instance.get<object>('search.exclude', Object, {})
+    const vscodeWatcherExclude = Settings.instance.get<object>('files.watcherExclude', Object, {})
+    const all = [
+        ...Object.keys(alwaysExclude),
+        ...Object.keys(vscodeFilesExclude),
+        ...Object.keys(vscodeSearchExclude),
+        ...Object.keys(vscodeWatcherExclude),
+    ]
+    return globDirs(all)
+}
+const getExcludePatternOnce = once(getExcludePattern)
+
 /**
  * WatchedFiles lets us index files in the current registry. It is used
  * for CFN templates among other things. WatchedFiles holds a list of pairs of
@@ -230,10 +258,19 @@ export abstract class WatchedFiles<T> implements vscode.Disposable {
      */
     public async rebuild(): Promise<void> {
         this.reset()
+
+        const exclude = getExcludePatternOnce()
         for (const glob of this.globs) {
-            const itemUris = await vscode.workspace.findFiles(glob, alwaysExclude)
-            for (const item of itemUris) {
-                await this.addItemToRegistry(item, true)
+            try {
+                const found = await vscode.workspace.findFiles(glob, exclude)
+                for (const item of found) {
+                    await this.addItemToRegistry(item, true)
+                }
+            } catch (e) {
+                const err = e as Error
+                if (err.name !== 'Canceled') {
+                    getLogger().error('watchedFiles: findFiles("%s", "%s"): %s', glob, exclude, err.message)
+                }
             }
         }
     }

src/shared/logger/activation.ts

@@ -48,7 +48,7 @@ export async function activate(
     )
 
     setLogger(mainLogger)
-    getLogger().error(`log level: ${getLogLevel()}`)
+    getLogger().info(`log level: ${getLogLevel()}`)
 
     // channel logger
     setLogger(

src/shared/utilities/vsCodeUtils.ts

@@ -169,6 +169,30 @@ export function isUntitledScheme(uri: vscode.Uri): boolean {
     return uri.scheme === 'untitled'
 }
 
+/**
+ * Creates a glob pattern that matches all directories specified in `dirs`.
+ *
+ * "/" and "*" chars are trimmed from `dirs` items, and the final glob is defined such that the
+ * directories and their contents are matched any _any_ depth.
+ *
+ * Example: `['foo', '**\/bar/'] => "**\/{foo,bar}/"`
+ */
+export function globDirs(dirs: string[]): string {
+    const excludePatternsStr = dirs.reduce((prev, current) => {
+        // Trim all "*" and "/" chars.
+        // Note that the replace() patterns and order is intentionaly so that "**/*foo*/**" yields "*foo*".
+        const scrubbed = current
+            .replace(/^\**/, '')
+            .replace(/^[/\\]*/, '')
+            .replace(/\**$/, '')
+            .replace(/[/\\]*$/, '')
+        const comma = prev === '' ? '' : ','
+        return `${prev}${comma}${scrubbed}`
+    }, '')
+    const excludePattern = `**/{${excludePatternsStr}}/`
+    return excludePattern
+}
+
 // If the VSCode URI is not a file then return the string representation, otherwise normalize the filesystem path
 export function normalizeVSCodeUri(uri: vscode.Uri): string {
     if (uri.scheme !== 'file') {

src/test/shared/utilities/vscodeUtils.test.ts

@@ -7,6 +7,7 @@ import * as assert from 'assert'
 import { VSCODE_EXTENSION_ID } from '../../../shared/extensions'
 import * as vscodeUtil from '../../../shared/utilities/vsCodeUtils'
 import * as vscode from 'vscode'
+import { getExcludePattern } from '../../../shared/fs/watchedFiles'
 
 describe('vscodeUtils', async function () {
     it('activateExtension(), isExtensionActive()', async function () {
@@ -20,6 +21,17 @@ describe('vscodeUtils', async function () {
         await vscodeUtil.activateExtension(VSCODE_EXTENSION_ID.awstoolkit, false)
         assert.deepStrictEqual(vscodeUtil.isExtensionActive(VSCODE_EXTENSION_ID.awstoolkit), true)
     })
+
+    it('globDirs()', async function () {
+        const input = ['foo', '**/bar/**', '*baz*', '**/*with.star*/**', '/zub', 'zim/', '/zoo/']
+        assert.deepStrictEqual(vscodeUtil.globDirs(input), '**/{foo,bar,baz,*with.star*,zub,zim,zoo}/')
+    })
+
+    it('watchedFiles.getExcludePattern()', async function () {
+        // If vscode defaults change in the future, just update this test.
+        // We intentionally want visibility into real-world defaults.
+        assert.match(getExcludePattern(), /node_modules,bower_components,\*\.code-search,/)
+    })
 })
 
 describe('isExtensionInstalled()', function () {