Amazon Q: Greatly reduce indexing time for at workspace feature (#4784)

* Amazon Q: Greatly reduce indexing time for at workspace feature

1. Optimize the file traversing logic to do early pruning
and remove redundant checks
2. Add .venv to the ignore pattern
3. Only add files to the indexing results not directories
4. In a testing project reduced the indexing time from 4m30s to 15s
5. Also did the same optimiztaion to /dev zip file traversing.

* addressing comments

* detektMain and addressing comments

Author: andrewyuq

.changes/next-release/bugfix-97b2d317-4d76-4518-bcfb-20d359e1873b.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "Amazon Q: Optimized the workspace indexing logic which makes the indexing time now only 5-10% of what it was before."
+}

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/editor/context/project/ProjectContextProvider.kt

@@ -10,8 +10,10 @@ import com.fasterxml.jackson.module.kotlin.readValue
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.guessProjectDir
-import com.intellij.openapi.vcs.changes.ChangeListManager
-import com.intellij.openapi.vfs.VfsUtil
+import com.intellij.openapi.vfs.VfsUtilCore
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileVisitor
+import com.intellij.openapi.vfs.isFile
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
@@ -155,6 +157,7 @@ class ProjectContextProvider(val project: Project, private val encoderServer: En
         logger.info { "project context index response code: ${connection.responseCode} for ${project.name}" }
         val duration = (System.currentTimeMillis() - indexStartTime).toDouble()
         val startUrl = getStartUrl(project)
+        logger.debug { "project context index time: ${duration}ms" }
         if (connection.responseCode == 200) {
             val usage = getUsage()
             TelemetryHelper.recordIndexWorkspace(duration, filesResult.files.size, filesResult.fileSize, true, usage?.memoryUsage, usage?.cpuUsage, startUrl)
@@ -261,23 +264,36 @@ class ProjectContextProvider(val project: Project, private val encoderServer: En
     }
 
     private fun isBuildOrBin(filePath: String): Boolean {
-        val regex = Regex("""[/\\](bin|build|node_modules|venv|env|\.idea)[/\\]""", RegexOption.IGNORE_CASE)
+        val regex = Regex("""[/\\](bin|build|node_modules|venv|.venv|env|\.idea|.conda)[/\\]""", RegexOption.IGNORE_CASE)
         return regex.find(filePath) != null
     }
 
     private fun collectFiles(): FileCollectionResult {
         val collectedFiles = mutableListOf<String>()
         var currentTotalFileSize = 0L
-        val changeListManager = ChangeListManager.getInstance(project)
         val featureDevSessionContext = FeatureDevSessionContext(project)
-        val allFiles = project.guessProjectDir()?.let {
-            VfsUtil.collectChildrenRecursively(it).filter { child ->
-                !changeListManager.isIgnoredFile(child) &&
-                    !isBuildOrBin(child.path) &&
-                    runBlocking { !featureDevSessionContext.ignoreFile(child, scope) } &&
-                    child.length <= 10 * 1024 * 1024
-            }
-        }.orEmpty()
+        val allFiles = mutableListOf<VirtualFile>()
+        project.guessProjectDir()?.let {
+            VfsUtilCore.visitChildrenRecursively(
+                it,
+                object : VirtualFileVisitor<Unit>(NO_FOLLOW_SYMLINKS) {
+                    // TODO: refactor this along with /dev & codescan file traversing logic
+                    override fun visitFile(file: VirtualFile): Boolean {
+                        if ((file.isDirectory && isBuildOrBin(file.name)) ||
+                            runBlocking { featureDevSessionContext.ignoreFile(file.name, scope) } ||
+                            (file.isFile && file.length > 10 * 1024 * 1024)
+                        ) {
+                            return false
+                        }
+                        if (file.isFile) {
+                            allFiles.add(file)
+                            return false
+                        }
+                        return true
+                    }
+                }
+            )
+        }
 
         for (file in allFiles) {
             if (willExceedPayloadLimit(currentTotalFileSize, file.length)) {

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -89,15 +89,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         return ZipCreationResult(zippedProject, checkSum256, zippedProject.length())
     }
 
-    private suspend fun ignoreFile(file: File, scope: CoroutineScope): Boolean = with(scope) {
-        val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
-            async {
-                pattern.containsMatchIn(file.path)
-            }
-        }
-        deferredResults.any { it.await() }
-    }
-
     fun isFileExtensionAllowed(file: VirtualFile): Boolean {
         // if it is a directory, it is allowed
         if (file.isDirectory) return true
@@ -110,7 +101,16 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         return !isFileExtensionAllowed(file)
     }
 
-    suspend fun ignoreFile(file: VirtualFile, scope: CoroutineScope): Boolean = ignoreFile(File(file.path), scope)
+    suspend fun ignoreFile(file: VirtualFile, scope: CoroutineScope): Boolean = ignoreFile(file.path, scope)
+
+    suspend fun ignoreFile(path: String, scope: CoroutineScope): Boolean = with(scope) {
+        val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
+            async {
+                pattern.containsMatchIn(path)
+            }
+        }
+        deferredResults.any { it.await() }
+    }
 
     suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) {
         val files = mutableListOf<VirtualFile>()
@@ -121,26 +121,26 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
             projectRoot,
             object : VirtualFileVisitor<Unit>() {
                 override fun visitFile(file: VirtualFile): Boolean {
-                    val isFileIgnoredByPattern = runBlocking { ignoreFile(file, this) }
                     val isFileIgnoredByExtension = runBlocking { ignoreFileByExtension(file, this) }
-                    val isFileIgnored = isFileIgnoredByExtension || isFileIgnoredByPattern
-
                     if (isFileIgnoredByExtension) {
                         val extension = file.extension.orEmpty()
                         ignoredExtensionMap[extension] = (ignoredExtensionMap[extension] ?: 0) + 1
+                        return false
+                    }
+                    val isFileIgnoredByPattern = runBlocking { ignoreFile(file.name, this) }
+                    if (isFileIgnoredByPattern) {
+                        return false
                     }
 
-                    if (file.isFile && !isFileIgnored) {
+                    if (file.isFile) {
                         totalSize += file.length
                         files.add(file)
 
                         if (maxProjectSizeBytes != null && totalSize > maxProjectSizeBytes) {
                             throw RepoSizeLimitError(message("amazonqFeatureDev.content_length.error_text"))
                         }
-                        return true
-                    } else {
-                        return !isFileIgnored
                     }
+                    return true
                 }
             }
         )