feat: Allow including binary files when auto build is enabled (#5299)

* feat: Allow including binary files when auto build is enabled

* attempt fixing test failures on windows

* fix windows tests

Author: osdemah

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt

@@ -3,6 +3,7 @@
 
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.testFramework.RuleChain
+import org.junit.Assert.assertEquals
 import org.junit.Assert.assertFalse
 import org.junit.Assert.assertTrue
 import org.junit.Before
@@ -72,7 +73,51 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
     }
 
     @Test
-    fun testZipProject() {
+    fun testZipProjectWithoutAutoDev() {
+        checkZipProject(
+            false,
+            setOf(
+                "src/MyClass.java",
+                "gradlew",
+                "gradlew.bat",
+                "README.md",
+                "gradle/wrapper/gradle-wrapper.properties",
+                "builder/GetTestBuilder.java",
+                "settings.gradle",
+                "build.gradle",
+                ".gitignore",
+            )
+        )
+    }
+
+    @Test
+    fun testZipProjectWithAutoDev() {
+        checkZipProject(
+            true,
+            setOf(
+                "src/MyClass.java",
+                "icons/menu.svg",
+                "assets/header.jpg",
+                "gradle/wrapper/gradle-wrapper.jar",
+                "gradle/wrapper/gradle-wrapper.properties",
+                "images/logo.png",
+                "builder/GetTestBuilder.java",
+                "gradlew",
+                "README.md",
+                ".gitignore",
+                "License.md",
+                "output.bin",
+                "archive.zip",
+                "gradlew.bat",
+                "license.txt",
+                "build.gradle",
+                "devfile.yaml",
+                "settings.gradle"
+            )
+        )
+    }
+
+    fun checkZipProject(autoBuildEnabled: Boolean, expectedFiles: Set<String>) {
         addFilesToProjectModule(
             ".gitignore",
             ".gradle/cached.jar",
@@ -97,9 +142,12 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
             "build/outputs",
             "dist/bundle.js",
             "gradle/wrapper/gradle-wrapper.jar",
+            "devfile.yaml",
         )
 
-        val zipResult = featureDevSessionContext.getProjectZip(false)
+        projectRule.fixture.addFileToModule(module, "large-file.txt", "loblob".repeat(1024 * 1024))
+
+        val zipResult = featureDevSessionContext.getProjectZip(autoBuildEnabled)
         val zipPath = zipResult.payload.path
 
         val zippedFiles = mutableSetOf<String>()
@@ -111,19 +159,6 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
             }
         }
 
-        val expectedFiles = setOf(
-            "src/MyClass.java",
-            "gradlew",
-            "gradlew.bat",
-            "README.md",
-            "gradle/wrapper/gradle-wrapper.properties",
-            "builder/GetTestBuilder.java",
-            "settings.gradle",
-            "build.gradle",
-            "gradle/wrapper/gradle-wrapper.jar",
-            ".gitignore",
-        )
-
-        assertTrue(zippedFiles == expectedFiles)
+        assertEquals(zippedFiles, expectedFiles)
     }
 }

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -10,14 +10,14 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileVisitor
 import com.intellij.openapi.vfs.isFile
 import com.intellij.platform.ide.progress.withBackgroundProgress
-import kotlinx.coroutines.async
 import kotlinx.coroutines.flow.channelFlow
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.apache.commons.codec.digest.DigestUtils
 import org.apache.commons.io.FileUtils
 import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
+import software.aws.toolkits.jetbrains.services.amazonq.QConstants.MAX_FILE_SIZE_BYTES
 import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
 import software.aws.toolkits.jetbrains.utils.isDevFile
 import software.aws.toolkits.resources.AwsCoreBundle
@@ -33,7 +33,6 @@ import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 import java.util.Base64
 import java.util.UUID
-import kotlin.coroutines.coroutineContext
 import kotlin.io.path.Path
 import kotlin.io.path.createParentDirectories
 import kotlin.io.path.getPosixFilePermissions
@@ -47,8 +46,7 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
 class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
     // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.
 
-    private val requiredFilesForExecution = setOf("gradle/wrapper/gradle-wrapper.jar")
-    private val additionalGitIgnoreRules = setOf(
+    private val additionalGitIgnoreFolderRules = setOf(
         ".aws-sam",
         ".gem",
         ".git",
@@ -58,6 +56,12 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         ".project",
         ".rvm",
         ".svn",
+        "node_modules",
+        "build",
+        "dist",
+    )
+
+    private val additionalGitIgnoreBinaryFilesRules = setOf(
         "*.zip",
         "*.bin",
         "*.png",
@@ -70,9 +74,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         "license.md",
         "License.md",
         "LICENSE.md",
-        "node_modules",
-        "build",
-        "dist"
     )
 
     // well known source files that do not have extensions
@@ -90,12 +91,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     // selectedSourceFolder: is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
     private var _selectedSourceFolder = projectRoot
     private var ignorePatternsWithGitIgnore = emptyList<Regex>()
+    private var ignorePatternsForBinaryFiles = additionalGitIgnoreBinaryFilesRules
+        .map { convertGitIgnorePatternToRegex(it) }
+        .mapNotNull { pattern ->
+            runCatching { Regex(pattern) }.getOrNull()
+        }
     private val gitIgnoreFile = File(selectedSourceFolder.path, ".gitignore")
 
     init {
         ignorePatternsWithGitIgnore = try {
             buildList {
-                addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
+                addAll(additionalGitIgnoreFolderRules.map { convertGitIgnorePatternToRegex(it) })
                 addAll(parseGitIgnore())
             }.mapNotNull { pattern ->
                 runCatching { Regex(pattern) }.getOrNull()
@@ -130,35 +136,49 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         return ALLOWED_CODE_EXTENSIONS.contains(extension)
     }
 
-    private fun ignoreFileByExtension(file: VirtualFile) =
-        !isFileExtensionAllowed(file)
+    fun ignoreFile(file: VirtualFile, applyExtraBinaryFilesRules: Boolean = true): Boolean = ignoreFile(file.presentableUrl, applyExtraBinaryFilesRules)
+
+    fun ignoreFile(path: String, applyExtraBinaryFilesRules: Boolean = true): Boolean {
+        val allIgnoreRules = if (applyExtraBinaryFilesRules) ignorePatternsWithGitIgnore + ignorePatternsForBinaryFiles else ignorePatternsWithGitIgnore
+        val matchedRules = allIgnoreRules.map { pattern ->
+            // avoid partial match (pattern.containsMatchIn) since it causes us matching files
+            // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
+            // we convert the glob rules to regex, add a trailing /* to all rules and then match
+            // entries against them by adding a trailing /.
+            // TODO: Add unit tests for gitignore matching
+            val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
+            pattern.matches("$relative/")
+        }
+        return matchedRules.any { it }
+    }
 
-    suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl)
+    private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
 
-    suspend fun ignoreFile(path: String): Boolean {
-        if (requiredFilesForExecution.any { path.endsWith(it) }) {
+    private fun shouldIncludeInZipFile(file: VirtualFile, isAutoBuildFeatureEnabled: Boolean): Boolean {
+        // large files always ignored
+        if (file.length > MAX_FILE_SIZE_BYTES) {
             return false
         }
-        // this method reads like something a JS dev would write and doesn't do what the author thinks
-        val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
-            withContext(coroutineContext) {
-                // avoid partial match (pattern.containsMatchIn) since it causes us matching files
-                // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
-                // we convert the glob rules to regex, add a trailing /* to all rules and then match
-                // entries against them by adding a trailing /.
-                // TODO: Add unit tests for gitignore matching
-                val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
-                async { pattern.matches("$relative/") }
-            }
+
+        // always respect gitignore rules and remove binary files if auto build is disabled
+        val isFileIgnoredByPattern = ignoreFile(file, !isAutoBuildFeatureEnabled)
+        if (isFileIgnoredByPattern) {
+            return false
         }
 
-        // this will serially iterate over and block
-        // ideally we race the results https://github.com/Kotlin/kotlinx.coroutines/issues/2867
-        // i.e. Promise.any(...)
-        return deferredResults.any { it.await() }
-    }
+        // all other files are included when auto build enabled
+        if (isAutoBuildFeatureEnabled) {
+            return true
+        }
+
+        // when auto build is disabled, only include files with well known extensions and names except "devfile.yam"
+        if (!isDevFile(file) && (wellKnown(file) || isFileExtensionAllowed(file))) {
+            return true
+        }
 
-    fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
+        // Any other files should not be included
+        return false
+    }
 
     suspend fun zipFiles(projectRoot: VirtualFile, isAutoBuildFeatureEnabled: Boolean?): File = withContext(getCoroutineBgContext()) {
         val files = mutableListOf<VirtualFile>()
@@ -169,26 +189,13 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
             projectRoot,
             object : VirtualFileVisitor<Unit>() {
                 override fun visitFile(file: VirtualFile): Boolean {
-                    val isWellKnown = runBlocking { wellKnown(file) }
-                    val isFileIgnoredByExtension = runBlocking { ignoreFileByExtension(file) }
-                    // if `isAutoBuildFeatureEnabled` is false, then filter devfile
-                    val isFilterDevFile = if (isAutoBuildFeatureEnabled == true) false else isDevFile(file)
-
-                    if (!isWellKnown && isFileIgnoredByExtension) {
+                    val isIncluded = shouldIncludeInZipFile(file, isAutoBuildFeatureEnabled == true)
+                    if (!isIncluded) {
                         val extension = file.extension.orEmpty()
                         ignoredExtensionMap[extension] = (ignoredExtensionMap[extension] ?: 0) + 1
                         return false
                     }
 
-                    if (isFilterDevFile) {
-                        return false
-                    }
-
-                    val isFileIgnoredByPattern = runBlocking { ignoreFile(file.name) }
-                    if (isFileIgnoredByPattern) {
-                        return false
-                    }
-
                     if (file.isFile) {
                         totalSize += file.length
                         files.add(file)

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt

@@ -6,4 +6,5 @@ package software.aws.toolkits.jetbrains.services.amazonq
 object QConstants {
     const val Q_MARKETPLACE_URI = "https://aws.amazon.com/q/developer/"
     const val CODEWHISPERER_LOGIN_HELP_URI = "https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/auth-access.html"
+    const val MAX_FILE_SIZE_BYTES = 1024 * 1024
 }

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt

@@ -88,7 +88,6 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
     "idl",
     "ini",
     "io",
-    "jar",
     "java",
     "jl",
     "js",