From d40a6b6dfa2abcc7bf9f4a02cdac060b974f982b Mon Sep 17 00:00:00 2001 From: Hamed Soleimani Date: Sat, 25 Jan 2025 05:11:33 +0000 Subject: [PATCH 1/3] feat: Allow including binary files when auto build is enabled --- .../FeatureDevSessionContextTest.kt | 67 ++++++++++--- .../amazonq/FeatureDevSessionContext.kt | 99 ++++++++++--------- .../jetbrains/services/amazonq/QConstants.kt | 1 + .../services/telemetry/TelemetryUtils.kt | 1 - 4 files changed, 107 insertions(+), 61 deletions(-) diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt index dfebd35097c..f77d0ce75b7 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt @@ -3,6 +3,7 @@ import com.intellij.openapi.vfs.VirtualFile import com.intellij.testFramework.RuleChain +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Before @@ -72,7 +73,51 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest } @Test - fun testZipProject() { + fun testZipProjectWithoutAutoDev() { + checkZipProject( + false, + setOf( + "src/MyClass.java", + "gradlew", + "gradlew.bat", + "README.md", + "gradle/wrapper/gradle-wrapper.properties", + "builder/GetTestBuilder.java", + "settings.gradle", + "build.gradle", + ".gitignore", + ) + ) + } + + @Test + fun testZipProjectWithAutoDev() { + checkZipProject( + true, + setOf( + "src/MyClass.java", + "icons/menu.svg", + "assets/header.jpg", + "gradle/wrapper/gradle-wrapper.jar", + "gradle/wrapper/gradle-wrapper.properties", + "images/logo.png", + "builder/GetTestBuilder.java", + "gradlew", + "README.md", + ".gitignore", + "License.md", + "output.bin", + "archive.zip", + "gradlew.bat", + "license.txt", + "build.gradle", + "devfile.yaml", + "settings.gradle" + ) + ) + } + + fun checkZipProject(autoBuildEnabled: Boolean, expectedFiles: Set) { addFilesToProjectModule( ".gitignore", ".gradle/cached.jar", @@ -97,9 +142,12 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest "build/outputs", "dist/bundle.js", "gradle/wrapper/gradle-wrapper.jar", + "devfile.yaml", ) - val zipResult = featureDevSessionContext.getProjectZip(false) + projectRule.fixture.addFileToModule(module, "large-file.txt", "loblob".repeat(1024 * 1024)) + + val zipResult = featureDevSessionContext.getProjectZip(autoBuildEnabled) val zipPath = zipResult.payload.path val zippedFiles = mutableSetOf() @@ -111,19 +159,6 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest } } - val expectedFiles = setOf( - "src/MyClass.java", - "gradlew", - "gradlew.bat", - "README.md", - "gradle/wrapper/gradle-wrapper.properties", - "builder/GetTestBuilder.java", - "settings.gradle", - "build.gradle", - "gradle/wrapper/gradle-wrapper.jar", - ".gitignore", - ) - - assertTrue(zippedFiles == expectedFiles) + assertEquals(zippedFiles, expectedFiles) } } diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt index 127c524bee6..c2e29bdf264 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt @@ -10,7 +10,6 @@ import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.VirtualFileVisitor import com.intellij.openapi.vfs.isFile import com.intellij.platform.ide.progress.withBackgroundProgress -import kotlinx.coroutines.async import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking @@ -18,6 +17,7 @@ import kotlinx.coroutines.withContext import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.io.FileUtils import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext +import software.aws.toolkits.jetbrains.services.amazonq.QConstants.MAX_FILE_SIZE_BYTES import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS import software.aws.toolkits.jetbrains.utils.isDevFile import software.aws.toolkits.resources.AwsCoreBundle @@ -33,7 +33,6 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption import java.util.Base64 import java.util.UUID -import kotlin.coroutines.coroutineContext import kotlin.io.path.Path import kotlin.io.path.createParentDirectories import kotlin.io.path.getPosixFilePermissions @@ -47,8 +46,7 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) { // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan. - private val requiredFilesForExecution = setOf("gradle/wrapper/gradle-wrapper.jar") - private val additionalGitIgnoreRules = setOf( + private val additionalGitIgnoreFolderRules = setOf( ".aws-sam", ".gem", ".git", @@ -58,6 +56,12 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo ".project", ".rvm", ".svn", + "node_modules", + "build", + "dist", + ) + + private val additionalGitIgnoreBinaryFilesRules = setOf( "*.zip", "*.bin", "*.png", @@ -70,9 +74,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo "license.md", "License.md", "LICENSE.md", - "node_modules", - "build", - "dist" ) // well known source files that do not have extensions @@ -90,12 +91,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo // selectedSourceFolder: is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading. private var _selectedSourceFolder = projectRoot private var ignorePatternsWithGitIgnore = emptyList() + private var ignorePatternsForBinaryFiles = additionalGitIgnoreBinaryFilesRules + .map { convertGitIgnorePatternToRegex(it) } + .mapNotNull { pattern -> + runCatching { Regex(pattern) }.getOrNull() + } private val gitIgnoreFile = File(selectedSourceFolder.path, ".gitignore") init { ignorePatternsWithGitIgnore = try { buildList { - addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) }) + addAll(additionalGitIgnoreFolderRules.map { convertGitIgnorePatternToRegex(it) }) addAll(parseGitIgnore()) }.mapNotNull { pattern -> runCatching { Regex(pattern) }.getOrNull() @@ -130,35 +136,49 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo return ALLOWED_CODE_EXTENSIONS.contains(extension) } - private fun ignoreFileByExtension(file: VirtualFile) = - !isFileExtensionAllowed(file) + fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl) + + fun ignoreFile(path: String, applyExtraBinaryFilesRules: Boolean = true): Boolean { + val allIgnoreRules = if (applyExtraBinaryFilesRules) ignorePatternsWithGitIgnore + ignorePatternsForBinaryFiles else ignorePatternsWithGitIgnore + val matchedRules = allIgnoreRules.map { pattern -> + // avoid partial match (pattern.containsMatchIn) since it causes us matching files + // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!) + // we convert the glob rules to regex, add a trailing /* to all rules and then match + // entries against them by adding a trailing /. + // TODO: Add unit tests for gitignore matching + val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path + pattern.matches("$relative/") + } + return matchedRules.any { it } + } - suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl) + private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) - suspend fun ignoreFile(path: String): Boolean { - if (requiredFilesForExecution.any { path.endsWith(it) }) { + private fun shouldIncludeInZipFile(file: VirtualFile, isAutoBuildFeatureEnabled: Boolean): Boolean { + // large files always ignored + if (file.length > MAX_FILE_SIZE_BYTES) { return false } - // this method reads like something a JS dev would write and doesn't do what the author thinks - val deferredResults = ignorePatternsWithGitIgnore.map { pattern -> - withContext(coroutineContext) { - // avoid partial match (pattern.containsMatchIn) since it causes us matching files - // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!) - // we convert the glob rules to regex, add a trailing /* to all rules and then match - // entries against them by adding a trailing /. - // TODO: Add unit tests for gitignore matching - val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path - async { pattern.matches("$relative/") } - } + + // always respect gitignore rules and remove binary files if auto build is disabled + val isFileIgnoredByPattern = ignoreFile(file.path, !isAutoBuildFeatureEnabled) + if (isFileIgnoredByPattern) { + return false } - // this will serially iterate over and block - // ideally we race the results https://github.com/Kotlin/kotlinx.coroutines/issues/2867 - // i.e. Promise.any(...) - return deferredResults.any { it.await() } - } + // all other files are included when auto build enabled + if (isAutoBuildFeatureEnabled) { + return true + } + + // when auto build is disabled, only include files with well known extensions and names except "devfile.yam" + if (!isDevFile(file) && (wellKnown(file) || isFileExtensionAllowed(file))) { + return true + } - fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) + // Any other files should not be included + return false + } suspend fun zipFiles(projectRoot: VirtualFile, isAutoBuildFeatureEnabled: Boolean?): File = withContext(getCoroutineBgContext()) { val files = mutableListOf() @@ -169,26 +189,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo projectRoot, object : VirtualFileVisitor() { override fun visitFile(file: VirtualFile): Boolean { - val isWellKnown = runBlocking { wellKnown(file) } - val isFileIgnoredByExtension = runBlocking { ignoreFileByExtension(file) } - // if `isAutoBuildFeatureEnabled` is false, then filter devfile - val isFilterDevFile = if (isAutoBuildFeatureEnabled == true) false else isDevFile(file) + if (file.isDirectory) { + return true + } - if (!isWellKnown && isFileIgnoredByExtension) { + val isIncluded = shouldIncludeInZipFile(file, isAutoBuildFeatureEnabled == true) + if (!isIncluded) { val extension = file.extension.orEmpty() ignoredExtensionMap[extension] = (ignoredExtensionMap[extension] ?: 0) + 1 return false } - if (isFilterDevFile) { - return false - } - - val isFileIgnoredByPattern = runBlocking { ignoreFile(file.name) } - if (isFileIgnoredByPattern) { - return false - } - if (file.isFile) { totalSize += file.length files.add(file) diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt index 3cc2bf81741..26424ecc17d 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt @@ -6,4 +6,5 @@ package software.aws.toolkits.jetbrains.services.amazonq object QConstants { const val Q_MARKETPLACE_URI = "https://aws.amazon.com/q/developer/" const val CODEWHISPERER_LOGIN_HELP_URI = "https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/auth-access.html" + const val MAX_FILE_SIZE_BYTES = 1024 * 1024 } diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt index b3c2e216811..c7e21874a89 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt @@ -88,7 +88,6 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "idl", "ini", "io", - "jar", "java", "jl", "js", From 184f12381ca12b55c26088ba19fe0077b0d0b95e Mon Sep 17 00:00:00 2001 From: Hamed Soleimani Date: Mon, 27 Jan 2025 18:23:30 +0000 Subject: [PATCH 2/3] attempt fixing test failures on windows --- .../jetbrains/services/amazonq/FeatureDevSessionContext.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt index c2e29bdf264..97c54cbc78b 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt @@ -189,10 +189,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo projectRoot, object : VirtualFileVisitor() { override fun visitFile(file: VirtualFile): Boolean { - if (file.isDirectory) { - return true - } - val isIncluded = shouldIncludeInZipFile(file, isAutoBuildFeatureEnabled == true) if (!isIncluded) { val extension = file.extension.orEmpty() From 6198e84885e8e73868621043d77f7effc4806104 Mon Sep 17 00:00:00 2001 From: Hamed Soleimani Date: Mon, 27 Jan 2025 20:09:03 +0000 Subject: [PATCH 3/3] fix windows tests --- .../jetbrains/services/amazonq/FeatureDevSessionContext.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt index 97c54cbc78b..781d3dd2581 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt @@ -136,7 +136,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo return ALLOWED_CODE_EXTENSIONS.contains(extension) } - fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl) + fun ignoreFile(file: VirtualFile, applyExtraBinaryFilesRules: Boolean = true): Boolean = ignoreFile(file.presentableUrl, applyExtraBinaryFilesRules) fun ignoreFile(path: String, applyExtraBinaryFilesRules: Boolean = true): Boolean { val allIgnoreRules = if (applyExtraBinaryFilesRules) ignorePatternsWithGitIgnore + ignorePatternsForBinaryFiles else ignorePatternsWithGitIgnore @@ -161,7 +161,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo } // always respect gitignore rules and remove binary files if auto build is disabled - val isFileIgnoredByPattern = ignoreFile(file.path, !isAutoBuildFeatureEnabled) + val isFileIgnoredByPattern = ignoreFile(file, !isAutoBuildFeatureEnabled) if (isFileIgnoredByPattern) { return false }