diff --git a/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json b/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json new file mode 100644 index 00000000000..25236e36a4e --- /dev/null +++ b/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json @@ -0,0 +1,4 @@ +{ + "type" : "bugfix", + "description" : "Amazon Q can update mvn and gradle build files" +} diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt index 60d7b674c9e..838066a1560 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt @@ -13,8 +13,17 @@ import org.mockito.kotlin.whenever import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FeatureDevService +import software.aws.toolkits.jetbrains.utils.rules.HeavyJavaCodeInsightTestFixtureRule +import software.aws.toolkits.jetbrains.utils.rules.addFileToModule +import java.util.zip.ZipFile + +class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTestFixtureRule()) { + + private fun addFilesToProjectModule(vararg path: String) { + val module = projectRule.module + path.forEach { projectRule.fixture.addFileToModule(module, it, it) } + } -class FeatureDevSessionContextTest : FeatureDevTestBase() { @Rule @JvmField val ruleChain = RuleChain(projectRule, disposableRule) @@ -40,6 +49,7 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() { fun testWithValidFile() { val ktFile = mock() whenever(ktFile.extension).thenReturn("kt") + whenever(ktFile.path).thenReturn("code.kt") assertTrue(featureDevSessionContext.isFileExtensionAllowed(ktFile)) } @@ -47,6 +57,56 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() { fun testWithInvalidFile() { val txtFile = mock() whenever(txtFile.extension).thenReturn("txt") + whenever(txtFile.path).thenReturn("file.txt") assertFalse(featureDevSessionContext.isFileExtensionAllowed(txtFile)) } + + @Test + fun testAllowedFilePath() { + val allowedPaths = listOf("build.gradle", "gradle.properties", ".mvn/wrapper/maven-wrapper.properties") + allowedPaths.forEach({ + val txtFile = mock() + whenever(txtFile.path).thenReturn(it) + whenever(txtFile.extension).thenReturn(it.split(".").last()) + assertTrue(featureDevSessionContext.isFileExtensionAllowed(txtFile)) + }) + } + + @Test + fun testZipProject() { + addFilesToProjectModule( + ".gradle/cached.jar", + "src/MyClass.java", + "gradlew", + "gradlew.bat", + "README.md", + "settings.gradle", + "build.gradle", + "gradle/wrapper/gradle-wrapper.properties", + ) + + val zipResult = featureDevSessionContext.getProjectZip() + val zipPath = zipResult.payload.path + + val zippedFiles = mutableSetOf() + ZipFile(zipPath).use { zipFile -> + for (entry in zipFile.entries()) { + if (!entry.name.endsWith("/")) { + zippedFiles.add(entry.name) + } + } + } + + val expectedFiles = setOf( + "src/MyClass.java", + "gradlew", + "gradlew.bat", + "README.md", + "settings.gradle", + "build.gradle", + "gradle/wrapper/gradle-wrapper.properties", + ) + + assertTrue(zippedFiles == expectedFiles) + } } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt index 8188cac9938..47d2c5ff6e9 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt @@ -372,6 +372,6 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod // Adding gitignore file and gitignore file member for testing. // The tests include the markdown file but not these two files. projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile - projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main") + projectRule.fixture.addFileToProject("/.idea/ref", "ref: refs/heads/main") } } diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt index 214b837bd93..1dc786b0cf7 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt @@ -16,21 +16,26 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.apache.commons.codec.digest.DigestUtils -import software.aws.toolkits.core.utils.outputStream -import software.aws.toolkits.core.utils.putNextEntry -import software.aws.toolkits.jetbrains.core.coroutines.EDT +import org.apache.commons.io.FileUtils import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS import software.aws.toolkits.resources.AwsCoreBundle import software.aws.toolkits.telemetry.AmazonqTelemetry import java.io.File import java.io.FileInputStream +import java.net.URI +import java.nio.file.FileSystem +import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption import java.util.Base64 -import java.util.zip.ZipOutputStream +import java.util.UUID import kotlin.coroutines.coroutineContext import kotlin.io.path.Path +import kotlin.io.path.createParentDirectories +import kotlin.io.path.getPosixFilePermissions import kotlin.io.path.relativeTo interface RepoSizeError { @@ -41,41 +46,45 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) { // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan. - private val ignorePatterns = setOf( - "\\.aws-sam", - "\\.svn", - "\\.hg/?", - "\\.rvm", - "\\.git/?", - "\\.gitignore", - "\\.project", - "\\.gem", - "/\\.idea/?", - "\\.zip$", - "\\.bin$", - "\\.png$", - "\\.jpg$", - "\\.svg$", - "\\.pyc$", - "/license\\.txt$", - "/License\\.txt$", - "/LICENSE\\.txt$", - "/license\\.md$", - "/License\\.md$", - "/LICENSE\\.md$", - "node_modules/?", - "build/?", - "dist/?" - ).map { Regex(it) } + private val additionalGitIgnoreRules = setOf( + ".aws-sam", + ".gem", + ".git", + ".gitignore", + ".gradle", + ".hg", + ".idea", + ".project", + ".rvm", + ".svn", + "*.zip", + "*.bin", + "*.png", + "*.jpg", + "*.svg", + "*.pyc", + "license.txt", + "License.txt", + "LICENSE.txt", + "license.md", + "License.md", + "LICENSE.md", + "node_modules", + "build", + "dist" + ) // well known source files that do not have extensions private val wellKnownSourceFiles = setOf( "Dockerfile", - "Dockerfile.build" + "Dockerfile.build", + "gradlew", + "mvnw" ) // projectRoot: is the directory where the project is located when selected to open a project. val projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}") + private val projectRootPath = Paths.get(projectRoot.path) ?: error("Can not find project root path") // selectedSourceFolder": is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading. private var _selectedSourceFolder = projectRoot @@ -85,10 +94,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo init { ignorePatternsWithGitIgnore = try { buildList { - addAll(ignorePatterns) - parseGitIgnore().mapNotNull { pattern -> - runCatching { Regex(pattern) }.getOrNull() - }.let { addAll(it) } + addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) }) + addAll(parseGitIgnore()) + }.mapNotNull { pattern -> + runCatching { Regex(pattern) }.getOrNull() } } catch (e: Exception) { emptyList() @@ -108,7 +117,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo fun isFileExtensionAllowed(file: VirtualFile): Boolean { // if it is a directory, it is allowed if (file.isDirectory) return true - val extension = file.extension ?: return false return ALLOWED_CODE_EXTENSIONS.contains(extension) } @@ -116,13 +124,19 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo private fun ignoreFileByExtension(file: VirtualFile) = !isFileExtensionAllowed(file) - suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.path) + suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl) suspend fun ignoreFile(path: String): Boolean { // this method reads like something a JS dev would write and doesn't do what the author thinks val deferredResults = ignorePatternsWithGitIgnore.map { pattern -> withContext(coroutineContext) { - async { pattern.containsMatchIn(path) } + // avoid partial match (pattern.containsMatchIn) since it causes us matching files + // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!) + // we convert the glob rules to regex, add a trailing /* to all rules and then match + // entries against them by adding a trailing /. + // TODO: Add unit tests for gitignore matching + val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path + async { pattern.matches("$relative/") } } } @@ -132,7 +146,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo return deferredResults.any { it.await() } } - private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) + fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) { val files = mutableListOf() @@ -187,22 +201,43 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo } } - createTemporaryZipFileAsync { zipOutput -> + val zipFilePath = createTemporaryZipFileAsync { zipfs -> + val posixFileAttributeSubstr = "posix" + val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains(posixFileAttributeSubstr) filesToIncludeFlow.collect { file -> - try { - val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath()) - zipOutput.putNextEntry(relativePath.toString(), Path(file.path)) - } catch (e: NoSuchFileException) { - // Noop: Skip if file was deleted + + if (!file.isDirectory) { + val externalFilePath = Path(file.path) + val relativePath = Path(file.path).relativeTo(projectRootPath) + val zipfsPath = zipfs.getPath("/$relativePath") + withContext(getCoroutineBgContext()) { + zipfsPath.createParentDirectories() + try { + Files.copy(externalFilePath, zipfsPath, StandardCopyOption.REPLACE_EXISTING) + if (isPosix) { + val zipPermissionAttributeName = "zip:permissions" + Files.setAttribute(zipfsPath, zipPermissionAttributeName, externalFilePath.getPosixFilePermissions()) + } + } catch (e: NoSuchFileException) { + // Noop: Skip if file was deleted + } + } } } } + zipFilePath }.toFile() - private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) { - val file = Files.createTempFile(null, ".zip") - ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) } - file + private suspend fun createTemporaryZipFileAsync(block: suspend (FileSystem) -> Unit): Path = withContext(getCoroutineBgContext()) { + // Don't use Files.createTempFile since the file must not be created for ZipFS to work + val tempFilePath: Path = Paths.get(FileUtils.getTempDirectory().absolutePath, "${UUID.randomUUID()}.zip") + val uri = URI.create("jar:${tempFilePath.toUri()}") + val env = hashMapOf("create" to "true") + val zipfs = FileSystems.newFileSystem(uri, env) + zipfs.use { + block(zipfs) + } + tempFilePath } private fun parseGitIgnore(): Set { @@ -220,7 +255,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo private fun convertGitIgnorePatternToRegex(pattern: String): String = pattern .replace(".", "\\.") .replace("*", ".*") - .let { if (it.endsWith("/")) "$it?" else it } // Handle directory-specific patterns by optionally matching trailing slash + .let { if (it.endsWith("/")) "$it.*" else "$it/.*" } // Add a trailing /* to all patterns. (we add a trailing / to all files when matching) var selectedSourceFolder: VirtualFile set(newRoot) { diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt index 429bd770350..81f4431f3ad 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt @@ -16,6 +16,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "bash", "bat", "boo", + "bms", "c", "cbl", "cc", @@ -27,6 +28,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "cljs", "cls", "cmake", + "cmd", "cob", "cobra", "coffee", @@ -122,11 +124,13 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "pike", "pir", "pl", + "pli", "pm", "pmod", "pp", "pro", "prolog", + "properties", "ps1", "psd1", "psm1", @@ -200,7 +204,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "xml", "yaml", "yml", - "zig" + "zig", ) fun scrubNames(messageToBeScrubbed: String, username: String? = getSystemUserName()): String {