diff --git a/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json b/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json new file mode 100644 index 00000000000..25236e36a4e --- /dev/null +++ b/.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json @@ -0,0 +1,4 @@ +{ + "type" : "bugfix", + "description" : "Amazon Q can update mvn and gradle build files" +} diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt index 60d7b674c9e..838066a1560 100644 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt +++ b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt @@ -13,8 +13,17 @@ import org.mockito.kotlin.whenever import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FeatureDevService +import software.aws.toolkits.jetbrains.utils.rules.HeavyJavaCodeInsightTestFixtureRule +import software.aws.toolkits.jetbrains.utils.rules.addFileToModule +import java.util.zip.ZipFile + +class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTestFixtureRule()) { + + private fun addFilesToProjectModule(vararg path: String) { + val module = projectRule.module + path.forEach { projectRule.fixture.addFileToModule(module, it, it) } + } -class FeatureDevSessionContextTest : FeatureDevTestBase() { @Rule @JvmField val ruleChain = RuleChain(projectRule, disposableRule) @@ -40,6 +49,7 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() { fun testWithValidFile() { val ktFile = mock() whenever(ktFile.extension).thenReturn("kt") + whenever(ktFile.path).thenReturn("code.kt") assertTrue(featureDevSessionContext.isFileExtensionAllowed(ktFile)) } @@ -47,6 +57,56 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() { fun testWithInvalidFile() { val txtFile = mock() whenever(txtFile.extension).thenReturn("txt") + whenever(txtFile.path).thenReturn("file.txt") assertFalse(featureDevSessionContext.isFileExtensionAllowed(txtFile)) } + + @Test + fun testAllowedFilePath() { + val allowedPaths = listOf("build.gradle", "gradle.properties", ".mvn/wrapper/maven-wrapper.properties") + allowedPaths.forEach({ + val txtFile = mock() + whenever(txtFile.path).thenReturn(it) + whenever(txtFile.extension).thenReturn(it.split(".").last()) + assertTrue(featureDevSessionContext.isFileExtensionAllowed(txtFile)) + }) + } + + @Test + fun testZipProject() { + addFilesToProjectModule( + ".gradle/cached.jar", + "src/MyClass.java", + "gradlew", + "gradlew.bat", + "README.md", + "settings.gradle", + "build.gradle", + "gradle/wrapper/gradle-wrapper.properties", + ) + + val zipResult = featureDevSessionContext.getProjectZip() + val zipPath = zipResult.payload.path + + val zippedFiles = mutableSetOf() + ZipFile(zipPath).use { zipFile -> + for (entry in zipFile.entries()) { + if (!entry.name.endsWith("/")) { + zippedFiles.add(entry.name) + } + } + } + + val expectedFiles = setOf( + "src/MyClass.java", + "gradlew", + "gradlew.bat", + "README.md", + "settings.gradle", + "build.gradle", + "gradle/wrapper/gradle-wrapper.properties", + ) + + assertTrue(zippedFiles == expectedFiles) + } } diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt index 8188cac9938..5f54b0676d0 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt @@ -19,6 +19,7 @@ import software.aws.toolkits.jetbrains.utils.rules.addFileToModule import software.aws.toolkits.jetbrains.utils.rules.addModule import software.aws.toolkits.telemetry.CodewhispererLanguage import java.io.BufferedInputStream +import java.io.File import java.util.zip.ZipInputStream import kotlin.io.path.relativeTo import kotlin.test.assertNotNull @@ -109,11 +110,12 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod } private fun setupCsharpProject() { + val fileSeparator = File.separator val testModule = projectRule.fixture.addModule("testModule") val testModule2 = projectRule.fixture.addModule("testModule2") testCs = projectRule.fixture.addFileToModule( testModule, - "/Test.cs", + "${fileSeparator}Test.cs", """ using Utils; using Helpers.Helper; @@ -131,7 +133,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod utilsCs = projectRule.fixture.addFileToModule( testModule, - "/Utils.cs", + "${fileSeparator}Utils.cs", """ public static class Utils { @@ -157,7 +159,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod helperCs = projectRule.fixture.addFileToModule( testModule, - "/Helpers/Helper.cs", + "${fileSeparator}Helpers${fileSeparator}Helper.cs", """ public static class Helper { @@ -201,7 +203,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod helpGo = projectRule.fixture.addFileToModule( testModule, - "/help.go", + "${fileSeparator}help.go", """ package main @@ -217,7 +219,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod utilsJs = projectRule.fixture.addFileToModule( testModule, - "/utils.js", + "${fileSeparator}utils.js", """ function add(num1, num2) { return num1 + num2; @@ -247,7 +249,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod testJson = projectRule.fixture.addFileToModule( testModule, - "/helpers/test3Json.json", + "${fileSeparator}helpers${fileSeparator}test3Json.json", """ { "AWSTemplateFormatVersion": "2010-09-09", @@ -303,7 +305,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod helperPy = projectRule.fixture.addFileToModule( testModule, - "/helpers/helper.py", + "${fileSeparator}helpers${fileSeparator}helper.py", """ from helpers import helper as h def subtract(num1, num2) @@ -319,13 +321,13 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod totalSize += helperPy.length totalLines += helperPy.toNioPath().toFile().readLines().size - readMeMd = projectRule.fixture.addFileToModule(testModule, "/ReadMe.md", "### Now included").virtualFile + readMeMd = projectRule.fixture.addFileToModule(testModule, "${fileSeparator}ReadMe.md", "### Now included").virtualFile totalSize += readMeMd.length totalLines += readMeMd.toNioPath().toFile().readLines().size testTf = projectRule.fixture.addFileToModule( testModule2, - "/testTf.tf", + "${fileSeparator}testTf.tf", """ # Create example resource for three S3 buckets using for_each, where the bucket prefix are in variable with list containing [prod, staging, dev] @@ -345,7 +347,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod testYaml = projectRule.fixture.addFileToModule( testModule2, - "/testYaml.yaml", + "${fileSeparator}testYaml.yaml", """ AWSTemplateFormatVersion: "2010-09-09" @@ -371,7 +373,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod // Adding gitignore file and gitignore file member for testing. // The tests include the markdown file but not these two files. - projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile - projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main") + projectRule.fixture.addFileToProject("$fileSeparator.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile + projectRule.fixture.addFileToProject("$fileSeparator.idea${fileSeparator}ref", "ref: refs/heads/main") } } diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt index 98a0f92f082..928842ae94f 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt @@ -16,21 +16,26 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.apache.commons.codec.digest.DigestUtils -import software.aws.toolkits.core.utils.outputStream -import software.aws.toolkits.core.utils.putNextEntry -import software.aws.toolkits.jetbrains.core.coroutines.EDT +import org.apache.commons.io.FileUtils import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS import software.aws.toolkits.resources.AwsCoreBundle import software.aws.toolkits.telemetry.AmazonqTelemetry import java.io.File import java.io.FileInputStream +import java.net.URI +import java.nio.file.FileSystem +import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption import java.util.Base64 -import java.util.zip.ZipOutputStream +import java.util.UUID import kotlin.coroutines.coroutineContext import kotlin.io.path.Path +import kotlin.io.path.createParentDirectories +import kotlin.io.path.getPosixFilePermissions import kotlin.io.path.relativeTo interface RepoSizeError { @@ -41,41 +46,45 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) { // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan. - private val ignorePatterns = setOf( - "\\.aws-sam", - "\\.svn", - "\\.hg/?", - "\\.rvm", - "\\.git/?", - "\\.gitignore", - "\\.project", - "\\.gem", - "/\\.idea/?", - "\\.zip$", - "\\.bin$", - "\\.png$", - "\\.jpg$", - "\\.svg$", - "\\.pyc$", - "/license\\.txt$", - "/License\\.txt$", - "/LICENSE\\.txt$", - "/license\\.md$", - "/License\\.md$", - "/LICENSE\\.md$", - "node_modules/?", - "build/?", - "dist/?" - ).map { Regex(it) } + private val additionalGitIgnoreRules = setOf( + ".aws-sam", + ".gem", + ".git", + ".gitignore", + ".gradle", + ".hg", + ".idea", + ".project", + ".rvm", + ".svn", + "*.zip", + "*.bin", + "*.png", + "*.jpg", + "*.svg", + "*.pyc", + "license.txt", + "License.txt", + "LICENSE.txt", + "license.md", + "License.md", + "LICENSE.md", + "node_modules", + "build", + "dist" + ) // well known source files that do not have extensions private val wellKnownSourceFiles = setOf( "Dockerfile", - "Dockerfile.build" + "Dockerfile.build", + "gradlew", + "mvnw" ) // projectRoot: is the directory where the project is located when selected to open a project. val projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}") + private val projectRootPath = Paths.get(projectRoot.path) ?: error("Can not find project root path") // selectedSourceFolder": is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading. private var _selectedSourceFolder = projectRoot @@ -85,10 +94,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo init { ignorePatternsWithGitIgnore = try { buildList { - addAll(ignorePatterns) - parseGitIgnore().mapNotNull { pattern -> - runCatching { Regex(pattern) }.getOrNull() - }.let { addAll(it) } + addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) }) + addAll(parseGitIgnore()) + }.mapNotNull { pattern -> + runCatching { Regex(pattern) }.getOrNull() } } catch (e: Exception) { emptyList() @@ -108,7 +117,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo fun isFileExtensionAllowed(file: VirtualFile): Boolean { // if it is a directory, it is allowed if (file.isDirectory) return true - val extension = file.extension ?: return false return ALLOWED_CODE_EXTENSIONS.contains(extension) } @@ -122,7 +130,13 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo // this method reads like something a JS dev would write and doesn't do what the author thinks val deferredResults = ignorePatternsWithGitIgnore.map { pattern -> withContext(coroutineContext) { - async { pattern.containsMatchIn(path) } + // avoid partial match (pattern.containsMatchIn) since it causes us matching files + // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!) + // we convert the glob rules to regex, add a trailing /* to all rules and then match + // entries against them by adding a trailing /. + // TODO: Add unit tests for gitignore matching + val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path + async { pattern.matches("$relative/") } } } @@ -132,7 +146,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo return deferredResults.any { it.await() } } - private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) + fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name) suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) { val files = mutableListOf() @@ -187,18 +201,36 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo } } - createTemporaryZipFileAsync { zipOutput -> + val zipFilePath = createTemporaryZipFileAsync { zipfs -> + val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix") filesToIncludeFlow.collect { file -> - val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath()) - zipOutput.putNextEntry(relativePath.toString(), Path(file.path)) + if (!file.isDirectory) { + val externalFilePath = Path(file.path) + val relativePath = Path(file.path).relativeTo(projectRootPath) + val zipfsPath = zipfs.getPath("/$relativePath") + runBlocking { + zipfsPath.createParentDirectories() + Files.copy(externalFilePath, zipfsPath, StandardCopyOption.REPLACE_EXISTING) + if (isPosix) { + Files.setAttribute(zipfsPath, "zip:permissions", externalFilePath.getPosixFilePermissions()) + } + } + } } } + zipFilePath }.toFile() - private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) { - val file = Files.createTempFile(null, ".zip") - ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) } - file + private suspend fun createTemporaryZipFileAsync(block: suspend (FileSystem) -> Unit): Path = withContext(getCoroutineBgContext()) { + // Don't use Files.createTempFile since the file must not be created for ZipFS to work + val tempFilePath: Path = Paths.get(FileUtils.getTempDirectory().absolutePath, "${UUID.randomUUID()}.zip") + val uri = URI.create("jar:${tempFilePath.toUri()}") + val env = hashMapOf("create" to "true") + val zipfs = FileSystems.newFileSystem(uri, env) + zipfs.use { + block(zipfs) + } + tempFilePath } private fun parseGitIgnore(): Set { @@ -216,7 +248,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo private fun convertGitIgnorePatternToRegex(pattern: String): String = pattern .replace(".", "\\.") .replace("*", ".*") - .let { if (it.endsWith("/")) "$it?" else it } // Handle directory-specific patterns by optionally matching trailing slash + .let { if (it.endsWith("/")) "$it.*" else "$it/.*" } // Add a trailing /* to all patterns. (we add a trailing / to all files when matching) var selectedSourceFolder: VirtualFile set(newRoot) { diff --git a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt index 429bd770350..81f4431f3ad 100644 --- a/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt +++ b/plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt @@ -16,6 +16,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "bash", "bat", "boo", + "bms", "c", "cbl", "cc", @@ -27,6 +28,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "cljs", "cls", "cmake", + "cmd", "cob", "cobra", "coffee", @@ -122,11 +124,13 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "pike", "pir", "pl", + "pli", "pm", "pmod", "pp", "pro", "prolog", + "properties", "ps1", "psd1", "psm1", @@ -200,7 +204,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf( "xml", "yaml", "yml", - "zig" + "zig", ) fun scrubNames(messageToBeScrubbed: String, username: String? = getSystemUserName()): String {