fix(amazonq): file premissions are now preserved and mvn and gradle files are kept in the repo archives

.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "Amazon Q can update mvn and gradle build files"
+}

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt

@@ -13,8 +13,17 @@ import org.mockito.kotlin.whenever
 import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FeatureDevService
+import software.aws.toolkits.jetbrains.utils.rules.HeavyJavaCodeInsightTestFixtureRule
+import software.aws.toolkits.jetbrains.utils.rules.addFileToModule
+import java.util.zip.ZipFile
+
+class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTestFixtureRule()) {
+
+    private fun addFilesToProjectModule(vararg path: String) {
+        val module = projectRule.module
+        path.forEach { projectRule.fixture.addFileToModule(module, it, it) }
+    }
 
-class FeatureDevSessionContextTest : FeatureDevTestBase() {
     @Rule
     @JvmField
     val ruleChain = RuleChain(projectRule, disposableRule)
@@ -40,13 +49,64 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() {
     fun testWithValidFile() {
         val ktFile = mock<VirtualFile>()
         whenever(ktFile.extension).thenReturn("kt")
+        whenever(ktFile.path).thenReturn("code.kt")
         assertTrue(featureDevSessionContext.isFileExtensionAllowed(ktFile))
     }
 
     @Test
     fun testWithInvalidFile() {
         val txtFile = mock<VirtualFile>()
         whenever(txtFile.extension).thenReturn("txt")
+        whenever(txtFile.path).thenReturn("file.txt")
         assertFalse(featureDevSessionContext.isFileExtensionAllowed(txtFile))
     }
+
+    @Test
+    fun testAllowedFilePath() {
+        val allowedPaths = listOf("build.gradle", "gradle.properties", ".mvn/wrapper/maven-wrapper.properties")
+        allowedPaths.forEach({
+            val txtFile = mock<VirtualFile>()
+            whenever(txtFile.path).thenReturn(it)
+            whenever(txtFile.extension).thenReturn(it.split(".").last())
+            assertTrue(featureDevSessionContext.isFileExtensionAllowed(txtFile))
+        })
+    }
+
+    @Test
+    fun testZipProject() {
+        addFilesToProjectModule(
+            ".gradle/cached.jar",
+            "src/MyClass.java",
+            "gradlew",
+            "gradlew.bat",
+            "README.md",
+            "settings.gradle",
+            "build.gradle",
+            "gradle/wrapper/gradle-wrapper.properties",
+        )
+
+        val zipResult = featureDevSessionContext.getProjectZip()
+        val zipPath = zipResult.payload.path
+
+        val zippedFiles = mutableSetOf<String>()
+        ZipFile(zipPath).use { zipFile ->
+            for (entry in zipFile.entries()) {
+                if (!entry.name.endsWith("/")) {
+                    zippedFiles.add(entry.name)
+                }
+            }
+        }
+
+        val expectedFiles = setOf(
+            "src/MyClass.java",
+            "gradlew",
+            "gradlew.bat",
+            "README.md",
+            "settings.gradle",
+            "build.gradle",
+            "gradle/wrapper/gradle-wrapper.properties",
+        )
+
+        assertTrue(zippedFiles == expectedFiles)
+    }
 }

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt

@@ -372,6 +372,6 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
         // Adding gitignore file and gitignore file member for testing.
         // The tests include the markdown file but not these two files.
         projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile
-        projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main")
+        projectRule.fixture.addFileToProject("/.idea/ref", "ref: refs/heads/main")
     }
 }

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -16,21 +16,26 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.apache.commons.codec.digest.DigestUtils
-import software.aws.toolkits.core.utils.outputStream
-import software.aws.toolkits.core.utils.putNextEntry
-import software.aws.toolkits.jetbrains.core.coroutines.EDT
+import org.apache.commons.io.FileUtils
 import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
 import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
 import software.aws.toolkits.resources.AwsCoreBundle
 import software.aws.toolkits.telemetry.AmazonqTelemetry
 import java.io.File
 import java.io.FileInputStream
+import java.net.URI
+import java.nio.file.FileSystem
+import java.nio.file.FileSystems
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 import java.util.Base64
-import java.util.zip.ZipOutputStream
+import java.util.UUID
 import kotlin.coroutines.coroutineContext
 import kotlin.io.path.Path
+import kotlin.io.path.createParentDirectories
+import kotlin.io.path.getPosixFilePermissions
 import kotlin.io.path.relativeTo
 
 interface RepoSizeError {
@@ -41,41 +46,45 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
 class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
     // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.
 
-    private val ignorePatterns = setOf(
-        "\\.aws-sam",
-        "\\.svn",
-        "\\.hg/?",
-        "\\.rvm",
-        "\\.git/?",
-        "\\.gitignore",
-        "\\.project",
-        "\\.gem",
-        "/\\.idea/?",
-        "\\.zip$",
-        "\\.bin$",
-        "\\.png$",
-        "\\.jpg$",
-        "\\.svg$",
-        "\\.pyc$",
-        "/license\\.txt$",
-        "/License\\.txt$",
-        "/LICENSE\\.txt$",
-        "/license\\.md$",
-        "/License\\.md$",
-        "/LICENSE\\.md$",
-        "node_modules/?",
-        "build/?",
-        "dist/?"
-    ).map { Regex(it) }
+    private val additionalGitIgnoreRules = setOf(
+        ".aws-sam",
+        ".gem",
+        ".git",
+        ".gitignore",
+        ".gradle",
+        ".hg",
+        ".idea",
+        ".project",
+        ".rvm",
+        ".svn",
+        "*.zip",
+        "*.bin",
+        "*.png",
+        "*.jpg",
+        "*.svg",
+        "*.pyc",
+        "license.txt",
+        "License.txt",
+        "LICENSE.txt",
+        "license.md",
+        "License.md",
+        "LICENSE.md",
+        "node_modules",
+        "build",
+        "dist"
+    )
 
     // well known source files that do not have extensions
     private val wellKnownSourceFiles = setOf(
         "Dockerfile",
-        "Dockerfile.build"
+        "Dockerfile.build",
+        "gradlew",
+        "mvnw"
     )
 
     // projectRoot: is the directory where the project is located when selected to open a project.
     val projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}")
+    private val projectRootPath = Paths.get(projectRoot.path) ?: error("Can not find project root path")
 
     // selectedSourceFolder": is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
     private var _selectedSourceFolder = projectRoot
@@ -85,10 +94,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     init {
         ignorePatternsWithGitIgnore = try {
             buildList {
-                addAll(ignorePatterns)
-                parseGitIgnore().mapNotNull { pattern ->
-                    runCatching { Regex(pattern) }.getOrNull()
-                }.let { addAll(it) }
+                addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
+                addAll(parseGitIgnore())
+            }.mapNotNull { pattern ->
+                runCatching { Regex(pattern) }.getOrNull()
             }
         } catch (e: Exception) {
             emptyList()
@@ -108,21 +117,26 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     fun isFileExtensionAllowed(file: VirtualFile): Boolean {
         // if it is a directory, it is allowed
         if (file.isDirectory) return true
-
         val extension = file.extension ?: return false
         return ALLOWED_CODE_EXTENSIONS.contains(extension)
     }
 
     private fun ignoreFileByExtension(file: VirtualFile) =
         !isFileExtensionAllowed(file)
 
-    suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.path)
+    suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl)
 
     suspend fun ignoreFile(path: String): Boolean {
         // this method reads like something a JS dev would write and doesn't do what the author thinks
         val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
             withContext(coroutineContext) {
-                async { pattern.containsMatchIn(path) }
+                // avoid partial match (pattern.containsMatchIn) since it causes us matching files
+                // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
+                // we convert the glob rules to regex, add a trailing /* to all rules and then match
+                // entries against them by adding a trailing /.
+                // TODO: Add unit tests for gitignore matching
+                val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
+                async { pattern.matches("$relative/") }
             }
         }
 
@@ -132,7 +146,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         return deferredResults.any { it.await() }
     }
 
-    private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
+    fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
 
     suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) {
         val files = mutableListOf<VirtualFile>()
@@ -187,22 +201,41 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
             }
         }
 
-        createTemporaryZipFileAsync { zipOutput ->
+        val zipFilePath = createTemporaryZipFileAsync { zipfs ->
+            val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix")
             filesToIncludeFlow.collect { file ->
-                try {
-                    val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath())
-                    zipOutput.putNextEntry(relativePath.toString(), Path(file.path))
-                } catch (e: NoSuchFileException) {
-                    // Noop: Skip if file was deleted
+
+                if (!file.isDirectory) {
+                    val externalFilePath = Path(file.path)
+                    val relativePath = Path(file.path).relativeTo(projectRootPath)
+                    val zipfsPath = zipfs.getPath("/$relativePath")
+                    runBlocking {
+                        zipfsPath.createParentDirectories()
+                        try {
+                            Files.copy(externalFilePath, zipfsPath, StandardCopyOption.REPLACE_EXISTING)
+                            if (isPosix) {
+                                Files.setAttribute(zipfsPath, "zip:permissions", externalFilePath.getPosixFilePermissions())
+                            }
+                        } catch (e: NoSuchFileException) {
+                            // Noop: Skip if file was deleted
+                        }
+                    }
                 }
             }
         }
+        zipFilePath
     }.toFile()
 
-    private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) {
-        val file = Files.createTempFile(null, ".zip")
-        ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) }
-        file
+    private suspend fun createTemporaryZipFileAsync(block: suspend (FileSystem) -> Unit): Path = withContext(getCoroutineBgContext()) {
+        // Don't use Files.createTempFile since the file must not be created for ZipFS to work
+        val tempFilePath: Path = Paths.get(FileUtils.getTempDirectory().absolutePath, "${UUID.randomUUID()}.zip")
+        val uri = URI.create("jar:${tempFilePath.toUri()}")
+        val env = hashMapOf("create" to "true")
+        val zipfs = FileSystems.newFileSystem(uri, env)
+        zipfs.use {
+            block(zipfs)
+        }
+        tempFilePath
     }
 
     private fun parseGitIgnore(): Set<String> {
@@ -220,7 +253,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     private fun convertGitIgnorePatternToRegex(pattern: String): String = pattern
         .replace(".", "\\.")
         .replace("*", ".*")
-        .let { if (it.endsWith("/")) "$it?" else it } // Handle directory-specific patterns by optionally matching trailing slash
+        .let { if (it.endsWith("/")) "$it.*" else "$it/.*" } // Add a trailing /* to all patterns. (we add a trailing / to all files when matching)
 
     var selectedSourceFolder: VirtualFile
         set(newRoot) {

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt

@@ -16,6 +16,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
     "bash",
     "bat",
     "boo",
+    "bms",
     "c",
     "cbl",
     "cc",
@@ -27,6 +28,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
     "cljs",
     "cls",
     "cmake",
+    "cmd",
     "cob",
     "cobra",
     "coffee",
@@ -122,11 +124,13 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
     "pike",
     "pir",
     "pl",
+    "pli",
     "pm",
     "pmod",
     "pp",
     "pro",
     "prolog",
+    "properties",
     "ps1",
     "psd1",
     "psm1",
@@ -200,7 +204,7 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
     "xml",
     "yaml",
     "yml",
-    "zig"
+    "zig",
 )
 
 fun scrubNames(messageToBeScrubbed: String, username: String? = getSystemUserName()): String {