Preserve file permissions in the zip archive, fix bugs in gitignore matching and keep mvm and gradle files

.changes/next-release/bugfix-2df57233-310e-4788-be6c-f3b37767e60a.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "Preserve file permissions in the zip archive, fix bugs in gitignore matching and keep mvm and gradle files"
+}

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt

@@ -40,13 +40,25 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() {
     fun testWithValidFile() {
         val ktFile = mock<VirtualFile>()
         whenever(ktFile.extension).thenReturn("kt")
+        whenever(ktFile.path).thenReturn("code.kt")
         assertTrue(featureDevSessionContext.isFileExtensionAllowed(ktFile))
     }
 
     @Test
     fun testWithInvalidFile() {
         val txtFile = mock<VirtualFile>()
         whenever(txtFile.extension).thenReturn("txt")
+        whenever(txtFile.path).thenReturn("file.txt")
         assertFalse(featureDevSessionContext.isFileExtensionAllowed(txtFile))
     }
+
+    @Test
+    fun testAllowedFilePath() {
+        val allowedPaths = listOf("gradlew", "build.gradle", "gradle.properties", ".mvn/wrapper/maven-wrapper.properties")
+        allowedPaths.forEach({
+            val txtFile = mock<VirtualFile>()
+            whenever(txtFile.path).thenReturn(it)
+            assertTrue(featureDevSessionContext.isFileExtensionAllowed(txtFile))
+        })
+    }
 }

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt

@@ -372,6 +372,6 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
         // Adding gitignore file and gitignore file member for testing.
         // The tests include the markdown file but not these two files.
         projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile
-        projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main")
+        projectRule.fixture.addFileToProject("/.idea/ref", "ref: refs/heads/main")
     }
 }

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -16,21 +16,27 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.apache.commons.codec.digest.DigestUtils
-import software.aws.toolkits.core.utils.outputStream
-import software.aws.toolkits.core.utils.putNextEntry
+import org.apache.commons.io.FileUtils
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
 import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
 import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
 import software.aws.toolkits.resources.AwsCoreBundle
 import software.aws.toolkits.telemetry.AmazonqTelemetry
 import java.io.File
 import java.io.FileInputStream
+import java.net.URI
+import java.nio.file.FileSystem
+import java.nio.file.FileSystems
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 import java.util.Base64
-import java.util.zip.ZipOutputStream
+import java.util.UUID
 import kotlin.coroutines.coroutineContext
 import kotlin.io.path.Path
+import kotlin.io.path.createParentDirectories
+import kotlin.io.path.getPosixFilePermissions
 import kotlin.io.path.relativeTo
 
 interface RepoSizeError {
@@ -41,41 +47,49 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
 class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
     // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.
 
-    private val ignorePatterns = setOf(
-        "\\.aws-sam",
-        "\\.svn",
-        "\\.hg/?",
-        "\\.rvm",
-        "\\.git/?",
-        "\\.gitignore",
-        "\\.project",
-        "\\.gem",
-        "/\\.idea/?",
-        "\\.zip$",
-        "\\.bin$",
-        "\\.png$",
-        "\\.jpg$",
-        "\\.svg$",
-        "\\.pyc$",
-        "/license\\.txt$",
-        "/License\\.txt$",
-        "/LICENSE\\.txt$",
-        "/license\\.md$",
-        "/License\\.md$",
-        "/LICENSE\\.md$",
-        "node_modules/?",
-        "build/?",
-        "dist/?"
-    ).map { Regex(it) }
+    private val additionalGitIgnoreRules = setOf(
+        ".aws-sam",
+        ".gem",
+        ".git",
+        ".gitignore",
+        ".gradle",
+        ".hg",
+        ".idea",
+        ".project",
+        ".rvm",
+        ".svn",
+        "*.zip",
+        "*.bin",
+        "*.png",
+        "*.jpg",
+        "*.svg",
+        "*.pyc",
+        "license.txt",
+        "License.txt",
+        "LICENSE.txt",
+        "license.md",
+        "License.md",
+        "LICENSE.md",
+        "node_modules",
+        "build",
+        "dist"
+    )
 
     // well known source files that do not have extensions
     private val wellKnownSourceFiles = setOf(
         "Dockerfile",
         "Dockerfile.build"
     )
 
+    // patterns to explicitly allow unless matched with gitignore rules
+    private val allowedPatterns = setOf(
+        ".*mvn.*",
+        ".*gradle.*",
+    ).map { Regex(it) }
+
     // projectRoot: is the directory where the project is located when selected to open a project.
     val projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}")
+    private val projectRootPath = Paths.get(projectRoot.path) ?: error("Can not find project root path")
 
     // selectedSourceFolder": is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
     private var _selectedSourceFolder = projectRoot
@@ -85,10 +99,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     init {
         ignorePatternsWithGitIgnore = try {
             buildList {
-                addAll(ignorePatterns)
-                parseGitIgnore().mapNotNull { pattern ->
-                    runCatching { Regex(pattern) }.getOrNull()
-                }.let { addAll(it) }
+                addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
+                addAll(parseGitIgnore())
+            }.mapNotNull { pattern ->
+                runCatching { Regex(pattern) }.getOrNull()
             }
         } catch (e: Exception) {
             emptyList()
@@ -108,7 +122,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     fun isFileExtensionAllowed(file: VirtualFile): Boolean {
         // if it is a directory, it is allowed
         if (file.isDirectory) return true
-
+        val explicitAllowed = allowedPatterns.map { pattern -> pattern.matches(file.path) }.any { it }
+        if (explicitAllowed) {
+            return true
+        }
         val extension = file.extension ?: return false
         return ALLOWED_CODE_EXTENSIONS.contains(extension)
     }
@@ -122,7 +139,13 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         // this method reads like something a JS dev would write and doesn't do what the author thinks
         val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
             withContext(coroutineContext) {
-                async { pattern.containsMatchIn(path) }
+                // avoid partial match (pattern.containsMatchIn) since it causes us matching files
+                // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
+                // we convert the glob rules to regex, add a trailing /* to all rules and then match
+                // entries against them by adding a trailing /.
+                // TODO: Add unit tests for gitignore matching
+                val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
+                async { pattern.matches("$relative/") }
             }
         }
 
@@ -187,18 +210,34 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
             }
         }
 
-        createTemporaryZipFileAsync { zipOutput ->
+        val zipFilePath = createTemporaryZipFileAsync { zipfs ->
             filesToIncludeFlow.collect { file ->
-                val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath())
-                zipOutput.putNextEntry(relativePath.toString(), Path(file.path))
+                if (!file.isDirectory) {
+                    val externalFilePath = Path(file.path)
+                    val externalFilePermissions = externalFilePath.getPosixFilePermissions()
+                    val relativePath = Path(file.path).relativeTo(projectRootPath)
+                    val zipfsPath = zipfs.getPath("/$relativePath")
+                    withContext(EDT) {
+                        zipfsPath.createParentDirectories()
+                        Files.copy(externalFilePath, zipfsPath, StandardCopyOption.REPLACE_EXISTING)
+                        Files.setAttribute(zipfsPath, "zip:permissions", externalFilePermissions)
+                    }
+                }
             }
         }
+        zipFilePath
     }.toFile()
 
-    private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) {
-        val file = Files.createTempFile(null, ".zip")
-        ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) }
-        file
+    private suspend fun createTemporaryZipFileAsync(block: suspend (FileSystem) -> Unit): Path = withContext(EDT) {
+        // Don't use Files.createTempFile since the file must not be created for ZipFS to work
+        val tempFilePath: Path = Paths.get(FileUtils.getTempDirectory().absolutePath, "${UUID.randomUUID()}.zip")
+        val uri = URI.create("jar:file:$tempFilePath")
+        val env = hashMapOf("create" to "true")
+        val zipfs = FileSystems.newFileSystem(uri, env)
+        zipfs.use {
+            block(zipfs)
+        }
+        tempFilePath
     }
 
     private fun parseGitIgnore(): Set<String> {
@@ -216,7 +255,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     private fun convertGitIgnorePatternToRegex(pattern: String): String = pattern
         .replace(".", "\\.")
         .replace("*", ".*")
-        .let { if (it.endsWith("/")) "$it?" else it } // Handle directory-specific patterns by optionally matching trailing slash
+        .let { if (it.endsWith("/")) "$it.*" else "$it/.*" } // Add a trailing /* to all patterns. (we add a trailing / to all files when matching)
 
     var selectedSourceFolder: VirtualFile
         set(newRoot) {