fix(amazonq): archives now preserve file premissions and amazon q can edit nvm/gradle files

Author: Hamed Soleimani

plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt

@@ -13,8 +13,17 @@ import org.mockito.kotlin.whenever
 import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.FeatureDevTestBase
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.FeatureDevService
+import software.aws.toolkits.jetbrains.utils.rules.HeavyJavaCodeInsightTestFixtureRule
+import software.aws.toolkits.jetbrains.utils.rules.addFileToModule
+import java.util.zip.ZipFile
+
+class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTestFixtureRule()) {
+
+    private fun addFilesToProjectModule(vararg path: String) {
+        val module = projectRule.module
+        path.forEach { projectRule.fixture.addFileToModule(module, it, it) }
+    }
 
-class FeatureDevSessionContextTest : FeatureDevTestBase() {
     @Rule
     @JvmField
     val ruleChain = RuleChain(projectRule, disposableRule)
@@ -40,13 +49,64 @@ class FeatureDevSessionContextTest : FeatureDevTestBase() {
     fun testWithValidFile() {
         val ktFile = mock<VirtualFile>()
         whenever(ktFile.extension).thenReturn("kt")
+        whenever(ktFile.path).thenReturn("code.kt")
         assertTrue(featureDevSessionContext.isFileExtensionAllowed(ktFile))
     }
 
     @Test
     fun testWithInvalidFile() {
         val txtFile = mock<VirtualFile>()
         whenever(txtFile.extension).thenReturn("txt")
+        whenever(txtFile.path).thenReturn("file.txt")
         assertFalse(featureDevSessionContext.isFileExtensionAllowed(txtFile))
     }
+
+    @Test
+    fun testAllowedFilePath() {
+        val allowedPaths = listOf("build.gradle", "gradle.properties", ".mvn/wrapper/maven-wrapper.properties")
+        allowedPaths.forEach({
+            val txtFile = mock<VirtualFile>()
+            whenever(txtFile.path).thenReturn(it)
+            whenever(txtFile.extension).thenReturn(it.split(".").last())
+            assertTrue(featureDevSessionContext.isFileExtensionAllowed(txtFile))
+        })
+    }
+
+    @Test
+    fun testZipProject() {
+        addFilesToProjectModule(
+            ".gradle/cached.jar",
+            "src/MyClass.java",
+            "gradlew",
+            "gradlew.bat",
+            "README.md",
+            "settings.gradle",
+            "build.gradle",
+            "gradle/wrapper/gradle-wrapper.properties",
+        )
+
+        val zipResult = featureDevSessionContext.getProjectZip()
+        val zipPath = zipResult.payload.path
+
+        val zippedFiles = mutableSetOf<String>()
+        ZipFile(zipPath).use { zipFile ->
+            for (entry in zipFile.entries()) {
+                if (!entry.name.endsWith("/")) {
+                    zippedFiles.add(entry.name)
+                }
+            }
+        }
+
+        val expectedFiles = setOf(
+            "src/MyClass.java",
+            "gradlew",
+            "gradlew.bat",
+            "README.md",
+            "settings.gradle",
+            "build.gradle",
+            "gradle/wrapper/gradle-wrapper.properties",
+        )
+
+        assertTrue(zippedFiles == expectedFiles)
+    }
 }

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt

@@ -113,7 +113,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
         val testModule2 = projectRule.fixture.addModule("testModule2")
         testCs = projectRule.fixture.addFileToModule(
             testModule,
-            "/Test.cs",
+            "Test.cs",
             """
             using Utils;
             using Helpers.Helper;
@@ -131,7 +131,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         utilsCs = projectRule.fixture.addFileToModule(
             testModule,
-            "/Utils.cs",
+            "Utils.cs",
             """
             public static class Utils
             {
@@ -157,7 +157,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         helperCs = projectRule.fixture.addFileToModule(
             testModule,
-            "/Helpers/Helper.cs",
+            "Helpers/Helper.cs",
             """
             public static class Helper
             {
@@ -201,7 +201,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         helpGo = projectRule.fixture.addFileToModule(
             testModule,
-            "/help.go",
+            "help.go",
             """
                 package main
 
@@ -217,7 +217,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         utilsJs = projectRule.fixture.addFileToModule(
             testModule,
-            "/utils.js",
+            "utils.js",
             """
             function add(num1, num2) {
               return num1 + num2;
@@ -247,7 +247,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         testJson = projectRule.fixture.addFileToModule(
             testModule,
-            "/helpers/test3Json.json",
+            "helpers/test3Json.json",
             """
                 {
                     "AWSTemplateFormatVersion": "2010-09-09",
@@ -303,7 +303,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         helperPy = projectRule.fixture.addFileToModule(
             testModule,
-            "/helpers/helper.py",
+            "helpers/helper.py",
             """
             from helpers import helper as h
             def subtract(num1, num2)
@@ -319,13 +319,13 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
         totalSize += helperPy.length
         totalLines += helperPy.toNioPath().toFile().readLines().size
 
-        readMeMd = projectRule.fixture.addFileToModule(testModule, "/ReadMe.md", "### Now included").virtualFile
+        readMeMd = projectRule.fixture.addFileToModule(testModule, "ReadMe.md", "### Now included").virtualFile
         totalSize += readMeMd.length
         totalLines += readMeMd.toNioPath().toFile().readLines().size
 
         testTf = projectRule.fixture.addFileToModule(
             testModule2,
-            "/testTf.tf",
+            "testTf.tf",
             """
                 # Create example resource for three S3 buckets using for_each, where the bucket prefix are in variable with list containing [prod, staging, dev]
                 
@@ -345,7 +345,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         testYaml = projectRule.fixture.addFileToModule(
             testModule2,
-            "/testYaml.yaml",
+            "testYaml.yaml",
             """
                 AWSTemplateFormatVersion: "2010-09-09"
                 
@@ -371,7 +371,7 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
 
         // Adding gitignore file and gitignore file member for testing.
         // The tests include the markdown file but not these two files.
-        projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile
-        projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main")
+        projectRule.fixture.addFileToProject(".gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile
+        projectRule.fixture.addFileToProject(".idea/ref", "ref: refs/heads/main")
     }
 }

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -16,21 +16,26 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.apache.commons.codec.digest.DigestUtils
-import software.aws.toolkits.core.utils.outputStream
-import software.aws.toolkits.core.utils.putNextEntry
-import software.aws.toolkits.jetbrains.core.coroutines.EDT
+import org.apache.commons.io.FileUtils
 import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
 import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
 import software.aws.toolkits.resources.AwsCoreBundle
 import software.aws.toolkits.telemetry.AmazonqTelemetry
 import java.io.File
 import java.io.FileInputStream
+import java.net.URI
+import java.nio.file.FileSystem
+import java.nio.file.FileSystems
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 import java.util.Base64
-import java.util.zip.ZipOutputStream
+import java.util.UUID
 import kotlin.coroutines.coroutineContext
 import kotlin.io.path.Path
+import kotlin.io.path.createParentDirectories
+import kotlin.io.path.getPosixFilePermissions
 import kotlin.io.path.relativeTo
 
 interface RepoSizeError {
@@ -41,41 +46,45 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
 class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
     // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.
 
-    private val ignorePatterns = setOf(
-        "\\.aws-sam",
-        "\\.svn",
-        "\\.hg/?",
-        "\\.rvm",
-        "\\.git/?",
-        "\\.gitignore",
-        "\\.project",
-        "\\.gem",
-        "/\\.idea/?",
-        "\\.zip$",
-        "\\.bin$",
-        "\\.png$",
-        "\\.jpg$",
-        "\\.svg$",
-        "\\.pyc$",
-        "/license\\.txt$",
-        "/License\\.txt$",
-        "/LICENSE\\.txt$",
-        "/license\\.md$",
-        "/License\\.md$",
-        "/LICENSE\\.md$",
-        "node_modules/?",
-        "build/?",
-        "dist/?"
-    ).map { Regex(it) }
+    private val additionalGitIgnoreRules = setOf(
+        ".aws-sam",
+        ".gem",
+        ".git",
+        ".gitignore",
+        ".gradle",
+        ".hg",
+        ".idea",
+        ".project",
+        ".rvm",
+        ".svn",
+        "*.zip",
+        "*.bin",
+        "*.png",
+        "*.jpg",
+        "*.svg",
+        "*.pyc",
+        "license.txt",
+        "License.txt",
+        "LICENSE.txt",
+        "license.md",
+        "License.md",
+        "LICENSE.md",
+        "node_modules",
+        "build",
+        "dist"
+    )
 
     // well known source files that do not have extensions
     private val wellKnownSourceFiles = setOf(
         "Dockerfile",
-        "Dockerfile.build"
+        "Dockerfile.build",
+        "gradlew",
+        "mvnw"
     )
 
     // projectRoot: is the directory where the project is located when selected to open a project.
     val projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}")
+    private val projectRootPath = Paths.get(projectRoot.path) ?: error("Can not find project root path")
 
     // selectedSourceFolder": is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
     private var _selectedSourceFolder = projectRoot
@@ -85,10 +94,10 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     init {
         ignorePatternsWithGitIgnore = try {
             buildList {
-                addAll(ignorePatterns)
-                parseGitIgnore().mapNotNull { pattern ->
-                    runCatching { Regex(pattern) }.getOrNull()
-                }.let { addAll(it) }
+                addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
+                addAll(parseGitIgnore())
+            }.mapNotNull { pattern ->
+                runCatching { Regex(pattern) }.getOrNull()
             }
         } catch (e: Exception) {
             emptyList()
@@ -108,7 +117,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     fun isFileExtensionAllowed(file: VirtualFile): Boolean {
         // if it is a directory, it is allowed
         if (file.isDirectory) return true
-
         val extension = file.extension ?: return false
         return ALLOWED_CODE_EXTENSIONS.contains(extension)
     }
@@ -122,7 +130,13 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         // this method reads like something a JS dev would write and doesn't do what the author thinks
         val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
             withContext(coroutineContext) {
-                async { pattern.containsMatchIn(path) }
+                // avoid partial match (pattern.containsMatchIn) since it causes us matching files
+                // against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
+                // we convert the glob rules to regex, add a trailing /* to all rules and then match
+                // entries against them by adding a trailing /.
+                // TODO: Add unit tests for gitignore matching
+                val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
+                async { pattern.matches("$relative/") }
             }
         }
 
@@ -132,7 +146,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
         return deferredResults.any { it.await() }
     }
 
-    private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
+    fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
 
     suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) {
         val files = mutableListOf<VirtualFile>()
@@ -187,18 +201,36 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
             }
         }
 
-        createTemporaryZipFileAsync { zipOutput ->
+        val zipFilePath = createTemporaryZipFileAsync { zipfs ->
+            val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix")
             filesToIncludeFlow.collect { file ->
-                val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath())
-                zipOutput.putNextEntry(relativePath.toString(), Path(file.path))
+                if (!file.isDirectory) {
+                    val externalFilePath = Path(file.path)
+                    val relativePath = Path(file.path).relativeTo(projectRootPath)
+                    val zipfsPath = zipfs.getPath("/$relativePath")
+                    runBlocking {
+                        zipfsPath.createParentDirectories()
+                        Files.copy(externalFilePath, zipfsPath, StandardCopyOption.REPLACE_EXISTING)
+                        if (isPosix) {
+                            Files.setAttribute(zipfsPath, "zip:permissions", externalFilePath.getPosixFilePermissions())
+                        }
+                    }
+                }
             }
         }
+        zipFilePath
     }.toFile()
 
-    private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) {
-        val file = Files.createTempFile(null, ".zip")
-        ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) }
-        file
+    private suspend fun createTemporaryZipFileAsync(block: suspend (FileSystem) -> Unit): Path = withContext(getCoroutineBgContext()) {
+        // Don't use Files.createTempFile since the file must not be created for ZipFS to work
+        val tempFilePath: Path = Paths.get(FileUtils.getTempDirectory().absolutePath, "${UUID.randomUUID()}.zip")
+        val uri = URI.create("jar:${tempFilePath.toUri()}")
+        val env = hashMapOf("create" to "true")
+        val zipfs = FileSystems.newFileSystem(uri, env)
+        zipfs.use {
+            block(zipfs)
+        }
+        tempFilePath
     }
 
     private fun parseGitIgnore(): Set<String> {
@@ -216,7 +248,7 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
     private fun convertGitIgnorePatternToRegex(pattern: String): String = pattern
         .replace(".", "\\.")
         .replace("*", ".*")
-        .let { if (it.endsWith("/")) "$it?" else it } // Handle directory-specific patterns by optionally matching trailing slash
+        .let { if (it.endsWith("/")) "$it.*" else "$it/.*" } // Add a trailing /* to all patterns. (we add a trailing / to all files when matching)
 
     var selectedSourceFolder: VirtualFile
         set(newRoot) {