Revert "Revert "feat(amazonq): add image context support (#5846)" (#5890)"

.changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json

@@ -0,0 +1,4 @@
+{
+  "type" : "feature",
+  "description" : "Add image context support"
+}

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt

@@ -3,6 +3,7 @@
 
 package software.aws.toolkits.jetbrains.services.amazonq.toolwindow
 
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.intellij.idea.AppMode
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.components.service
@@ -20,6 +21,8 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import software.aws.toolkits.core.utils.error
+import software.aws.toolkits.core.utils.getLogger
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
 import software.aws.toolkits.jetbrains.isDeveloperMode
 import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitContext
@@ -44,7 +47,12 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.auth.isDocAvailable
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.auth.isFeatureDevAvailable
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isCodeTransformAvailable
 import software.aws.toolkits.resources.message
+import java.awt.datatransfer.DataFlavor
+import java.awt.dnd.DropTarget
+import java.awt.dnd.DropTargetDropEvent
+import java.io.File
 import java.util.concurrent.CompletableFuture
+import javax.imageio.ImageIO.read
 import javax.swing.JButton
 
 class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Disposable {
@@ -122,12 +130,76 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di
 
                 withContext(EDT) {
                     browser.complete(
-                        Browser(this@AmazonQPanel, webUri, project).also {
-                            wrapper.setContent(it.component())
+                        Browser(this@AmazonQPanel, webUri, project).also { browserInstance ->
+                            wrapper.setContent(browserInstance.component())
+
+                            // Add DropTarget to the browser component
+                            // JCEF does not propagate OS-level dragenter, dragOver and drop into DOM.
+                            // As an alternative, enabling the native drag in JCEF,
+                            // and let the native handling the drop event, and update the UI through JS bridge.
+                            val dropTarget = object : DropTarget() {
+                                override fun drop(dtde: DropTargetDropEvent) {
+                                    try {
+                                        dtde.acceptDrop(dtde.dropAction)
+                                        val transferable = dtde.transferable
+                                        if (transferable.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) {
+                                            val fileList = transferable.getTransferData(DataFlavor.javaFileListFlavor) as List<*>
+
+                                            val errorMessages = mutableListOf<String>()
+                                            val validImages = mutableListOf<File>()
+                                            val allowedTypes = setOf("jpg", "jpeg", "png", "gif", "webp")
+                                            val maxFileSize = 3.75 * 1024 * 1024 // 3.75MB in bytes
+                                            val maxDimension = 8000
+
+                                            for (file in fileList as List<File>) {
+                                                val validationResult = validateImageFile(file, allowedTypes, maxFileSize, maxDimension)
+                                                if (validationResult != null) {
+                                                    errorMessages.add(validationResult)
+                                                } else {
+                                                    validImages.add(file)
+                                                }
+                                            }
+
+                                            // File count restriction
+                                            if (validImages.size > 20) {
+                                                errorMessages.add("A maximum of 20 images can be added to a single message.")
+                                                validImages.subList(20, validImages.size).clear()
+                                            }
+
+                                            val json = OBJECT_MAPPER.writeValueAsString(validImages)
+                                            browserInstance.jcefBrowser.cefBrowser.executeJavaScript(
+                                                "window.handleNativeDrop('$json')",
+                                                browserInstance.jcefBrowser.cefBrowser.url,
+                                                0
+                                            )
+
+                                            val errorJson = OBJECT_MAPPER.writeValueAsString(errorMessages)
+                                            browserInstance.jcefBrowser.cefBrowser.executeJavaScript(
+                                                "window.handleNativeNotify('$errorJson')",
+                                                browserInstance.jcefBrowser.cefBrowser.url,
+                                                0
+                                            )
+                                            dtde.dropComplete(true)
+                                        } else {
+                                            dtde.dropComplete(false)
+                                        }
+                                    } catch (e: Exception) {
+                                        LOG.error { "Failed to handle file drop operation: ${e.message}" }
+                                        dtde.dropComplete(false)
+                                    }
+                                }
+                            }
+
+                            // Set DropTarget on the browser component and its children
+                            browserInstance.component()?.let { component ->
+                                component.dropTarget = dropTarget
+                                // Also try setting on parent if needed
+                                component.parent?.dropTarget = dropTarget
+                            }
 
                             initConnections()
-                            connectUi(it)
-                            connectApps(it)
+                            connectUi(browserInstance)
+                            connectApps(browserInstance)
 
                             loadingPanel.stopLoading()
                         }
@@ -211,6 +283,36 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di
         }
     }
 
+    private fun validateImageFile(file: File, allowedTypes: Set<String>, maxFileSize: Double, maxDimension: Int): String? {
+        val fileName = file.name
+        val ext = fileName.substringAfterLast('.', "").lowercase()
+
+        if (ext !in allowedTypes) {
+            return "$fileName: File must be an image in JPEG, PNG, GIF, or WebP format."
+        }
+
+        if (file.length() > maxFileSize) {
+            return "$fileName: Image must be no more than 3.75MB in size."
+        }
+
+        return try {
+            val img = read(file)
+            when {
+                img == null -> "$fileName: File could not be read as an image."
+                img.width > maxDimension -> "$fileName: Image must be no more than 8,000px in width."
+                img.height > maxDimension -> "$fileName: Image must be no more than 8,000px in height."
+                else -> null
+            }
+        } catch (e: Exception) {
+            "$fileName: File could not be read as an image."
+        }
+    }
+
+    companion object {
+        private val LOG = getLogger<AmazonQPanel>()
+        private val OBJECT_MAPPER = jacksonObjectMapper()
+    }
+
     override fun dispose() {
     }
 }

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt

@@ -85,6 +85,9 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
         // setup empty state. The message request handlers use this for storing state
         // that's persistent between page loads.
         jcefBrowser.setProperty("state", "")
+        jcefBrowser.jbCefClient.addDragHandler({ browser, dragData, mask ->
+            true // Allow drag operations
+        }, jcefBrowser.cefBrowser)
         // load the web app
         jcefBrowser.loadHTML(
             getWebviewHTML(
@@ -122,7 +125,7 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
             <script type="text/javascript" charset="UTF-8" src="$webUri" defer onload="init()"></script>
 
             <script type="text/javascript">
-
+                
                 const init = () => {
                     const hybridChatConnector = connectorAdapter.initiateAdapter(
                      ${MeetQSettings.getInstance().reinvent2024OnboardingCount < MAX_ONBOARDING_PAGE_COUNT},
@@ -139,7 +142,7 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
                         },
 
                      "${activeProfile?.profileName.orEmpty()}")
-                    amazonQChat.createChat(
+                    const qChat = amazonQChat.createChat(
                         {
                             postMessage: message => {
                                 $postMessageToJavaJsCode
@@ -155,6 +158,29 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
                         hybridChatConnector,
                         ${CodeWhispererFeatureConfigService.getInstance().getFeatureConfigJsonString()}                     
                     );
+
+                    window.handleNativeDrop = function(filePath) {
+                        const parsedFilePath = JSON.parse(filePath);
+                        const contextArray = parsedFilePath.map(fullPath => {
+                            const fileName = fullPath.split(/[\\/]/).pop();
+                            return {
+                                command: fileName,
+                                label: 'image',
+                                route: [fullPath],
+                                description: fullPath
+                            };
+                        });
+                        qChat.addCustomContextToPrompt(qChat.getSelectedTabId(), contextArray);
+                    };
+
+                    window.handleNativeNotify = function(errorMessages) {
+                        const messages = JSON.parse(errorMessages);
+                        messages.forEach(msg => {
+                            qChat.notify({
+                                content: msg
+                            })
+                        });
+                    };
                 }
             </script>        
         """.trimIndent()

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt

@@ -77,6 +77,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSe
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_MCP_SERVERS_REQUEST_METHOD
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_RULES_REQUEST_METHOD
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
+import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG
+import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_SETTINGS
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_WORKSPACE_SETTINGS_KEY
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenSettingsNotification
@@ -489,6 +491,19 @@ class BrowserConnector(
                         )
                     }
             }
+
+            OPEN_FILE_DIALOG -> {
+                handleChat(AmazonQChatServer.showOpenFileDialog, node)
+                    .whenComplete { response, _ ->
+                        browser.postChat(
+                            FlareUiMessage(
+                                command = OPEN_FILE_DIALOG_REQUEST_METHOD,
+                                params = response
+                            )
+                        )
+                    }
+            }
+
             LIST_RULES_REQUEST_METHOD -> {
                 handleChat(AmazonQChatServer.listRules, node)
                     .whenComplete { response, _ ->

plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt

@@ -45,6 +45,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LinkClickParams
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ListConversationsParams
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
+import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PROMPT_INPUT_OPTIONS_CHANGE
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PromptInputOptionChangeParams
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.RULE_CLICK_REQUEST_METHOD
@@ -248,4 +249,10 @@ object AmazonQChatServer : JsonRpcMethodProvider {
         TELEMETRY_EVENT,
         Any::class.java
     )
+
+    val showOpenFileDialog = JsonRpcRequest(
+        OPEN_FILE_DIALOG_REQUEST_METHOD,
+        Any::class.java,
+        LSPAny::class.java
+    )
 }

plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt

@@ -25,7 +25,9 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_S
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIFF
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams
+import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD
+import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult
 import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata
@@ -45,6 +47,9 @@ interface AmazonQLanguageClient : LanguageClient {
     @JsonRequest(SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD)
     fun showSaveFileDialog(params: ShowSaveFileDialogParams): CompletableFuture<ShowSaveFileDialogResult>
 
+    @JsonRequest(SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD)
+    fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture<LSPAny>
+
     @JsonRequest(GET_SERIALIZED_CHAT_REQUEST_METHOD)
     fun getSerializedChat(params: LSPAny): CompletableFuture<GetSerializedChatResult>