From a4e3b4d44e3fa8712b481cc5d3a5ff3f45cce0ab Mon Sep 17 00:00:00 2001 From: samgst-amazon Date: Wed, 9 Jul 2025 16:49:32 -0700 Subject: [PATCH] Revert "Revert "feat(amazonq): add image context support (#5846)" (#5890)" This reverts commit 161bf136d5f13d1f540faab6d0995242199cf4c4. --- ...-ddbdde68-769a-468f-bb42-e5aece51c729.json | 4 + .../amazonq/toolwindow/AmazonQPanel.kt | 110 +++++++++++++++++- .../services/amazonq/webview/Browser.kt | 30 ++++- .../amazonq/webview/BrowserConnector.kt | 15 +++ .../services/amazonq/lsp/AmazonQChatServer.kt | 7 ++ .../amazonq/lsp/AmazonQLanguageClient.kt | 5 + .../amazonq/lsp/AmazonQLanguageClientImpl.kt | 73 ++++++++++++ .../lsp/model/aws/chat/FlareChatCommands.kt | 3 + .../aws/chat/ShowOpenFileDialogParams.kt | 12 ++ 9 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 .changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json create mode 100644 plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/ShowOpenFileDialogParams.kt diff --git a/.changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json b/.changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json new file mode 100644 index 00000000000..c2163d366b6 --- /dev/null +++ b/.changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json @@ -0,0 +1,4 @@ +{ + "type" : "feature", + "description" : "Add image context support" +} \ No newline at end of file diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt index 53323b638b2..7fb75af2198 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt @@ -3,6 +3,7 @@ package software.aws.toolkits.jetbrains.services.amazonq.toolwindow +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.intellij.idea.AppMode import com.intellij.openapi.Disposable import com.intellij.openapi.components.service @@ -20,6 +21,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import software.aws.toolkits.core.utils.error +import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.jetbrains.core.coroutines.EDT import software.aws.toolkits.jetbrains.isDeveloperMode import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitContext @@ -44,7 +47,12 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.auth.isDocAvailable import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.auth.isFeatureDevAvailable import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isCodeTransformAvailable import software.aws.toolkits.resources.message +import java.awt.datatransfer.DataFlavor +import java.awt.dnd.DropTarget +import java.awt.dnd.DropTargetDropEvent +import java.io.File import java.util.concurrent.CompletableFuture +import javax.imageio.ImageIO.read import javax.swing.JButton class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Disposable { @@ -122,12 +130,76 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di withContext(EDT) { browser.complete( - Browser(this@AmazonQPanel, webUri, project).also { - wrapper.setContent(it.component()) + Browser(this@AmazonQPanel, webUri, project).also { browserInstance -> + wrapper.setContent(browserInstance.component()) + + // Add DropTarget to the browser component + // JCEF does not propagate OS-level dragenter, dragOver and drop into DOM. + // As an alternative, enabling the native drag in JCEF, + // and let the native handling the drop event, and update the UI through JS bridge. + val dropTarget = object : DropTarget() { + override fun drop(dtde: DropTargetDropEvent) { + try { + dtde.acceptDrop(dtde.dropAction) + val transferable = dtde.transferable + if (transferable.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) { + val fileList = transferable.getTransferData(DataFlavor.javaFileListFlavor) as List<*> + + val errorMessages = mutableListOf() + val validImages = mutableListOf() + val allowedTypes = setOf("jpg", "jpeg", "png", "gif", "webp") + val maxFileSize = 3.75 * 1024 * 1024 // 3.75MB in bytes + val maxDimension = 8000 + + for (file in fileList as List) { + val validationResult = validateImageFile(file, allowedTypes, maxFileSize, maxDimension) + if (validationResult != null) { + errorMessages.add(validationResult) + } else { + validImages.add(file) + } + } + + // File count restriction + if (validImages.size > 20) { + errorMessages.add("A maximum of 20 images can be added to a single message.") + validImages.subList(20, validImages.size).clear() + } + + val json = OBJECT_MAPPER.writeValueAsString(validImages) + browserInstance.jcefBrowser.cefBrowser.executeJavaScript( + "window.handleNativeDrop('$json')", + browserInstance.jcefBrowser.cefBrowser.url, + 0 + ) + + val errorJson = OBJECT_MAPPER.writeValueAsString(errorMessages) + browserInstance.jcefBrowser.cefBrowser.executeJavaScript( + "window.handleNativeNotify('$errorJson')", + browserInstance.jcefBrowser.cefBrowser.url, + 0 + ) + dtde.dropComplete(true) + } else { + dtde.dropComplete(false) + } + } catch (e: Exception) { + LOG.error { "Failed to handle file drop operation: ${e.message}" } + dtde.dropComplete(false) + } + } + } + + // Set DropTarget on the browser component and its children + browserInstance.component()?.let { component -> + component.dropTarget = dropTarget + // Also try setting on parent if needed + component.parent?.dropTarget = dropTarget + } initConnections() - connectUi(it) - connectApps(it) + connectUi(browserInstance) + connectApps(browserInstance) loadingPanel.stopLoading() } @@ -211,6 +283,36 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di } } + private fun validateImageFile(file: File, allowedTypes: Set, maxFileSize: Double, maxDimension: Int): String? { + val fileName = file.name + val ext = fileName.substringAfterLast('.', "").lowercase() + + if (ext !in allowedTypes) { + return "$fileName: File must be an image in JPEG, PNG, GIF, or WebP format." + } + + if (file.length() > maxFileSize) { + return "$fileName: Image must be no more than 3.75MB in size." + } + + return try { + val img = read(file) + when { + img == null -> "$fileName: File could not be read as an image." + img.width > maxDimension -> "$fileName: Image must be no more than 8,000px in width." + img.height > maxDimension -> "$fileName: Image must be no more than 8,000px in height." + else -> null + } + } catch (e: Exception) { + "$fileName: File could not be read as an image." + } + } + + companion object { + private val LOG = getLogger() + private val OBJECT_MAPPER = jacksonObjectMapper() + } + override fun dispose() { } } diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt index 27ec40a7894..c092e1bf2f8 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt @@ -85,6 +85,9 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project) // setup empty state. The message request handlers use this for storing state // that's persistent between page loads. jcefBrowser.setProperty("state", "") + jcefBrowser.jbCefClient.addDragHandler({ browser, dragData, mask -> + true // Allow drag operations + }, jcefBrowser.cefBrowser) // load the web app jcefBrowser.loadHTML( getWebviewHTML( @@ -122,7 +125,7 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project) """.trimIndent() diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt index 915a3a23343..5ac079c8ac6 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt @@ -77,6 +77,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSe import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_MCP_SERVERS_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_RULES_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_SETTINGS import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_WORKSPACE_SETTINGS_KEY import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenSettingsNotification @@ -489,6 +491,19 @@ class BrowserConnector( ) } } + + OPEN_FILE_DIALOG -> { + handleChat(AmazonQChatServer.showOpenFileDialog, node) + .whenComplete { response, _ -> + browser.postChat( + FlareUiMessage( + command = OPEN_FILE_DIALOG_REQUEST_METHOD, + params = response + ) + ) + } + } + LIST_RULES_REQUEST_METHOD -> { handleChat(AmazonQChatServer.listRules, node) .whenComplete { response, _ -> diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt index b2c8854cf6d..b2ac40fed03 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt @@ -45,6 +45,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LinkClickParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ListConversationsParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PROMPT_INPUT_OPTIONS_CHANGE import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PromptInputOptionChangeParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.RULE_CLICK_REQUEST_METHOD @@ -248,4 +249,10 @@ object AmazonQChatServer : JsonRpcMethodProvider { TELEMETRY_EVENT, Any::class.java ) + + val showOpenFileDialog = JsonRpcRequest( + OPEN_FILE_DIALOG_REQUEST_METHOD, + Any::class.java, + LSPAny::class.java + ) } diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt index 82f7a296327..e52d1e33296 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt @@ -25,7 +25,9 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_S import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIFF import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata @@ -45,6 +47,9 @@ interface AmazonQLanguageClient : LanguageClient { @JsonRequest(SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD) fun showSaveFileDialog(params: ShowSaveFileDialogParams): CompletableFuture + @JsonRequest(SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD) + fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture + @JsonRequest(GET_SERIALIZED_CHAT_REQUEST_METHOD) fun getSerializedChat(params: LSPAny): CompletableFuture diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClientImpl.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClientImpl.kt index 31eaa4d86b4..fe5f70bec8d 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClientImpl.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClientImpl.kt @@ -10,6 +10,8 @@ import com.intellij.ide.BrowserUtil import com.intellij.notification.NotificationAction import com.intellij.notification.NotificationType import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.fileChooser.FileChooser +import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory import com.intellij.openapi.fileChooser.FileChooserFactory import com.intellij.openapi.fileChooser.FileSaverDescriptor import com.intellij.openapi.fileEditor.FileEditorManager @@ -56,6 +58,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.FileP import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_SERIALIZED_CHAT_REQUEST_METHOD import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams +import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata @@ -255,6 +258,76 @@ class AmazonQLanguageClientImpl(private val project: Project) : AmazonQLanguageC ) } + override fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture = + CompletableFuture.supplyAsync( + { + // Handle the case where both canSelectFiles and canSelectFolders are false (should never be sent from flare) + if (!params.canSelectFiles && !params.canSelectFolders) { + return@supplyAsync mapOf("uris" to emptyList()) as LSPAny + } + + val descriptor = when { + params.canSelectFolders && params.canSelectFiles -> { + if (params.canSelectMany) { + FileChooserDescriptorFactory.createAllButJarContentsDescriptor() + } else { + FileChooserDescriptorFactory.createSingleFileOrFolderDescriptor() + } + } + params.canSelectFolders -> { + if (params.canSelectMany) { + FileChooserDescriptorFactory.createMultipleFoldersDescriptor() + } else { + FileChooserDescriptorFactory.createSingleFolderDescriptor() + } + } + else -> { + if (params.canSelectMany) { + FileChooserDescriptorFactory.createMultipleFilesNoJarsDescriptor() + } else { + FileChooserDescriptorFactory.createSingleFileNoJarsDescriptor() + } + } + }.apply { + withTitle( + params.title ?: when { + params.canSelectFolders && params.canSelectFiles -> "Select Files or Folders" + params.canSelectFolders -> "Select Folders" + else -> "Select Files" + } + ) + withDescription( + when { + params.canSelectFolders && params.canSelectFiles -> "Choose files or folders to open" + params.canSelectFolders -> "Choose folders to open" + else -> "Choose files to open" + } + ) + + // Apply file filters if provided + if (params.filters.isNotEmpty() && !params.canSelectFolders) { + // Create a combined list of all allowed extensions + val allowedExtensions = params.filters.values.flatten().toSet() + + withFileFilter { virtualFile -> + if (virtualFile.isDirectory) { + true // Always allow directories for navigation + } else { + val extension = virtualFile.extension?.lowercase() + extension != null && allowedExtensions.contains(extension) + } + } + } + } + + val chosenFiles = FileChooser.chooseFiles(descriptor, project, null) + val uris = chosenFiles.map { it.path } + + mapOf("uris" to uris) as LSPAny + }, + ApplicationManager.getApplication()::invokeLater + ) + override fun getSerializedChat(params: LSPAny): CompletableFuture { val requestId = UUID.randomUUID().toString() val result = CompletableFuture() diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/FlareChatCommands.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/FlareChatCommands.kt index 4de4c4501c2..19f1379ae10 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/FlareChatCommands.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/FlareChatCommands.kt @@ -52,12 +52,15 @@ const val PROMPT_INPUT_OPTIONS_CHANGE = "aws/chat/promptInputOptionChange" const val SEND_CHAT_COMMAND_PROMPT = "aws/chat/sendChatPrompt" const val SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD = "aws/showSaveFileDialog" +const val SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD = "aws/showOpenFileDialog" const val STOP_CHAT_RESPONSE = "stopChatResponse" const val SEND_TO_PROMPT = "sendToPrompt" +const val OPEN_FILE_DIALOG = "openFileDialog" const val TELEMETRY_EVENT = "telemetry/event" // https://github.com/aws/language-server-runtimes/blob/112feba70219a98a12f13727d67c540205fa9c9f/types/chat.ts#L32 const val LIST_MCP_SERVERS_REQUEST_METHOD = "aws/chat/listMcpServers" const val MCP_SERVER_CLICK_REQUEST_METHOD = "aws/chat/mcpServerClick" +const val OPEN_FILE_DIALOG_REQUEST_METHOD = "aws/chat/openFileDialog" const val LIST_RULES_REQUEST_METHOD = "aws/chat/listRules" const val RULE_CLICK_REQUEST_METHOD = "aws/chat/ruleClick" diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/ShowOpenFileDialogParams.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/ShowOpenFileDialogParams.kt new file mode 100644 index 00000000000..77a9da2e794 --- /dev/null +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/ShowOpenFileDialogParams.kt @@ -0,0 +1,12 @@ +// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat + +data class ShowOpenFileDialogParams( + val canSelectFiles: Boolean, + val canSelectFolders: Boolean, + val canSelectMany: Boolean, + val filters: Map>, + val title: String?, +)