Skip to content

Commit 83ac4f5

Browse files
zuoyaofuyzhangokrli
authored
feat(amazonq): add image context support (#5846)
* feat(q): enable image context * feat(q): add image context support * add change log + fix some styles * pass the dropped files from Kotlin to webView * use params to select different file descriptor * add feature flag * fixed comments * Update plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/model/ExtendedClientMetadata.kt not set default value Co-authored-by: Richard Li <[email protected]> * fix comments * use old function in order to compile * fix ci * fix more styles * use updated path * more CI fixes * change to use jacksonObjectMapper * fix ci --------- Co-authored-by: Yang Zhang <[email protected]> Co-authored-by: yzhangok <[email protected]> Co-authored-by: Richard Li <[email protected]>
1 parent 5608b8f commit 83ac4f5

File tree

10 files changed

+255
-6
lines changed

10 files changed

+255
-6
lines changed

‎.changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type" : "feature",
3+
"description" : "Add image context support"
4+
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/toolwindow/AmazonQPanel.kt

Lines changed: 106 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package software.aws.toolkits.jetbrains.services.amazonq.toolwindow
55

6+
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
67
import com.intellij.idea.AppMode
78
import com.intellij.openapi.Disposable
89
import com.intellij.openapi.components.service
@@ -20,6 +21,8 @@ import kotlinx.coroutines.CoroutineScope
2021
import kotlinx.coroutines.flow.first
2122
import kotlinx.coroutines.launch
2223
import kotlinx.coroutines.withContext
24+
import software.aws.toolkits.core.utils.error
25+
import software.aws.toolkits.core.utils.getLogger
2326
import software.aws.toolkits.jetbrains.core.coroutines.EDT
2427
import software.aws.toolkits.jetbrains.isDeveloperMode
2528
import software.aws.toolkits.jetbrains.services.amazonq.apps.AmazonQAppInitContext
@@ -44,7 +47,12 @@ import software.aws.toolkits.jetbrains.services.amazonqDoc.auth.isDocAvailable
4447
import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.auth.isFeatureDevAvailable
4548
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isCodeTransformAvailable
4649
import software.aws.toolkits.resources.message
50+
import java.awt.datatransfer.DataFlavor
51+
import java.awt.dnd.DropTarget
52+
import java.awt.dnd.DropTargetDropEvent
53+
import java.io.File
4754
import java.util.concurrent.CompletableFuture
55+
import javax.imageio.ImageIO.read
4856
import javax.swing.JButton
4957

5058
class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Disposable {
@@ -122,12 +130,76 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di
122130

123131
withContext(EDT) {
124132
browser.complete(
125-
Browser(this@AmazonQPanel, webUri, project).also {
126-
wrapper.setContent(it.component())
133+
Browser(this@AmazonQPanel, webUri, project).also { browserInstance ->
134+
wrapper.setContent(browserInstance.component())
135+
136+
// Add DropTarget to the browser component
137+
// JCEF does not propagate OS-level dragenter, dragOver and drop into DOM.
138+
// As an alternative, enabling the native drag in JCEF,
139+
// and let the native handling the drop event, and update the UI through JS bridge.
140+
val dropTarget = object : DropTarget() {
141+
override fun drop(dtde: DropTargetDropEvent) {
142+
try {
143+
dtde.acceptDrop(dtde.dropAction)
144+
val transferable = dtde.transferable
145+
if (transferable.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) {
146+
val fileList = transferable.getTransferData(DataFlavor.javaFileListFlavor) as List<*>
147+
148+
val errorMessages = mutableListOf<String>()
149+
val validImages = mutableListOf<File>()
150+
val allowedTypes = setOf("jpg", "jpeg", "png", "gif", "webp")
151+
val maxFileSize = 3.75 * 1024 * 1024 // 3.75MB in bytes
152+
val maxDimension = 8000
153+
154+
for (file in fileList as List<File>) {
155+
val validationResult = validateImageFile(file, allowedTypes, maxFileSize, maxDimension)
156+
if (validationResult != null) {
157+
errorMessages.add(validationResult)
158+
} else {
159+
validImages.add(file)
160+
}
161+
}
162+
163+
// File count restriction
164+
if (validImages.size > 20) {
165+
errorMessages.add("A maximum of 20 images can be added to a single message.")
166+
validImages.subList(20, validImages.size).clear()
167+
}
168+
169+
val json = OBJECT_MAPPER.writeValueAsString(validImages)
170+
browserInstance.jcefBrowser.cefBrowser.executeJavaScript(
171+
"window.handleNativeDrop('$json')",
172+
browserInstance.jcefBrowser.cefBrowser.url,
173+
0
174+
)
175+
176+
val errorJson = OBJECT_MAPPER.writeValueAsString(errorMessages)
177+
browserInstance.jcefBrowser.cefBrowser.executeJavaScript(
178+
"window.handleNativeNotify('$errorJson')",
179+
browserInstance.jcefBrowser.cefBrowser.url,
180+
0
181+
)
182+
dtde.dropComplete(true)
183+
} else {
184+
dtde.dropComplete(false)
185+
}
186+
} catch (e: Exception) {
187+
LOG.error { "Failed to handle file drop operation: ${e.message}" }
188+
dtde.dropComplete(false)
189+
}
190+
}
191+
}
192+
193+
// Set DropTarget on the browser component and its children
194+
browserInstance.component()?.let { component ->
195+
component.dropTarget = dropTarget
196+
// Also try setting on parent if needed
197+
component.parent?.dropTarget = dropTarget
198+
}
127199

128200
initConnections()
129-
connectUi(it)
130-
connectApps(it)
201+
connectUi(browserInstance)
202+
connectApps(browserInstance)
131203

132204
loadingPanel.stopLoading()
133205
}
@@ -211,6 +283,36 @@ class AmazonQPanel(val project: Project, private val scope: CoroutineScope) : Di
211283
}
212284
}
213285

286+
private fun validateImageFile(file: File, allowedTypes: Set<String>, maxFileSize: Double, maxDimension: Int): String? {
287+
val fileName = file.name
288+
val ext = fileName.substringAfterLast('.', "").lowercase()
289+
290+
if (ext !in allowedTypes) {
291+
return "$fileName: File must be an image in JPEG, PNG, GIF, or WebP format."
292+
}
293+
294+
if (file.length() > maxFileSize) {
295+
return "$fileName: Image must be no more than 3.75MB in size."
296+
}
297+
298+
return try {
299+
val img = read(file)
300+
when {
301+
img == null -> "$fileName: File could not be read as an image."
302+
img.width > maxDimension -> "$fileName: Image must be no more than 8,000px in width."
303+
img.height > maxDimension -> "$fileName: Image must be no more than 8,000px in height."
304+
else -> null
305+
}
306+
} catch (e: Exception) {
307+
"$fileName: File could not be read as an image."
308+
}
309+
}
310+
311+
companion object {
312+
private val LOG = getLogger<AmazonQPanel>()
313+
private val OBJECT_MAPPER = jacksonObjectMapper()
314+
}
315+
214316
override fun dispose() {
215317
}
216318
}

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
8585
// setup empty state. The message request handlers use this for storing state
8686
// that's persistent between page loads.
8787
jcefBrowser.setProperty("state", "")
88+
jcefBrowser.jbCefClient.addDragHandler({ browser, dragData, mask ->
89+
true // Allow drag operations
90+
}, jcefBrowser.cefBrowser)
8891
// load the web app
8992
jcefBrowser.loadHTML(
9093
getWebviewHTML(
@@ -122,7 +125,7 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
122125
<script type="text/javascript" charset="UTF-8" src="$webUri" defer onload="init()"></script>
123126
124127
<script type="text/javascript">
125-
128+
126129
const init = () => {
127130
const hybridChatConnector = connectorAdapter.initiateAdapter(
128131
${MeetQSettings.getInstance().reinvent2024OnboardingCount < MAX_ONBOARDING_PAGE_COUNT},
@@ -139,7 +142,7 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
139142
},
140143
141144
"${activeProfile?.profileName.orEmpty()}")
142-
amazonQChat.createChat(
145+
const qChat = amazonQChat.createChat(
143146
{
144147
postMessage: message => {
145148
$postMessageToJavaJsCode
@@ -155,6 +158,29 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
155158
hybridChatConnector,
156159
${CodeWhispererFeatureConfigService.getInstance().getFeatureConfigJsonString()}
157160
);
161+
162+
window.handleNativeDrop = function(filePath) {
163+
const parsedFilePath = JSON.parse(filePath);
164+
const contextArray = parsedFilePath.map(fullPath => {
165+
const fileName = fullPath.split(/[\\/]/).pop();
166+
return {
167+
command: fileName,
168+
label: 'image',
169+
route: [fullPath],
170+
description: fullPath
171+
};
172+
});
173+
qChat.addCustomContextToPrompt(qChat.getSelectedTabId(), contextArray);
174+
};
175+
176+
window.handleNativeNotify = function(errorMessages) {
177+
const messages = JSON.parse(errorMessages);
178+
messages.forEach(msg => {
179+
qChat.notify({
180+
content: msg
181+
})
182+
});
183+
};
158184
}
159185
</script>
160186
""".trimIndent()

‎plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSe
7777
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_MCP_SERVERS_REQUEST_METHOD
7878
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_RULES_REQUEST_METHOD
7979
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
80+
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG
81+
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
8082
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_SETTINGS
8183
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_WORKSPACE_SETTINGS_KEY
8284
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenSettingsNotification
@@ -489,6 +491,19 @@ class BrowserConnector(
489491
)
490492
}
491493
}
494+
495+
OPEN_FILE_DIALOG -> {
496+
handleChat(AmazonQChatServer.showOpenFileDialog, node)
497+
.whenComplete { response, _ ->
498+
browser.postChat(
499+
FlareUiMessage(
500+
command = OPEN_FILE_DIALOG_REQUEST_METHOD,
501+
params = response
502+
)
503+
)
504+
}
505+
}
506+
492507
LIST_RULES_REQUEST_METHOD -> {
493508
handleChat(AmazonQChatServer.listRules, node)
494509
.whenComplete { response, _ ->

‎plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_
4545
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LinkClickParams
4646
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ListConversationsParams
4747
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
48+
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
4849
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PROMPT_INPUT_OPTIONS_CHANGE
4950
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PromptInputOptionChangeParams
5051
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.RULE_CLICK_REQUEST_METHOD
@@ -248,4 +249,10 @@ object AmazonQChatServer : JsonRpcMethodProvider {
248249
TELEMETRY_EVENT,
249250
Any::class.java
250251
)
252+
253+
val showOpenFileDialog = JsonRpcRequest(
254+
OPEN_FILE_DIALOG_REQUEST_METHOD,
255+
Any::class.java,
256+
LSPAny::class.java
257+
)
251258
}

‎plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_S
2525
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult
2626
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIFF
2727
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams
28+
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD
2829
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD
30+
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams
2931
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams
3032
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult
3133
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata
@@ -45,6 +47,9 @@ interface AmazonQLanguageClient : LanguageClient {
4547
@JsonRequest(SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD)
4648
fun showSaveFileDialog(params: ShowSaveFileDialogParams): CompletableFuture<ShowSaveFileDialogResult>
4749

50+
@JsonRequest(SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD)
51+
fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture<LSPAny>
52+
4853
@JsonRequest(GET_SERIALIZED_CHAT_REQUEST_METHOD)
4954
fun getSerializedChat(params: LSPAny): CompletableFuture<GetSerializedChatResult>
5055

0 commit comments

Comments
 (0)