From f54635efd00903b9f0d35db92b481658e0804887 Mon Sep 17 00:00:00 2001 From: Garrick Aden-Buie Date: Wed, 27 Aug 2025 11:43:00 -0400 Subject: [PATCH 1/3] chore: If there are _any_ tool results, move entire turn to "assistant" role --- pkg-r/R/contents_shinychat.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg-r/R/contents_shinychat.R b/pkg-r/R/contents_shinychat.R index c45e436f..38130556 100644 --- a/pkg-r/R/contents_shinychat.R +++ b/pkg-r/R/contents_shinychat.R @@ -377,8 +377,8 @@ S7::method(contents_shinychat, S7::new_S3_class(c("Chat", "R6"))) <- function( x }) - # Turns containing only tool results are converted into assistant turns - if (every(turn@contents, S7::S7_inherits, ellmer::ContentToolResult)) { + # Turns containing tool results are converted into assistant turns + if (some(turn@contents, S7::S7_inherits, ellmer::ContentToolResult)) { turn@role <- "assistant" return(turn) } From fa7dfa25460385a0ba6474e8f725dab36da8d025 Mon Sep 17 00:00:00 2001 From: Garrick Aden-Buie Date: Wed, 27 Aug 2025 11:43:36 -0400 Subject: [PATCH 2/3] fix: Don't throw if the tool result can't be turned into a string It's not our job, ellmer will do the throwing, this path shouldn't fail --- pkg-r/R/contents_shinychat.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg-r/R/contents_shinychat.R b/pkg-r/R/contents_shinychat.R index 38130556..59bbeb7c 100644 --- a/pkg-r/R/contents_shinychat.R +++ b/pkg-r/R/contents_shinychat.R @@ -347,7 +347,7 @@ tool_string <- function(x) { } else if (is.character(x@value)) { paste(x@value, collapse = "\n") } else { - jsonlite::toJSON(x@value, auto_unbox = TRUE, pretty = 2) + jsonlite::toJSON(x@value, auto_unbox = TRUE, pretty = 2, force = TRUE) } } From e47a11f23f8930767344ac0b2775585520cb5a9a Mon Sep 17 00:00:00 2001 From: Garrick Aden-Buie Date: Wed, 27 Aug 2025 16:39:54 -0400 Subject: [PATCH 3/3] feat: Support image/pdf content in tool results --- pkg-r/R/contents_shinychat.R | 56 ++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/pkg-r/R/contents_shinychat.R b/pkg-r/R/contents_shinychat.R index 59bbeb7c..a50cb6a5 100644 --- a/pkg-r/R/contents_shinychat.R +++ b/pkg-r/R/contents_shinychat.R @@ -136,6 +136,20 @@ S7::method(contents_shinychat, ellmer::ContentText) <- function(content) { content@text } +S7::method(contents_shinychat, ellmer::ContentPDF) <- function(content) { + htmltools::div( + class = "shinychat-pdf badge fs-6 text-bg-secondary", + htmltools::span( + class = "shinychat-pdf__icon me-1", + htmltools::HTML(ICON_FILE_PDF_FILL) + ), + htmltools::span( + class = "shinychat-pdf__filename font-monospace", + content@filename + ) + ) +} + new_tool_card <- function(type, request_id, tool_name, ...) { type <- arg_match(type, c("request", "result")) @@ -316,18 +330,22 @@ tool_result_display <- function(content, display = NULL) { use_basic_display <- opt_shinychat_tool_display() == "basic" if (tool_errored(content) || use_basic_display || !has_display) { - return(list(value = tool_string(content), value_type = "code")) + return(tool_default_display(content)) } if (is.list(display)) { has_type <- intersect(c("html", "markdown", "text"), names(display)) if (length(has_type) > 0) { value_type <- has_type[1] - return(list(value = display[[value_type]], value_type = value_type)) + return(as_tool_display(display[[value_type]], value_type)) } } - list(value = tool_string(content), value_type = "code") + tool_default_display(content) +} + +as_tool_display <- function(value, type = "code") { + list(value = value, value_type = type) } # Copied from @@ -336,21 +354,33 @@ tool_errored <- function(x) !is.null(x@error) tool_error_string <- function(x) { if (inherits(x@error, "condition")) conditionMessage(x@error) else x@error } -tool_string <- function(x) { +tool_default_display <- function(x) { if (tool_errored(x)) { # Changed from original: if tool errored, just return the error message - strip_ansi(tool_error_string(x)) - } else if (inherits(x@value, "AsIs")) { - x@value - } else if (inherits(x@value, "json")) { - x@value - } else if (is.character(x@value)) { - paste(x@value, collapse = "\n") + as_tool_display(strip_ansi(tool_error_string(x))) + } + value <- x@value + if (inherits(value, "AsIs")) { + as_tool_display(value) + } else if (inherits(value, "json")) { + as_tool_display(value) + } else if (is.character(value)) { + as_tool_display(paste(value, collapse = "\n")) + } else if (is_content_extra(value)) { + as_tool_display(htmltools::div(contents_shinychat(value)), "html") + } else if (is.list(value) && every(value, is_content_extra)) { + as_tool_display(htmltools::div(map(value, contents_shinychat)), "html") } else { - jsonlite::toJSON(x@value, auto_unbox = TRUE, pretty = 2, force = TRUE) + as_tool_display( + jsonlite::toJSON(value, auto_unbox = TRUE, pretty = 2, force = TRUE) + ) } } +is_content_extra <- function(x) { + S7::S7_inherits(x, ellmer::ContentImage) || + S7::S7_inherits(x, ellmer::ContentPDF) +} S7::method(contents_shinychat, ellmer::Turn) <- function(content) { # Process all contents in the turn, filtering out empty results @@ -427,3 +457,5 @@ S7::method(contents_shinychat, S7::new_S3_class(c("Chat", "R6"))) <- function( compact(messages) } + +ICON_FILE_PDF_FILL <- "\n"