diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 69a682f2df4..6d3499b3741 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1,5 +1,8 @@ use std::collections::HashMap; +use std::path::PathBuf; +use codex_utils_image::decode_data_url; +use codex_utils_image::load_and_resize_bytes; use codex_utils_image::load_and_resize_to_fit; use mcp_types::CallToolResult; use mcp_types::ContentBlock; @@ -203,6 +206,24 @@ fn unsupported_image_error_placeholder(path: &std::path::Path, mime: &str) -> Co } } +fn inline_image_error_placeholder(error: impl std::fmt::Display) -> ContentItem { + ContentItem::InputText { + text: format!("Codex could not read the provided image data: {error}"), + } +} + +fn invalid_inline_image_error_placeholder(error: impl std::fmt::Display) -> ContentItem { + ContentItem::InputText { + text: format!("Provided image data is invalid: {error}"), + } +} + +fn unsupported_inline_image_error_placeholder(mime: &str) -> ContentItem { + ContentItem::InputText { + text: format!("Codex cannot attach image data: unsupported image format `{mime}`."), + } +} + impl From for ResponseItem { fn from(item: ResponseInputItem) -> Self { match item { @@ -302,7 +323,37 @@ impl From> for ResponseInputItem { .into_iter() .filter_map(|c| match c { UserInput::Text { text } => Some(ContentItem::InputText { text }), - UserInput::Image { image_url } => Some(ContentItem::InputImage { image_url }), + UserInput::Image { image_url } => { + if image_url.starts_with("data:") { + let inline = match decode_data_url(&image_url) { + Ok(inline) => inline, + Err(err) => return Some(inline_image_error_placeholder(err)), + }; + if let Some(mime) = inline.mime.as_deref() + && !mime.starts_with("image/") + && mime != "application/octet-stream" + { + return Some(unsupported_inline_image_error_placeholder(mime)); + } + match load_and_resize_bytes( + inline.bytes, + PathBuf::from(""), + ) { + Ok(image) => Some(ContentItem::InputImage { + image_url: image.into_data_url(), + }), + Err(err) => { + if err.is_invalid_image() { + Some(invalid_inline_image_error_placeholder(err)) + } else { + Some(inline_image_error_placeholder(err)) + } + } + } + } else { + Some(ContentItem::InputImage { image_url }) + } + } UserInput::LocalImage { path } => match load_and_resize_to_fit(&path) { Ok(image) => Some(ContentItem::InputImage { image_url: image.into_data_url(), @@ -556,6 +607,89 @@ mod tests { use pretty_assertions::assert_eq; use tempfile::tempdir; + #[test] + fn data_url_images_are_processed_locally() { + let data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAD0AAAA9CAYAAAAeYmHpAAAE6klEQVR4Aeyau44UVxCGx1fZsmRLlm3Zoe0XcGQ5cUiCCIgJeS9CHgAhMkISQnIuGQgJEkBcxLW+nqnZ6uqqc+nuWRC7q/P3qetf9e+MtOwyX25O4Nep6JPyop++0qev9HrfgZ+F6r2DuB/vHOrt/UIkqdDHYvujOW6fO7h/CNEI+a5jc+pBR8uy0jVFsziYu5HtfSUk+Io34q921hLNctFSX0gwww+S8wce8K1LfCU+cYW4888aov8NxqvQILUPPReLOrm6zyLxa4i+6VZuFbJo8d1MOHZm+7VUtB/aIvhPWc/3SWg49JcwFLlHxuXKjtyloo+YNhuW3VS+WPBuUEMvCFKjEDVgFBQHXrnazpqiSxNZCkQ1kYiozsbm9Oz7l4i2Il7vGccGNWAc3XosDrZe/9P3ZnMmzHNEQw4smf8RQ87XEAMsC7Az0Au+dgXerfH4+sHvEc0SYGic8WBBUGqFH2gN7yDrazy7m2pbRTeRmU3+MjZmr1h6LJgPbGy23SI6GlYT0brQ71IY8Us4PNQCm+zepSbaD2BY9xCaAsD9IIj/IzFmKMSdHHonwdZATbTnYREf6/VZGER98N9yCWIvXQwXDoDdhZJoT8jwLnJXDB9w4Sb3e6nK5ndzlkTLnP3JBu4LKkbrYrU69gCVceV0JvpyuW1xlsUVngzhwMetn/XamtTORF9IO5YnWNiyeF9zCAfqR3fUW+vZZKLtgP+ts8BmQRBREAdRDhH3o8QuRh/YucNFz2BEjxbRN6LGzphfKmvP6v6QhqIQyZ8XNJ0W0X83MR1PEcJBNO2KC2Z1TW/v244scp9FwRViZxIOBF0Lctk7ZVSavdLvRlV1hz/ysUi9sr8CIcB3nvWBwA93ykTz18eAYxQ6N/K2DkPA1lv3iXCwmDUT7YkjIby9siXueIJj9H+pzSqJ9oIuJWTUgSSt4WO7o/9GGg0viR4VinNRUDoIj34xoCd6pxD3aK3zfdbnx5v1J3ZNNEJsE0sBG7N27ReDrJc4sFxz7dI/ZAbOmmiKvHBitQXpAdR6+F7v+/ol/tOouUV01EeMZQF2BoQDn6dP4XNr+j9GZEtEK1/L8pFw7bd3a53tsTa7WD+054jOFmPg1XBKPQgnqFfmFcy32ZRvjmiIIQTYFvyDxQ8nH8WIwwGwlyDjDznnilYyFr6njrlZwsKkBpO59A7OwgdzPEWRm+G+oeb7IfyNuzjEEVLrOVxJsxvxwF8kmCM6I2QYmJunz4u4TrADpfl7mlbRTWQ7VmrBzh3+C9f6Grc3YoGN9dg/SXFthpRsT6vobfXRs2VBlgBHXVMLHjDNbIZv1sZ9+X3hB09cXdH1JKViyG0+W9bWZDa/r2f9zAFR71sTzGpMSWz2iI4YssWjWo3REy1MDGjdwe5e0dFSiAC1JakBvu4/CUS8Eh6dqHdU0Or0ioY3W5ClSqDXAy7/6SRfgw8vt4I+tbvvNtFT2kVDhY5+IGb1rCqYaXNF08vSALsXCPmt0kQNqJT1p5eI1mkIV/BxCY1z85lOzeFbPBQHURkkPTlwTYK9gTVE25l84IbFFN+YJDHjdpn0gq6mrHht0dkcjbM4UL9283O5p77GN+SPW/QwVB4IUYg7Or+Kp7naR6qktP98LNF2UxWo9yObPIT9KYg+hK4i56no4rfnM0qeyFf6AwAAAP//trwR3wAAAAZJREFUAwBZ0sR75itw5gAAAABJRU5ErkJggg==".to_string(); + + let item = ResponseInputItem::from(vec![UserInput::Image { + image_url: data_url, + }]); + + let ResponseInputItem::Message { content, .. } = item else { + panic!("expected message response input"); + }; + match content.as_slice() { + [ContentItem::InputImage { image_url }] => { + assert!(image_url.starts_with("data:image/")); + } + [ContentItem::InputText { text }] => { + panic!("expected input image, got placeholder: {text}"); + } + _ => panic!("expected single input content item"), + } + } + + #[test] + fn data_url_with_generic_mime_is_processed_locally() { + let payload = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMB/ee9bQAAAABJRU5ErkJggg=="; + let item = ResponseInputItem::from(vec![UserInput::Image { + image_url: format!("data:application/octet-stream;base64,{}", payload), + }]); + + let ResponseInputItem::Message { content, .. } = item else { + panic!("expected message response input"); + }; + match content.as_slice() { + [ContentItem::InputImage { image_url }] => { + assert!(image_url.starts_with("data:image/")); + } + [ContentItem::InputText { text }] => { + panic!("expected input image, got placeholder: {text}"); + } + _ => panic!("expected single input content item"), + } + } + + #[test] + fn data_url_with_missing_mime_is_processed_locally() { + let payload = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMB/ee9bQAAAABJRU5ErkJggg=="; + let item = ResponseInputItem::from(vec![UserInput::Image { + image_url: format!("data:;base64,{}", payload), + }]); + + let ResponseInputItem::Message { content, .. } = item else { + panic!("expected message response input"); + }; + match content.as_slice() { + [ContentItem::InputImage { image_url }] => { + assert!(image_url.starts_with("data:image/")); + } + [ContentItem::InputText { text }] => { + panic!("expected input image, got placeholder: {text}"); + } + _ => panic!("expected single input content item"), + } + } + + #[test] + fn data_url_with_non_image_mime_renders_placeholder() { + let item = ResponseInputItem::from(vec![UserInput::Image { + image_url: "data:text/plain;base64,SGVsbG8=".to_string(), + }]); + + let ResponseInputItem::Message { content, .. } = item else { + panic!("expected message response input"); + }; + let [ContentItem::InputText { text }] = content.as_slice() else { + panic!("expected single input text content item"); + }; + + assert_eq!( + text, + "Codex cannot attach image data: unsupported image format `text/plain`." + ); + } + #[test] fn serializes_success_as_plain_string() -> Result<()> { let item = ResponseInputItem::FunctionCallOutput { diff --git a/codex-rs/utils/image/src/error.rs b/codex-rs/utils/image/src/error.rs index 6bd055115dd..6c125f64d77 100644 --- a/codex-rs/utils/image/src/error.rs +++ b/codex-rs/utils/image/src/error.rs @@ -23,6 +23,8 @@ pub enum ImageProcessingError { #[source] source: image::ImageError, }, + #[error("invalid data URL: {message}")] + DataUrl { message: String }, } impl ImageProcessingError { diff --git a/codex-rs/utils/image/src/lib.rs b/codex-rs/utils/image/src/lib.rs index 0950bf176ea..c7b1b4ec9d1 100644 --- a/codex-rs/utils/image/src/lib.rs +++ b/codex-rs/utils/image/src/lib.rs @@ -1,5 +1,6 @@ use std::num::NonZeroUsize; use std::path::Path; +use std::path::PathBuf; use std::sync::LazyLock; use crate::error::ImageProcessingError; @@ -30,6 +31,12 @@ pub struct EncodedImage { pub height: u32, } +#[derive(Debug, Clone)] +pub struct InlineImageData { + pub mime: Option, + pub bytes: Vec, +} + impl EncodedImage { pub fn into_data_url(self) -> String { let encoded = BASE64_STANDARD.encode(&self.bytes); @@ -45,6 +52,84 @@ pub fn load_and_resize_to_fit(path: &Path) -> Result, + path_for_error: PathBuf, +) -> Result { + process_image_bytes(bytes, path_for_error) +} + +pub fn decode_data_url(image_url: &str) -> Result { + let Some(rest) = image_url.strip_prefix("data:") else { + return Err(ImageProcessingError::DataUrl { + message: "missing data URL prefix".to_string(), + }); + }; + let Some((header, data)) = rest.split_once(',') else { + return Err(ImageProcessingError::DataUrl { + message: "missing data URL header separator".to_string(), + }); + }; + + let mut mime = None; + let mut is_base64 = false; + + let mut parts = header.split(';'); + if let Some(first) = parts.next() + && !first.is_empty() + { + mime = Some(first.to_string()); + } + for part in parts { + if part == "base64" { + is_base64 = true; + } + } + + if !is_base64 { + return Err(ImageProcessingError::DataUrl { + message: "data URL is not base64 encoded".to_string(), + }); + } + + let decoded = + BASE64_STANDARD + .decode(data.trim()) + .map_err(|err| ImageProcessingError::DataUrl { + message: format!("invalid base64 image data: {err}"), + })?; + + Ok(InlineImageData { + mime, + bytes: decoded, + }) +} + +fn read_file_bytes(path: &Path, path_for_error: &Path) -> Result, ImageProcessingError> { + match tokio::runtime::Handle::try_current() { + // If we're inside a Tokio runtime, avoid block_on (it panics on worker threads). + // Use block_in_place and do a standard blocking read safely. + Ok(_) => tokio::task::block_in_place(|| std::fs::read(path)).map_err(|source| { + ImageProcessingError::Read { + path: path_for_error.to_path_buf(), + source, + } + }), + // Outside a runtime, just read synchronously. + Err(_) => std::fs::read(path).map_err(|source| ImageProcessingError::Read { + path: path_for_error.to_path_buf(), + source, + }), + } +} + +fn process_image_bytes( + file_bytes: Vec, + path_for_error: PathBuf, +) -> Result { let key = sha1_digest(&file_bytes); IMAGE_CACHE.get_or_try_insert_with(key, move || { @@ -56,7 +141,7 @@ pub fn load_and_resize_to_fit(path: &Path) -> Result Result Result, ImageProcessingError> { - match tokio::runtime::Handle::try_current() { - // If we're inside a Tokio runtime, avoid block_on (it panics on worker threads). - // Use block_in_place and do a standard blocking read safely. - Ok(_) => tokio::task::block_in_place(|| std::fs::read(path)).map_err(|source| { - ImageProcessingError::Read { - path: path_for_error.to_path_buf(), - source, - } - }), - // Outside a runtime, just read synchronously. - Err(_) => std::fs::read(path).map_err(|source| ImageProcessingError::Read { - path: path_for_error.to_path_buf(), - source, - }), - } -} - fn encode_image( image: &DynamicImage, preferred_format: ImageFormat,