diff --git a/crates/viewer/re_data_ui/src/blob.rs b/crates/viewer/re_data_ui/src/blob.rs index 9ded593c9130..32bdbdd97799 100644 --- a/crates/viewer/re_data_ui/src/blob.rs +++ b/crates/viewer/re_data_ui/src/blob.rs @@ -1,21 +1,20 @@ -use std::sync::Arc; - +use crate::image::ImageUi; +use crate::video::VideoUi; +use crate::{EntityDataUi, find_and_deserialize_archetype_mono_component}; +use re_chunk_store::UnitChunkShared; use re_log_types::EntityPath; use re_types::{ - ComponentDescriptor, RowId, + ComponentDescriptor, RowId, archetypes, components, components::{Blob, MediaType, VideoTimestamp}, }; +use re_types_core::Component as _; +use re_ui::list_item::ListItemContentButtonsExt as _; use re_ui::{ UiExt as _, icons, list_item::{self, PropertyContent}, }; use re_viewer_context::{StoredBlobCacheKey, UiLayout, ViewerContext}; - -use crate::{ - EntityDataUi, - image::image_preview_ui, - video::{show_decoded_frame_info, video_asset_result_ui}, -}; +use std::sync::Arc; impl EntityDataUi for Blob { fn entity_data_ui( @@ -39,20 +38,20 @@ impl EntityDataUi for Blob { // This can also help a user debug if they log the contents of `.png` file with a `image/jpeg` `MediaType`. let media_type = MediaType::guess_from_data(self); + let blob_ui = BlobUi::new( + ctx, + entity_path, + component_descriptor, + row_id, + self.0.clone(), + media_type.as_ref(), + None, + ); + if ui_layout.is_single_line() { ui.horizontal(|ui| { - blob_preview_and_save_ui( - ctx, - ui, - ui_layout, - query, - entity_path, - component_descriptor, - row_id, - self, - media_type.as_ref(), - None, - ); + ui.set_truncate_style(); + blob_ui.data_ui(ctx, ui, ui_layout, query, entity_path); ui.label(compact_size_string); @@ -85,167 +84,12 @@ impl EntityDataUi for Blob { ) .on_hover_text("Failed to detect media type (Mime) from magic header bytes"); } - - blob_preview_and_save_ui( - ctx, - ui, - ui_layout, - query, - entity_path, - component_descriptor, - row_id, - self, - media_type.as_ref(), - None, - ); + blob_ui.data_ui(ctx, ui, ui_layout, query, entity_path); }); } } } -#[allow(clippy::too_many_arguments)] -pub fn blob_preview_and_save_ui( - ctx: &re_viewer_context::ViewerContext<'_>, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - blob_component_descriptor: &ComponentDescriptor, - blob_row_id: Option, - blob: &re_types::datatypes::Blob, - media_type: Option<&MediaType>, - video_timestamp: Option, -) { - #[allow(unused_assignments)] // Not used when targeting web. - let mut image = None; - let mut video_result_for_frame_preview = None; - - if let Some(blob_row_id) = blob_row_id { - if !ui_layout.is_single_line() && ui_layout != UiLayout::Tooltip { - exif_ui( - ui, - StoredBlobCacheKey::new(blob_row_id, blob_component_descriptor), - blob, - ); - } - - // Try to treat it as an image: - image = ctx - .store_context - .caches - .entry(|c: &mut re_viewer_context::ImageDecodeCache| { - c.entry(blob_row_id, blob_component_descriptor, blob, media_type) - }) - .ok(); - - if let Some(image) = &image { - if !ui_layout.is_single_line() { - ui.list_item_flat_noninteractive( - PropertyContent::new("Image format").value_text(image.format.to_string()), - ); - } - - let colormap = None; // TODO(andreas): Rely on default here for now. - image_preview_ui(ctx, ui, ui_layout, query, entity_path, image, colormap); - } else { - // Try to treat it as a video. - let video_result = - ctx.store_context - .caches - .entry(|c: &mut re_viewer_context::VideoAssetCache| { - let debug_name = entity_path.to_string(); - c.entry( - debug_name, - blob_row_id, - blob_component_descriptor, - blob, - media_type, - ctx.app_options().video_decoder_settings(), - ) - }); - video_asset_result_ui(ui, ui_layout, &video_result); - video_result_for_frame_preview = Some(video_result); - } - } - - if !ui_layout.is_single_line() && ui_layout != UiLayout::Tooltip { - ui.horizontal(|ui| { - let text = if cfg!(target_arch = "wasm32") { - "Download blob…" - } else { - "Save blob…" - }; - if ui - .add(egui::Button::image_and_text( - icons::DOWNLOAD.as_image(), - text, - )) - .clicked() - { - let mut file_name = entity_path - .last() - .map_or("blob", |name| name.unescaped_str()) - .to_owned(); - - if let Some(file_extension) = media_type.as_ref().and_then(|mt| mt.file_extension()) - { - file_name.push('.'); - file_name.push_str(file_extension); - } - - ctx.command_sender().save_file_dialog( - re_capabilities::MainThreadToken::from_egui_ui(ui), - &file_name, - "Save blob".to_owned(), - blob.to_vec(), - ); - } - - if let Some(image) = image { - let image_stats = ctx - .store_context - .caches - .entry(|c: &mut re_viewer_context::ImageStatsCache| c.entry(&image)); - let data_range = re_viewer_context::gpu_bridge::image_data_range_heuristic( - &image_stats, - &image.format, - ); - crate::image::copy_image_button_ui(ui, &image, data_range); - } - }); - - // Show a mini video player for video blobs: - if let Some(video_result) = &video_result_for_frame_preview - && let Ok(video) = video_result.as_ref() - { - ui.separator(); - - let video_timestamp = video_timestamp.unwrap_or_else(|| { - // TODO(emilk): Some time controls would be nice, - // but the point here is not to have a nice viewer, - // but to show the user what they have selected - ui.ctx().request_repaint(); // TODO(emilk): schedule a repaint just in time for the next frame of video - let time = ui.input(|i| i.time); - - if let Some(duration) = video.data_descr().duration() { - VideoTimestamp::from_secs(time % duration.as_secs_f64()) - } else { - // Invalid video or unknown timescale. - VideoTimestamp::from_nanos(0) - } - }); - let video_time = re_viewer_context::video_timestamp_component_to_video_time( - ctx, - video_timestamp, - video.data_descr().timescale, - ); - let video_buffers = std::iter::once(blob.as_ref()).collect(); - - show_decoded_frame_info(ctx, ui, ui_layout, video, video_time, &video_buffers); - } - } -} - /// Show EXIF data about the given blob (image), if possible. fn exif_ui(ui: &mut egui::Ui, key: StoredBlobCacheKey, blob: &re_types::datatypes::Blob) { let exif_result = ui.ctx().memory_mut(|mem| { @@ -281,3 +125,167 @@ fn exif_ui(ui: &mut egui::Ui, key: StoredBlobCacheKey, blob: &re_types::datatype }); } } + +/// Utility for displaying additional UI for blobs. +pub struct BlobUi { + descr: ComponentDescriptor, + blob: re_types::datatypes::Blob, + + /// Additional image ui if any. + image: Option, + + /// Additional video ui if the blob is a video. + video: Option, + + /// The row id of the blob. + row_id: Option, + + /// The media type of the blob if known (used to inform image and video uis). + media_type: Option, +} + +impl BlobUi { + pub fn from_components( + ctx: &ViewerContext<'_>, + entity_path: &re_log_types::EntityPath, + blob_descr: &ComponentDescriptor, + blob_chunk: &UnitChunkShared, + components: &[(ComponentDescriptor, UnitChunkShared)], + ) -> Option { + if blob_descr.component_type != Some(components::Blob::name()) { + return None; + } + + let blob = blob_chunk + .component_mono::(blob_descr)? + .ok()?; + + // Media type comes typically alongside the blob in various different archetypes. + // Look for the one that matches the blob's archetype. + let media_type = find_and_deserialize_archetype_mono_component::( + components, + blob_descr.archetype, + ) + .or_else(|| components::MediaType::guess_from_data(&blob)); + + // Video timestamp is only relevant here if it comes from a VideoFrameReference archetype. + // It doesn't show up in the blob's archetype. + let video_timestamp_descr = archetypes::VideoFrameReference::descriptor_timestamp(); + let video_timestamp = components + .iter() + .find_map(|(descr, chunk)| { + (descr == &video_timestamp_descr).then(|| { + chunk + .component_mono::(&video_timestamp_descr)? + .ok() + }) + }) + .flatten(); + + Some(Self::new( + ctx, + entity_path, + blob_descr, + blob_chunk.row_id(), + blob.0, + media_type.as_ref(), + video_timestamp, + )) + } + + pub fn new( + ctx: &re_viewer_context::ViewerContext<'_>, + entity_path: &re_log_types::EntityPath, + blob_component_descriptor: &ComponentDescriptor, + blob_row_id: Option, + blob: re_types::datatypes::Blob, + media_type: Option<&MediaType>, + video_timestamp: Option, + ) -> Self { + let mut image = None; + let mut video = None; + + if let Some(blob_row_id) = blob_row_id { + image = ImageUi::from_blob( + ctx, + blob_row_id, + blob_component_descriptor, + &blob, + media_type, + ); + + video = VideoUi::from_blob( + ctx, + entity_path, + blob_row_id, + blob_component_descriptor, + &blob, + media_type, + video_timestamp, + ); + } + Self { + image, + video, + row_id: blob_row_id, + descr: blob_component_descriptor.clone(), + blob, + media_type: media_type.cloned(), + } + } + + pub fn inline_download_button<'a>( + &'a self, + ctx: &'a ViewerContext<'_>, + entity_path: &'a EntityPath, + mut property_content: list_item::PropertyContent<'a>, + ) -> list_item::PropertyContent<'a> { + if let Some(image) = &self.image { + property_content = image.inline_copy_button(ctx, property_content); + } + property_content.with_action_button(&icons::DOWNLOAD, "Save blob…", || { + let mut file_name = entity_path + .last() + .map_or("blob", |name| name.unescaped_str()) + .to_owned(); + + if let Some(file_extension) = + self.media_type.as_ref().and_then(|mt| mt.file_extension()) + { + file_name.push('.'); + file_name.push_str(file_extension); + } + + ctx.command_sender().save_file_dialog( + re_capabilities::MainThreadToken::i_promise_i_am_on_the_main_thread(), + &file_name, + "Save blob".to_owned(), + self.blob.to_vec(), + ); + }) + } + + pub fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + ui_layout: UiLayout, + query: &re_chunk_store::LatestAtQuery, + entity_path: &EntityPath, + ) { + if let Some(row_id) = self.row_id + && !ui_layout.is_single_line() + && ui_layout != UiLayout::Tooltip + { + exif_ui(ui, StoredBlobCacheKey::new(row_id, &self.descr), &self.blob); + } + + if let Some(image) = &self.image { + image.data_ui(ctx, ui, ui_layout, query, entity_path); + } + + if let Some(video) = &self.video { + video.data_ui(ctx, ui, ui_layout, query); + } + } +} diff --git a/crates/viewer/re_data_ui/src/extra_data_ui.rs b/crates/viewer/re_data_ui/src/extra_data_ui.rs new file mode 100644 index 000000000000..0fb9a3f9a23c --- /dev/null +++ b/crates/viewer/re_data_ui/src/extra_data_ui.rs @@ -0,0 +1,74 @@ +use crate::{blob, image, video}; +use re_chunk_store::UnitChunkShared; +use re_types_core::ComponentDescriptor; +use re_ui::{UiLayout, list_item}; +use re_viewer_context::ViewerContext; + +pub enum ExtraDataUi { + Video(video::VideoUi), + Image(image::ImageUi), + Blob(blob::BlobUi), +} + +impl ExtraDataUi { + pub fn from_components( + ctx: &ViewerContext<'_>, + query: &re_chunk_store::LatestAtQuery, + entity_path: &re_log_types::EntityPath, + descr: &ComponentDescriptor, + chunk: &UnitChunkShared, + entity_components: &[(ComponentDescriptor, UnitChunkShared)], + ) -> Option { + blob::BlobUi::from_components(ctx, entity_path, descr, chunk, entity_components) + .map(ExtraDataUi::Blob) + .or_else(|| { + image::ImageUi::from_components(ctx, descr, chunk, entity_components) + .map(ExtraDataUi::Image) + }) + .or_else(|| { + video::VideoUi::from_components(ctx, query, entity_path, descr) + .map(ExtraDataUi::Video) + }) + } + + pub fn add_inline_buttons<'a>( + &'a self, + ctx: &'a ViewerContext<'_>, + main_thread_token: re_capabilities::MainThreadToken, + entity_path: &'a re_log_types::EntityPath, + mut property_content: list_item::PropertyContent<'a>, + ) -> list_item::PropertyContent<'a> { + match self { + Self::Video(_) => { + // Video streams are not copyable or downloadable + property_content + } + Self::Image(image) => { + property_content = image.inline_copy_button(ctx, property_content); + image.inline_download_button(ctx, main_thread_token, entity_path, property_content) + } + Self::Blob(blob) => blob.inline_download_button(ctx, entity_path, property_content), + } + } + + pub fn data_ui( + self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + layout: UiLayout, + query: &re_chunk_store::LatestAtQuery, + entity_path: &re_log_types::EntityPath, + ) { + match self { + Self::Video(video) => { + video.data_ui(ctx, ui, layout, query); + } + Self::Image(image) => { + image.data_ui(ctx, ui, layout, query, entity_path); + } + Self::Blob(blob) => { + blob.data_ui(ctx, ui, layout, query, entity_path); + } + } + } +} diff --git a/crates/viewer/re_data_ui/src/image.rs b/crates/viewer/re_data_ui/src/image.rs index 720733987ebb..32305d1778be 100644 --- a/crates/viewer/re_data_ui/src/image.rs +++ b/crates/viewer/re_data_ui/src/image.rs @@ -1,31 +1,21 @@ -use egui::{Button, NumExt as _, Vec2}; - +use crate::find_and_deserialize_archetype_mono_component; +use egui::{NumExt as _, Rangef, Vec2}; +use re_capabilities::MainThreadToken; +use re_chunk_store::UnitChunkShared; use re_renderer::renderer::ColormappedTexture; -use re_ui::icons; +use re_types::components; +use re_types::components::MediaType; +use re_types::datatypes::{ChannelDatatype, ColorModel}; +use re_types::image::ImageKind; +use re_types_core::{Component as _, ComponentDescriptor, RowId}; +use re_ui::list_item::ListItemContentButtonsExt as _; +use re_ui::{UiExt as _, icons, list_item}; +use re_viewer_context::gpu_bridge::image_data_range_heuristic; use re_viewer_context::{ ColormapWithRange, ImageInfo, ImageStatsCache, UiLayout, ViewerContext, gpu_bridge::{self, image_to_gpu}, }; -/// Show a button letting the user copy the image -pub fn copy_image_button_ui(ui: &mut egui::Ui, image: &ImageInfo, data_range: egui::Rangef) { - if ui - .add(Button::image_and_text(icons::COPY.as_image(), "Copy image")) - .on_hover_text("Copy image to system clipboard") - .clicked() - { - if let Some(rgba) = image.to_rgba8_image(data_range.into()) { - let egui_image = egui::ColorImage::from_rgba_unmultiplied( - [rgba.width() as _, rgba.height() as _], - bytemuck::cast_slice(rgba.as_raw()), - ); - ui.ctx().copy_image(egui_image); - } else { - re_log::error!("Invalid image"); - } - } -} - /// Show the given image with an appropriate size. /// /// For segmentation images, the annotation context is looked up. @@ -181,3 +171,267 @@ fn largest_size_that_fits_in(aspect_ratio: f32, max_size: Vec2) -> Vec2 { egui::vec2(max_size.x, max_size.x / aspect_ratio) } } + +fn rgb8_histogram_ui(ui: &mut egui::Ui, rgb: &[u8]) -> egui::Response { + use egui::Color32; + use itertools::Itertools as _; + + re_tracing::profile_function!(); + + let mut histograms = [[0_u64; 256]; 3]; + { + // TODO(emilk): this is slow, so cache the results! + re_tracing::profile_scope!("build"); + for pixel in rgb.chunks_exact(3) { + for c in 0..3 { + histograms[c][pixel[c] as usize] += 1; + } + } + } + + use egui_plot::{Bar, BarChart, Legend, Plot}; + + let names = ["R", "G", "B"]; + let colors = [Color32::RED, Color32::GREEN, Color32::BLUE]; + + let charts = histograms + .into_iter() + .enumerate() + .map(|(component, histogram)| { + let fill = colors[component].linear_multiply(0.5); + + BarChart::new( + "bar_chart", + histogram + .into_iter() + .enumerate() + .map(|(i, count)| { + Bar::new(i as _, count as _) + .width(1.0) // no gaps between bars + .fill(fill) + .vertical() + .stroke(egui::Stroke::NONE) + }) + .collect(), + ) + .color(colors[component]) + .name(names[component]) + }) + .collect_vec(); + + re_tracing::profile_scope!("show"); + Plot::new("rgb_histogram") + .legend(Legend::default()) + .height(200.0) + .show_axes([false; 2]) + .show(ui, |plot_ui| { + for chart in charts { + plot_ui.bar_chart(chart); + } + }) + .response +} + +pub struct ImageUi { + image: ImageInfo, + data_range: Rangef, + colormap_with_range: Option, +} + +impl ImageUi { + pub fn new(ctx: &ViewerContext<'_>, image: ImageInfo) -> Self { + let image_stats = ctx + .store_context + .caches + .entry(|c: &mut ImageStatsCache| c.entry(&image)); + let data_range = image_data_range_heuristic(&image_stats, &image.format); + Self { + image, + data_range, + colormap_with_range: None, + } + } + + pub fn from_blob( + ctx: &ViewerContext<'_>, + blob_row_id: RowId, + blob_component_descriptor: &ComponentDescriptor, + blob: &re_types::datatypes::Blob, + media_type: Option<&MediaType>, + ) -> Option { + ctx.store_context + .caches + .entry(|c: &mut re_viewer_context::ImageDecodeCache| { + c.entry(blob_row_id, blob_component_descriptor, blob, media_type) + }) + .ok() + .map(|image| Self::new(ctx, image)) + } + + pub fn from_components( + ctx: &ViewerContext<'_>, + image_buffer_descr: &ComponentDescriptor, + image_buffer_chunk: &UnitChunkShared, + entity_components: &[(ComponentDescriptor, UnitChunkShared)], + ) -> Option { + if image_buffer_descr.component_type != Some(components::ImageBuffer::name()) { + return None; + } + + let blob_row_id = image_buffer_chunk.row_id()?; + let image_buffer = image_buffer_chunk + .component_mono::(image_buffer_descr)? + .ok()?; + + let (image_format_descr, image_format_chunk) = + entity_components.iter().find(|(descr, _chunk)| { + descr.component_type == Some(components::ImageFormat::name()) + && descr.archetype == image_buffer_descr.archetype + })?; + let image_format = image_format_chunk + .component_mono::(image_format_descr)? + .ok()?; + + let kind = ImageKind::from_archetype_name(image_format_descr.archetype); + let image = ImageInfo::from_stored_blob( + blob_row_id, + image_buffer_descr, + image_buffer.0, + image_format.0, + kind, + ); + let image_stats = ctx + .store_context + .caches + .entry(|c: &mut ImageStatsCache| c.entry(&image)); + + let colormap = find_and_deserialize_archetype_mono_component::( + entity_components, + image_buffer_descr.archetype, + ); + let value_range = find_and_deserialize_archetype_mono_component::( + entity_components, + image_buffer_descr.archetype, + ); + + let colormap_with_range = colormap.map(|colormap| ColormapWithRange { + colormap, + value_range: value_range + .map(|r| [r.start() as _, r.end() as _]) + .unwrap_or_else(|| { + if kind == ImageKind::Depth { + ColormapWithRange::default_range_for_depth_images(&image_stats) + } else { + let (min, max) = image_stats.finite_range; + [min as _, max as _] + } + }), + }); + + let data_range = value_range.map_or_else( + || image_data_range_heuristic(&image_stats, &image.format), + |r| Rangef::new(r.start() as _, r.end() as _), + ); + + Some(Self { + image, + data_range, + colormap_with_range, + }) + } + + pub fn inline_copy_button<'a>( + &'a self, + ctx: &'a ViewerContext<'_>, + property_content: list_item::PropertyContent<'a>, + ) -> list_item::PropertyContent<'a> { + property_content.with_action_button(&icons::COPY, "Copy image", move || { + if let Some(rgba) = self.image.to_rgba8_image(self.data_range.into()) { + let egui_image = egui::ColorImage::from_rgba_unmultiplied( + [rgba.width() as _, rgba.height() as _], + bytemuck::cast_slice(rgba.as_raw()), + ); + ctx.egui_ctx().copy_image(egui_image); + re_log::info!("Copied image to clipboard"); + } else { + re_log::error!("Invalid image"); + } + }) + } + + pub fn inline_download_button<'a>( + &'a self, + ctx: &'a ViewerContext<'_>, + main_thread_token: MainThreadToken, + entity_path: &'a re_log_types::EntityPath, + property_content: list_item::PropertyContent<'a>, + ) -> list_item::PropertyContent<'a> { + property_content.with_action_button(&icons::DOWNLOAD, "Save image", move || { + match self.image.to_png(self.data_range.into()) { + Ok(png_bytes) => { + let file_name = format!( + "{}.png", + entity_path + .last() + .map_or("image", |name| name.unescaped_str()) + .to_owned() + ); + ctx.command_sender().save_file_dialog( + main_thread_token, + &file_name, + "Save image".to_owned(), + png_bytes, + ); + } + Err(err) => { + re_log::error!("{err}"); + } + } + }) + } + + pub fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + ui_layout: UiLayout, + query: &re_chunk_store::LatestAtQuery, + entity_path: &re_log_types::EntityPath, + ) { + let Self { + image, + data_range: _, + colormap_with_range, + } = self; + + image_preview_ui( + ctx, + ui, + ui_layout, + query, + entity_path, + image, + colormap_with_range.as_ref(), + ); + + if ui_layout.is_single_line() || ui_layout == UiLayout::Tooltip { + return; + } + + ui.list_item_flat_noninteractive( + list_item::PropertyContent::new("Image format").value_text(image.format.to_string()), + ); + + // TODO(emilk): we should really support histograms for all types of images + if image.format.pixel_format.is_none() + && image.format.color_model() == ColorModel::RGB + && image.format.datatype() == ChannelDatatype::U8 + { + ui.section_collapsing_header("Histogram") + .default_open(false) + .show(ui, |ui| { + rgb8_histogram_ui(ui, &image.buffer); + }); + } + } +} diff --git a/crates/viewer/re_data_ui/src/instance_path.rs b/crates/viewer/re_data_ui/src/instance_path.rs index 29c9d308ac5c..20ea40a55d8b 100644 --- a/crates/viewer/re_data_ui/src/instance_path.rs +++ b/crates/viewer/re_data_ui/src/instance_path.rs @@ -1,29 +1,20 @@ -use egui::{Rangef, RichText}; +use egui::RichText; +use re_capabilities::MainThreadToken; use std::collections::BTreeMap; use re_chunk_store::UnitChunkShared; use re_entity_db::InstancePath; use re_log_types::ComponentPath; use re_types::{ - ArchetypeName, Component, ComponentDescriptor, archetypes, components, - datatypes::{ChannelDatatype, ColorModel}, - image::ImageKind, + ArchetypeName, Component as _, ComponentDescriptor, components, reflection::ComponentDescriptorExt as _, }; +use re_ui::list_item::ListItemContentButtonsExt as _; use re_ui::{UiExt as _, design_tokens_of_visuals, list_item}; -use re_viewer_context::{ - ColormapWithRange, HoverHighlight, ImageInfo, ImageStatsCache, Item, UiLayout, - VideoStreamCache, ViewerContext, gpu_bridge::image_data_range_heuristic, - video_stream_time_from_query, -}; - -use crate::{ - blob::blob_preview_and_save_ui, - image::image_preview_ui, - video::{show_decoded_frame_info, video_stream_result_ui}, -}; +use re_viewer_context::{HoverHighlight, Item, UiLayout, ViewerContext}; use super::DataUi; +use crate::extra_data_ui::ExtraDataUi; impl DataUi for InstancePath { fn data_ui( @@ -159,9 +150,18 @@ impl DataUi for InstancePath { .cloned() .collect::>(); - preview_if_image_ui(ctx, ui, ui_layout, query, entity_path, &components); - preview_if_blob_ui(ctx, ui, ui_layout, query, entity_path, &components); - preview_if_video_stream_ui(ctx, ui, ui_layout, query, entity_path, &components); + for (descr, shared) in &components { + if let Some(data) = ExtraDataUi::from_components( + ctx, + query, + entity_path, + descr, + shared, + &components, + ) { + data.data_ui(ctx, ui, ui_layout, query, entity_path); + } + } } } } @@ -216,7 +216,16 @@ fn component_list_ui( list_item = list_item.force_hovered(is_hovered); } - let content = re_ui::list_item::PropertyContent::new( + let data = ExtraDataUi::from_components( + ctx, + query, + entity_path, + component_descr, + unit, + components, + ); + + let mut content = re_ui::list_item::PropertyContent::new( component_descr.archetype_field_name(), ) .with_icon(icon) @@ -251,6 +260,17 @@ fn component_list_ui( } }); + if let Some(data) = &data { + content = data + .add_inline_buttons( + ctx, + MainThreadToken::from_egui_ui(ui), + entity_path, + content, + ) + .with_always_show_buttons(true); + } + let response = list_item.show_flat(ui, content).on_hover_ui(|ui| { if let Some(component_type) = component_descr.component_type { component_type.data_ui_recording(ctx, ui, UiLayout::Tooltip); @@ -294,351 +314,3 @@ pub fn archetype_label_list_item_ui(ui: &mut egui::Ui, archetype: &Option, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - components: &[(ComponentDescriptor, UnitChunkShared)], -) { - // There might be several image buffers! - for (image_buffer_descr, image_buffer_chunk) in components - .iter() - .filter(|(descr, _chunk)| descr.component_type == Some(components::ImageBuffer::name())) - { - preview_single_image( - ctx, - ui, - ui_layout, - query, - entity_path, - components, - image_buffer_descr, - image_buffer_chunk, - ); - } -} - -#[allow(clippy::too_many_arguments)] -fn preview_single_image( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - components: &[(ComponentDescriptor, UnitChunkShared)], - image_buffer_descr: &ComponentDescriptor, - image_buffer_chunk: &UnitChunkShared, -) -> Option<()> { - let blob_row_id = image_buffer_chunk.row_id()?; - let image_buffer = image_buffer_chunk - .component_mono::(image_buffer_descr)? - .ok()?; - - let (image_format_descr, image_format_chunk) = components.iter().find(|(descr, _chunk)| { - descr.component_type == Some(components::ImageFormat::name()) - && descr.archetype == image_buffer_descr.archetype - })?; - let image_format = image_format_chunk - .component_mono::(image_format_descr)? - .ok()?; - - let kind = ImageKind::from_archetype_name(image_format_descr.archetype); - let image = ImageInfo::from_stored_blob( - blob_row_id, - image_buffer_descr, - image_buffer.0, - image_format.0, - kind, - ); - let image_stats = ctx - .store_context - .caches - .entry(|c: &mut ImageStatsCache| c.entry(&image)); - - let colormap = find_and_deserialize_archetype_mono_component::( - components, - image_buffer_descr.archetype, - ); - let value_range = find_and_deserialize_archetype_mono_component::( - components, - image_buffer_descr.archetype, - ); - - let colormap_with_range = colormap.map(|colormap| ColormapWithRange { - colormap, - value_range: value_range - .map(|r| [r.start() as _, r.end() as _]) - .unwrap_or_else(|| { - if kind == ImageKind::Depth { - ColormapWithRange::default_range_for_depth_images(&image_stats) - } else { - let (min, max) = image_stats.finite_range; - [min as _, max as _] - } - }), - }); - - image_preview_ui( - ctx, - ui, - ui_layout, - query, - entity_path, - &image, - colormap_with_range.as_ref(), - ); - - if ui_layout.is_single_line() || ui_layout == UiLayout::Tooltip { - return Some(()); - } - - let data_range = value_range.map_or_else( - || image_data_range_heuristic(&image_stats, &image.format), - |r| Rangef::new(r.start() as _, r.end() as _), - ); - ui.horizontal(|ui| { - image_download_button_ui(ctx, ui, entity_path, &image, data_range); - - crate::image::copy_image_button_ui(ui, &image, data_range); - }); - - // TODO(emilk): we should really support histograms for all types of images - if image.format.pixel_format.is_none() - && image.format.color_model() == ColorModel::RGB - && image.format.datatype() == ChannelDatatype::U8 - { - ui.section_collapsing_header("Histogram") - .default_open(false) - .show(ui, |ui| { - rgb8_histogram_ui(ui, &image.buffer); - }); - } - - Some(()) -} - -fn image_download_button_ui( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - entity_path: &re_log_types::EntityPath, - image: &ImageInfo, - data_range: egui::Rangef, -) { - let text = if cfg!(target_arch = "wasm32") { - "Download image…" - } else { - "Save image…" - }; - if ui.button(text).clicked() { - match image.to_png(data_range.into()) { - Ok(png_bytes) => { - let file_name = format!( - "{}.png", - entity_path - .last() - .map_or("image", |name| name.unescaped_str()) - .to_owned() - ); - ctx.command_sender().save_file_dialog( - re_capabilities::MainThreadToken::from_egui_ui(ui), - &file_name, - "Save image".to_owned(), - png_bytes, - ); - } - Err(err) => { - re_log::error!("{err}"); - } - } - } -} - -fn rgb8_histogram_ui(ui: &mut egui::Ui, rgb: &[u8]) -> egui::Response { - use egui::Color32; - use itertools::Itertools as _; - - re_tracing::profile_function!(); - - let mut histograms = [[0_u64; 256]; 3]; - { - // TODO(emilk): this is slow, so cache the results! - re_tracing::profile_scope!("build"); - for pixel in rgb.chunks_exact(3) { - for c in 0..3 { - histograms[c][pixel[c] as usize] += 1; - } - } - } - - use egui_plot::{Bar, BarChart, Legend, Plot}; - - let names = ["R", "G", "B"]; - let colors = [Color32::RED, Color32::GREEN, Color32::BLUE]; - - let charts = histograms - .into_iter() - .enumerate() - .map(|(component, histogram)| { - let fill = colors[component].linear_multiply(0.5); - - BarChart::new( - "bar_chart", - histogram - .into_iter() - .enumerate() - .map(|(i, count)| { - Bar::new(i as _, count as _) - .width(1.0) // no gaps between bars - .fill(fill) - .vertical() - .stroke(egui::Stroke::NONE) - }) - .collect(), - ) - .color(colors[component]) - .name(names[component]) - }) - .collect_vec(); - - re_tracing::profile_scope!("show"); - Plot::new("rgb_histogram") - .legend(Legend::default()) - .height(200.0) - .show_axes([false; 2]) - .show(ui, |plot_ui| { - for chart in charts { - plot_ui.bar_chart(chart); - } - }) - .response -} - -/// If this entity has a blob, preview it and show a download button. -fn preview_if_blob_ui( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - components: &[(ComponentDescriptor, UnitChunkShared)], -) { - // There might be several blobs, all with different meanings. - for (blob_descr, blob_chunk) in components - .iter() - .filter(|(descr, _chunk)| descr.component_type == Some(components::Blob::name())) - { - preview_single_blob( - ctx, - ui, - ui_layout, - query, - entity_path, - components, - blob_descr, - blob_chunk, - ); - } -} - -#[allow(clippy::too_many_arguments)] -fn preview_single_blob( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - components: &[(ComponentDescriptor, UnitChunkShared)], - blob_descr: &ComponentDescriptor, - blob_chunk: &UnitChunkShared, -) -> Option<()> { - let blob = blob_chunk - .component_mono::(blob_descr)? - .ok()?; - - // Media type comes typically alongside the blob in various different archetypes. - // Look for the one that matches the blob's archetype. - let media_type = find_and_deserialize_archetype_mono_component::( - components, - blob_descr.archetype, - ) - .or_else(|| components::MediaType::guess_from_data(&blob)); - - // Video timestamp is only relevant here if it comes from a VideoFrameReference archetype. - // It doesn't show up in the blob's archetype. - let video_timestamp_descr = archetypes::VideoFrameReference::descriptor_timestamp(); - let video_timestamp = components - .iter() - .find_map(|(descr, chunk)| { - (descr == &video_timestamp_descr).then(|| { - chunk - .component_mono::(&video_timestamp_descr)? - .ok() - }) - }) - .flatten(); - - blob_preview_and_save_ui( - ctx, - ui, - ui_layout, - query, - entity_path, - blob_descr, - blob_chunk.row_id(), - &blob, - media_type.as_ref(), - video_timestamp, - ); - - Some(()) -} - -fn preview_if_video_stream_ui( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - ui_layout: UiLayout, - query: &re_chunk_store::LatestAtQuery, - entity_path: &re_log_types::EntityPath, - components: &[(ComponentDescriptor, UnitChunkShared)], -) { - if components - .iter() - .all(|(descr, _chunk)| descr != &archetypes::VideoStream::descriptor_sample()) - { - return; - } - - let video_stream_result = ctx.store_context.caches.entry(|c: &mut VideoStreamCache| { - c.entry( - ctx.recording(), - entity_path, - query.timeline(), - ctx.app_options().video_decoder_settings(), - ) - }); - video_stream_result_ui(ui, ui_layout, &video_stream_result); - if let Ok(video) = video_stream_result { - let video = video.read(); - let time = video_stream_time_from_query(query); - let buffers = video.sample_buffers(); - show_decoded_frame_info(ctx, ui, ui_layout, &video.video_renderer, time, &buffers); - } -} - -/// Finds and deserializes the given component type if its descriptor matches the given archetype name. -fn find_and_deserialize_archetype_mono_component( - components: &[(ComponentDescriptor, UnitChunkShared)], - archetype_name: Option, -) -> Option { - components.iter().find_map(|(descr, chunk)| { - (descr.component_type == Some(C::name()) && descr.archetype == archetype_name) - .then(|| chunk.component_mono::(descr)?.ok()) - .flatten() - }) -} diff --git a/crates/viewer/re_data_ui/src/lib.rs b/crates/viewer/re_data_ui/src/lib.rs index c83c4bf7a6aa..3e3b091c7669 100644 --- a/crates/viewer/re_data_ui/src/lib.rs +++ b/crates/viewer/re_data_ui/src/lib.rs @@ -25,6 +25,7 @@ mod store_id; mod tensor; mod video; +mod extra_data_ui; pub mod item_ui; pub use crate::tensor::tensor_summary_ui_grid_contents; @@ -32,8 +33,9 @@ pub use component::ComponentPathLatestAtResults; pub use component_ui_registry::{add_to_registry, register_component_uis}; pub use image::image_preview_ui; pub use instance_path::archetype_label_list_item_ui; -use re_types_core::ArchetypeName; +use re_chunk_store::UnitChunkShared; use re_types_core::reflection::Reflection; +use re_types_core::{ArchetypeName, Component}; pub type ArchetypeComponentMap = std::collections::BTreeMap, Vec>; @@ -146,3 +148,15 @@ pub fn annotations( annotation_map.load(ctx, query, std::iter::once(entity_path)); annotation_map.find(entity_path) } + +/// Finds and deserializes the given component type if its descriptor matches the given archetype name. +fn find_and_deserialize_archetype_mono_component( + components: &[(ComponentDescriptor, UnitChunkShared)], + archetype_name: Option, +) -> Option { + components.iter().find_map(|(descr, chunk)| { + (descr.component_type == Some(C::name()) && descr.archetype == archetype_name) + .then(|| chunk.component_mono::(descr)?.ok()) + .flatten() + }) +} diff --git a/crates/viewer/re_data_ui/src/video.rs b/crates/viewer/re_data_ui/src/video.rs index e1af37c577c4..0e76504b4e85 100644 --- a/crates/viewer/re_data_ui/src/video.rs +++ b/crates/viewer/re_data_ui/src/video.rs @@ -4,12 +4,19 @@ use re_renderer::{ external::re_video::VideoLoadError, resource_managers::SourceImageDataFormat, video::VideoFrameTexture, }; +use re_types::components::{MediaType, VideoTimestamp}; +use re_types::{Archetype as _, archetypes}; +use re_types_core::{ComponentDescriptor, RowId}; use re_ui::{ UiExt as _, list_item::{self, PropertyContent}, }; use re_video::{FrameInfo, StableIndexDeque, VideoDataDescription}; -use re_viewer_context::{SharablePlayableVideoStream, UiLayout, VideoStreamProcessingError}; +use re_viewer_context::{ + SharablePlayableVideoStream, UiLayout, VideoStreamCache, VideoStreamProcessingError, + ViewerContext, video_stream_time_from_query, +}; +use std::sync::Arc; pub fn video_asset_result_ui( ui: &mut egui::Ui, @@ -35,16 +42,6 @@ pub fn video_asset_result_ui( }); } } - Err(VideoLoadError::MimeTypeIsNotAVideo { .. }) => { - // Don't show an error if this wasn't a video in the first place. - // Unfortunately we can't easily detect here if the Blob was _supposed_ to be a video, for that we'd need tagged components! - // (User may have confidently logged a non-video format as Video, we should tell them that!) - } - Err(VideoLoadError::UnrecognizedMimeType) => { - // If we couldn't detect the media type, - // we can't show an error for unrecognized formats since maybe this wasn't a video to begin with. - // See also `MediaTypeIsNotAVideo` case above. - } Err(err) => { let error_message = format!("Failed to play: {err}"); if ui_layout.is_single_line() { @@ -572,3 +569,135 @@ fn source_image_data_format_ui(ui: &mut egui::Ui, format: &SourceImageDataFormat } } } + +pub enum VideoUi { + Stream(Result), + Asset( + Arc>, + Option, + re_types::datatypes::Blob, + ), +} + +impl VideoUi { + pub fn from_blob( + ctx: &ViewerContext<'_>, + entity_path: &re_log_types::EntityPath, + blob_row_id: RowId, + blob_component_descriptor: &ComponentDescriptor, + blob: &re_types::datatypes::Blob, + media_type: Option<&MediaType>, + video_timestamp: Option, + ) -> Option { + let result = + ctx.store_context + .caches + .entry(|c: &mut re_viewer_context::VideoAssetCache| { + let debug_name = entity_path.to_string(); + c.entry( + debug_name, + blob_row_id, + blob_component_descriptor, + blob, + media_type, + ctx.app_options().video_decoder_settings(), + ) + }); + + let certain_this_is_a_video = + blob_component_descriptor.archetype == Some(archetypes::AssetVideo::name()); + + if let Err(err) = &*result + && !certain_this_is_a_video + && matches!( + err, + VideoLoadError::MimeTypeIsNotAVideo { .. } | VideoLoadError::UnrecognizedMimeType + ) + { + // Don't show an error if we weren't certain that this was a video and it turned out not to be one. + return None; + } + + Some(Self::Asset(result, video_timestamp, blob.clone())) + } + + pub fn from_components( + ctx: &ViewerContext<'_>, + query: &re_chunk_store::LatestAtQuery, + entity_path: &re_log_types::EntityPath, + descr: &ComponentDescriptor, + ) -> Option { + if descr != &archetypes::VideoStream::descriptor_sample() { + return None; + } + + let video_stream_result = ctx.store_context.caches.entry(|c: &mut VideoStreamCache| { + c.entry( + ctx.recording(), + entity_path, + query.timeline(), + ctx.app_options().video_decoder_settings(), + ) + }); + + Some(Self::Stream(video_stream_result)) + } + + pub fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + ui_layout: UiLayout, + query: &re_chunk_store::LatestAtQuery, + ) { + match self { + Self::Stream(video_stream_result) => { + video_stream_result_ui(ui, ui_layout, video_stream_result); + if let Ok(video) = video_stream_result { + let video = video.read(); + let time = video_stream_time_from_query(query); + let buffers = video.sample_buffers(); + show_decoded_frame_info( + ctx, + ui, + ui_layout, + &video.video_renderer, + time, + &buffers, + ); + } + } + Self::Asset(video_result, timestamp, blob) => { + video_asset_result_ui(ui, ui_layout, video_result); + // Show a mini video player for video blobs: + if !ui_layout.is_single_line() + && ui_layout != UiLayout::Tooltip + && let Ok(video) = video_result.as_ref() + { + let video_timestamp = timestamp.unwrap_or_else(|| { + // TODO(emilk): Some time controls would be nice, + // but the point here is not to have a nice viewer, + // but to show the user what they have selected + ui.ctx().request_repaint(); // TODO(emilk): schedule a repaint just in time for the next frame of video + let time = ui.input(|i| i.time); + + if let Some(duration) = video.data_descr().duration() { + VideoTimestamp::from_secs(time % duration.as_secs_f64()) + } else { + // Invalid video or unknown timescale. + VideoTimestamp::from_nanos(0) + } + }); + let video_time = re_viewer_context::video_timestamp_component_to_video_time( + ctx, + video_timestamp, + video.data_descr().timescale, + ); + let video_buffers = std::iter::once(blob.as_ref()).collect(); + + show_decoded_frame_info(ctx, ui, ui_layout, video, video_time, &video_buffers); + } + } + } + } +} diff --git a/crates/viewer/re_ui/src/list_item/item_buttons.rs b/crates/viewer/re_ui/src/list_item/item_buttons.rs index 834907f34f98..c0765ec87e66 100644 --- a/crates/viewer/re_ui/src/list_item/item_buttons.rs +++ b/crates/viewer/re_ui/src/list_item/item_buttons.rs @@ -153,6 +153,7 @@ where /// /// The `alt_text` will be used for accessibility (e.g. read by screen readers), /// and is also how we can query the button in tests. + /// The `alt_text` will also be used for the tooltip. /// /// See [`Self::with_button`] for more information. #[inline] @@ -169,6 +170,7 @@ where /// /// The `alt_text` will be used for accessibility (e.g. read by screen readers), /// and is also how we can query the button in tests. + /// The `alt_text` will also be used for the tooltip. /// /// See [`Self::with_button`] for more information. #[inline] @@ -179,13 +181,19 @@ where enabled: bool, on_click: impl FnOnce() + 'a, ) -> Self { - self.with_button(super::ItemActionButton::new(icon, alt_text, on_click).enabled(enabled)) + let hover_text = alt_text.into(); + self.with_button( + super::ItemActionButton::new(icon, &hover_text, on_click) + .enabled(enabled) + .hover_text(hover_text), + ) } /// Helper to add a [`super::ItemMenuButton`] to the right of the item. /// /// The `alt_text` will be used for accessibility (e.g. read by screen readers), /// and is also how we can query the button in tests. + /// The `alt_text` will also be used for the tooltip. /// /// See [`Self::with_button`] for more information. #[inline] @@ -195,7 +203,10 @@ where alt_text: impl Into, add_contents: impl FnOnce(&mut egui::Ui) + 'a, ) -> Self { - self.with_button(super::ItemMenuButton::new(icon, alt_text, add_contents)) + let hover_text = alt_text.into(); + self.with_button( + super::ItemMenuButton::new(icon, &hover_text, add_contents).hover_text(hover_text), + ) } /// Set the help text tooltip to be shown in the header. diff --git a/crates/viewer/re_viewer_context/src/cache/video_stream_cache.rs b/crates/viewer/re_viewer_context/src/cache/video_stream_cache.rs index 4eb42a621468..48f096c8ca61 100644 --- a/crates/viewer/re_viewer_context/src/cache/video_stream_cache.rs +++ b/crates/viewer/re_viewer_context/src/cache/video_stream_cache.rs @@ -146,8 +146,6 @@ pub type SharablePlayableVideoStream = Arc>; impl VideoStreamCache { /// Looks up a video stream + players. /// - /// Returns `None` if there was no video data for this entity on the given timeline. - /// /// The first time a video stream that is looked up that isn't in the cache, /// it creates all the necessary metadata. /// For any stream in the cache, metadata will be kept automatically up to date for incoming