Diegetic UI example (#19199)

https://github.com/user-attachments/assets/e7b428b9-4705-4d0a-8322-c7a80994b1a4

This demonstrates general-purpose world-space picking. This is not
limited to bevy_ui, it works with any render target and any picking
backend, e.g. this could be a viewport/portal in 3d space, it could be a
custom UI system, bevy_egui, sprites, etc, etc.

Because this uses UV coordinates, it also works with anything that can
be mapped to a surface, so you can display UIs on curved surfaces, etc.


https://github.com/user-attachments/assets/2926a5a1-16c9-4179-85fa-83a69fd55a96

### Discussion

- This illustrates some of the improvements I think we need to make to
picking - the picking ID should be removed in favor of a pointer kind -
you might have multiple mouse pointers on multiple surfaces, with
different unique IDs. It's probably worth revisiting if we can use
Entity as the ID instead of a custom one. When I first wrote this, I ran
into some unexpected issues that caused with pointers being despawned
and getting a new ID. We will just need to be careful of edge cases
there.

---------

Co-authored-by: IceSentry <[email protected]>
Co-authored-by: Alice Cecile <[email protected]>

Author: 3 people

examples/ui/render_ui_to_texture.rs

@@ -2,19 +2,29 @@
 
 use std::f32::consts::PI;
 
+use bevy::picking::PickingSystems;
 use bevy::{
-    asset::RenderAssetUsages,
+    asset::{uuid::Uuid, RenderAssetUsages},
     camera::RenderTarget,
-    color::palettes::css::GOLD,
+    color::palettes::css::{BLUE, GRAY, RED},
+    input::ButtonState,
+    picking::{
+        backend::ray::RayMap,
+        pointer::{Location, PointerAction, PointerId, PointerInput},
+    },
     prelude::*,
     render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureUsages},
+    window::{PrimaryWindow, WindowEvent},
 };
 
+const CUBE_POINTER_ID: PointerId = PointerId::Custom(Uuid::from_u128(90870987));
+
 fn main() {
     App::new()
         .add_plugins(DefaultPlugins)
         .add_systems(Startup, setup)
         .add_systems(Update, rotator_system)
+        .add_systems(First, drive_diegetic_pointer.in_set(PickingSystems::Input))
         .run();
 }
 
@@ -72,52 +82,166 @@ fn setup(
                 align_items: AlignItems::Center,
                 ..default()
             },
-            BackgroundColor(GOLD.into()),
+            BackgroundColor(GRAY.into()),
             UiTargetCamera(texture_camera),
         ))
         .with_children(|parent| {
-            parent.spawn((
-                Text::new("This is a cube"),
-                TextFont {
-                    font_size: 40.0,
-                    ..default()
-                },
-                TextColor::BLACK,
-            ));
+            parent
+                .spawn((
+                    Node {
+                        position_type: PositionType::Absolute,
+                        width: Val::Auto,
+                        height: Val::Auto,
+                        align_items: AlignItems::Center,
+                        padding: UiRect::all(Val::Px(20.)),
+                        ..default()
+                    },
+                    BorderRadius::all(Val::Px(10.)),
+                    BackgroundColor(BLUE.into()),
+                ))
+                .observe(
+                    |pointer: On<Pointer<Drag>>, mut nodes: Query<(&mut Node, &ComputedNode)>| {
+                        let (mut node, computed) = nodes.get_mut(pointer.target()).unwrap();
+                        node.left =
+                            Val::Px(pointer.pointer_location.position.x - computed.size.x / 2.0);
+                        node.top = Val::Px(pointer.pointer_location.position.y - 50.0);
+                    },
+                )
+                .observe(
+                    |pointer: On<Pointer<Over>>, mut colors: Query<&mut BackgroundColor>| {
+                        colors.get_mut(pointer.target()).unwrap().0 = RED.into();
+                    },
+                )
+                .observe(
+                    |pointer: On<Pointer<Out>>, mut colors: Query<&mut BackgroundColor>| {
+                        colors.get_mut(pointer.target()).unwrap().0 = BLUE.into();
+                    },
+                )
+                .with_children(|parent| {
+                    parent.spawn((
+                        Text::new("Drag Me!"),
+                        TextFont {
+                            font_size: 40.0,
+                            ..default()
+                        },
+                        TextColor::WHITE,
+                    ));
+                });
         });
 
-    let cube_size = 4.0;
-    let cube_handle = meshes.add(Cuboid::new(cube_size, cube_size, cube_size));
+    let mesh_handle = meshes.add(Cuboid::default());
 
     // This material has the texture that has been rendered.
     let material_handle = materials.add(StandardMaterial {
         base_color_texture: Some(image_handle),
         reflectance: 0.02,
         unlit: false,
-
         ..default()
     });
 
     // Cube with material containing the rendered UI texture.
     commands.spawn((
-        Mesh3d(cube_handle),
+        Mesh3d(mesh_handle),
         MeshMaterial3d(material_handle),
-        Transform::from_xyz(0.0, 0.0, 1.5).with_rotation(Quat::from_rotation_x(-PI / 5.0)),
+        Transform::from_xyz(0.0, 0.0, 1.5).with_rotation(Quat::from_rotation_x(PI)),
         Cube,
     ));
 
     // The main pass camera.
     commands.spawn((
         Camera3d::default(),
-        Transform::from_xyz(0.0, 0.0, 15.0).looking_at(Vec3::ZERO, Vec3::Y),
+        Transform::from_xyz(0.0, 0.0, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
     ));
+
+    commands.spawn(CUBE_POINTER_ID);
 }
 
-const ROTATION_SPEED: f32 = 0.5;
+const ROTATION_SPEED: f32 = 0.1;
 
 fn rotator_system(time: Res<Time>, mut query: Query<&mut Transform, With<Cube>>) {
     for mut transform in &mut query {
         transform.rotate_x(1.0 * time.delta_secs() * ROTATION_SPEED);
         transform.rotate_y(0.7 * time.delta_secs() * ROTATION_SPEED);
     }
 }
+
+/// Because bevy has no way to know how to map a mouse input to the UI texture, we need to write a
+/// system that tells it there is a pointer on the UI texture. We cast a ray into the scene and find
+/// the UV (2D texture) coordinates of the raycast hit. This UV coordinate is effectively the same
+/// as a pointer coordinate on a 2D UI rect.
+fn drive_diegetic_pointer(
+    mut cursor_last: Local<Vec2>,
+    mut raycast: MeshRayCast,
+    rays: Res<RayMap>,
+    cubes: Query<&Mesh3d, With<Cube>>,
+    ui_camera: Query<&Camera, With<Camera2d>>,
+    primary_window: Query<Entity, With<PrimaryWindow>>,
+    windows: Query<(Entity, &Window)>,
+    images: Res<Assets<Image>>,
+    manual_texture_views: Res<ManualTextureViews>,
+    mut window_events: EventReader<WindowEvent>,
+    mut pointer_input: EventWriter<PointerInput>,
+) -> Result {
+    // Get the size of the texture, so we can convert from dimensionless UV coordinates that span
+    // from 0 to 1, to pixel coordinates.
+    let target = ui_camera
+        .single()?
+        .target
+        .normalize(primary_window.single().ok())
+        .unwrap();
+    let target_info = target
+        .get_render_target_info(windows, &images, &manual_texture_views)
+        .unwrap();
+    let size = target_info.physical_size.as_vec2();
+
+    // Find raycast hits and update the virtual pointer.
+    let raycast_settings = MeshRayCastSettings {
+        visibility: RayCastVisibility::VisibleInView,
+        filter: &|entity| cubes.contains(entity),
+        early_exit_test: &|_| false,
+    };
+    for (_id, ray) in rays.iter() {
+        for (_cube, hit) in raycast.cast_ray(*ray, &raycast_settings) {
+            let position = size * hit.uv.unwrap();
+            if position != *cursor_last {
+                pointer_input.write(PointerInput::new(
+                    CUBE_POINTER_ID,
+                    Location {
+                        target: target.clone(),
+                        position,
+                    },
+                    PointerAction::Move {
+                        delta: position - *cursor_last,
+                    },
+                ));
+                *cursor_last = position;
+            }
+        }
+    }
+
+    // Pipe pointer button presses to the virtual pointer on the UI texture.
+    for window_event in window_events.read() {
+        if let WindowEvent::MouseButtonInput(input) = window_event {
+            let button = match input.button {
+                MouseButton::Left => PointerButton::Primary,
+                MouseButton::Right => PointerButton::Secondary,
+                MouseButton::Middle => PointerButton::Middle,
+                _ => continue,
+            };
+            let action = match input.state {
+                ButtonState::Pressed => PointerAction::Press(button),
+                ButtonState::Released => PointerAction::Release(button),
+            };
+            pointer_input.write(PointerInput::new(
+                CUBE_POINTER_ID,
+                Location {
+                    target: target.clone(),
+                    position: *cursor_last,
+                },
+                action,
+            ));
+        }
+    }
+
+    Ok(())
+}