LineMaterial (from lines.rs) incompatible with raycast picking? (And: gaps in mouse-tracking input)

[wbpaley]

Two related questions here. Pardon if that's not desired: should I be splitting them into two posts? (Seems once people spend the overhead reading I shouldn't impose it again...)

I'm trying to draw what look like 2d lines on a page in a 3d Bevy world, so I can later add 3d annotations. So I tried to smash together two standard Bevy examples: lines.rs and modified bits from bevy_mod_picking's event_listener.rs (the nifty callback closures).

Q1: My newbie interpretation of the error I get:
Invalid intersection check: `TriangleList` is the only supported `PrimitiveTopology`
makes me think I can't use both kinds of primitives, or need to label the Line* stuff I spawn as not to be compared while picking. That latter is not a solution for me: I'd like to be able to pick the lines, too. (Maybe I have to make meshes under them? Hope not.)

Q2: There are gaps in my lines that seem to indicate the On::<Pointer<Drag>> callback closure sometimes gets an incorrect delta. Or maybe I'm missing callback events somehow, or not interpreting them right? In the image below there were exactly three drags. (My code drops a teensy sphere at each position (for debugging here), and draws a line segment from (position + delta*-1) to position.)

Picture of problems, followed by a long-ish minimal example, follow.
(I shrank it as much as I could...)

TIA! Your caring that I get this right, and quick/correct responses in the past have been deeply appreciated.

use bevy_mod_picking::{DefaultPickingPlugins, PickableBundle};
use std::ops::{Add, Mul};
use bevy::core_pipeline::clear_color::ClearColorConfig;
use bevy::pbr::{MaterialPipeline, MaterialPipelineKey};
use bevy::prelude::*;
use bevy::render::camera::ScalingMode;
use bevy::render::mesh::{MeshVertexBufferLayout, PrimitiveTopology};
use bevy::render::render_resource::{AsBindGroup, PolygonMode, RenderPipelineDescriptor, ShaderRef, SpecializedMeshPipelineError};
use bevy_mod_picking::pointer::Location;
use bevy_mod_picking::prelude::{Drag, DragEnd, DragStart, Highlight, HighlightKind, On, Pickable, Pointer};

fn main() {
    App::new()
        .insert_resource(Msaa::Sample4)

        .add_plugins((DefaultPlugins.set(WindowPlugin {
            primary_window: Some(Window {
                resolution: (1920.0, 1080.0).into(),
                title: "theSame.page".to_string(),
                ..default()
            }),
            ..default()
        }),
        MaterialPlugin::<LineMaterial>::default()))

        // .add_plugins(DefaultPlugins.set(low_latency_window_plugin()))
        // All you need to do is add the picking plugin, with your backend of choice enabled in the
        // cargo features. By default, the bevy_mod_raycast backend is enabled via the
        // `backend_raycast` feature.
        .add_plugins(DefaultPickingPlugins)

        .add_systems(Startup, (
            setup,
            setup_page,
        ))
        .add_systems(Update, (
            draw_cursor,
            draw_system,
        ))
        .run();
}

#[derive(Component)]
struct CursorOnPage {
    entity: Entity,
    location: Location,
    delta: Vec2,
}

fn draw_system (
    mut query: Query<&CursorOnPage>,

    camera_query: Query<(&Camera, &GlobalTransform)>,
    ground_query: Query<&GlobalTransform, With<Ground>>,

    mut commands: Commands,
    mut meshes: ResMut<Assets<Mesh>>,
    mut materials: ResMut<Assets<StandardMaterial>>,
    mut materials2: ResMut<Assets<LineMaterial>>,
) {
    for pos in query.iter_mut() {
        let (camera, camera_transform) = camera_query.single();
        let ground = ground_query.single();

        //  project cursor pos through camera transform into world; find where that ray hits GROUND
        let Some(ray) = camera.viewport_to_world(
            camera_transform, pos.location.position)
            else { return; };
        let Some(distance) = ray.intersect_plane(ground.translation(), ground.up())
            else { return; };
        let here = ray.get_point(distance);

        //  mark this current vertex with a tiny sphere
        commands.spawn((
            PbrBundle {
                mesh: meshes.add(Mesh::from(shape::UVSphere { radius: 0.005, stacks: 4, sectors: 8 })),
                material: materials.add(Color::rgb(0.3, 0.1, 0.99).with_l(0.1).into()),
                transform: Transform::from_translation(here),
                ..default()
            },
        ));

        //  project again for the previous pos
        let Some(ray) = camera.viewport_to_world(
            camera_transform, pos.location.position.add(pos.delta.mul(-1.0)))
            else { return; };
        let Some(distance) = ray.intersect_plane(ground.translation(), ground.up())
            else { return; };
        let were = ray.get_point(distance);

        //  spawn a line strip from previous to current;  todo: not seg by seg!
        commands.spawn(MaterialMeshBundle {
            mesh: meshes.add(Mesh::from(LineStrip {
                points: vec![
                    were,
                    here,
                ],
            })),
            transform: Transform::from_xyz(0.0, 0.0001, 0.0),  // above ground
            material: materials2.add(LineMaterial { color: Color::BLUE }),
            ..default()
        });

        //  executed; don't come back;  todo: is there a better way to execute exactly once?
        commands.entity(pos.entity).remove::<CursorOnPage>();
    }
}

fn setup_page(
    mut commands: Commands,
    mut meshes: ResMut<Assets<Mesh>>,
    mut materials: ResMut<Assets<StandardMaterial>>,
) {
    // the page
    commands.spawn((
        PbrBundle {
            mesh: meshes.add(Mesh::from(shape::Plane::from_size(5.0))),
            material: materials.add(Color::rgba(0.95, 0.95, 0.95, 0.9).into()),
            ..default()
        },
        Ground,
    ))
        .insert((
            PickableBundle::default(),
            HIGHLIGHT_TINT.clone(),
            On::<Pointer<DragStart>>::target_insert(Pickable::IGNORE), // Disable picking
            On::<Pointer<Drag>>::commands_mut(|event, commands| {
                commands.entity(event.target).insert(CursorOnPage {
                    entity: event.target,
                    location: event.pointer_location.clone(),
                    delta: event.delta.clone(),
                });
            }),
            On::<Pointer<DragEnd>>::target_insert(Pickable::default()), // Re-enable picking
        ));
}

//  override default highlight states to return same color; todo: learn how to just disable
const HIGHLIGHT_TINT: Highlight<StandardMaterial> = Highlight {
    hovered: Some(HighlightKind::new_dynamic(|matl| StandardMaterial { ..matl.to_owned() })),
    pressed: Some(HighlightKind::new_dynamic(|matl| StandardMaterial { ..matl.to_owned() })),
    selected: Some(HighlightKind::new_dynamic(|matl| StandardMaterial { ..matl.to_owned() })),
};

#[derive(Component)]
struct Ground;

fn draw_cursor(
    camera_query: Query<(&Camera, &GlobalTransform)>,
    ground_query: Query<&GlobalTransform, With<Ground>>,
    windows: Query<&Window>,
    mut gizmos: Gizmos,
) {
    let (camera, camera_transform) = camera_query.single();
    let ground = ground_query.single();

    let Some(cursor_position) = windows.single().cursor_position() else { return; };

    // Calculate a ray pointing from the camera into the world based on the cursor's position.
    let Some(ray) = camera.viewport_to_world(camera_transform, cursor_position) else { return; };

    // Calculate if and where the ray is hitting the ground plane.
    let Some(distance) = ray.intersect_plane(ground.translation(), ground.up()) else { return; };
    let point = ray.get_point(distance);

    // Draw a circle just above the ground plane at that position.
    gizmos.circle(point + ground.up() * 0.01, ground.up(), 0.05, Color::GRAY);
}

fn setup(
    mut commands: Commands,
) {
    commands.spawn((Camera3dBundle {
        projection: OrthographicProjection {
            //  local US inches to world-standard meters
            scaling_mode: ScalingMode::FixedVertical(1.0/0.254),
            ..default()
        }.into(),
        camera_3d: Camera3d {
            clear_color: ClearColorConfig::Custom(Color::rgb(0.95, 0.95, 0.95)),
            ..default()
        },
        transform: Transform::from_xyz(0.0, 10.0, 0.0).looking_at(Vec3::ZERO, Vec3::Z),
        ..default()
    },));

    commands.spawn(DirectionalLightBundle {
        directional_light: DirectionalLight {
            color: Color::rgb(0.95, 0.95, 0.95),
            illuminance: 18000.0,
            ..default()
        },
        transform: Transform::from_xyz(1.0, 10.0, 2.0).looking_at(Vec3::ZERO, Vec3::Y),
        ..default()
    });
}

//  copied unmodified from Bevy standard demo lines.rs
//  pardon: new to this; don't yet know how to include by reference

#[derive(Asset, TypePath, Default, AsBindGroup, Debug, Clone)]
pub struct LineMaterial {
    #[uniform(0)]
    pub(crate) color: Color,
}

impl Material for LineMaterial {
    fn fragment_shader() -> ShaderRef {
        "shaders/line_material.wgsl".into()
    }

    fn specialize(
        _pipeline: &MaterialPipeline<Self>,
        descriptor: &mut RenderPipelineDescriptor,
        _layout: &MeshVertexBufferLayout,
        _key: MaterialPipelineKey<Self>,
    ) -> Result<(), SpecializedMeshPipelineError> {
        // This is the important part to tell bevy to render this material as a line between vertices
        descriptor.primitive.polygon_mode = PolygonMode::Line;
        Ok(())
    }
}

/// A list of lines with a start and end position
#[derive(Debug, Clone)]
pub struct LineList {
    pub lines: Vec<(Vec3, Vec3)>,
}

impl From<LineList> for Mesh {
    fn from(line: LineList) -> Self {
        let vertices: Vec<_> = line.lines.into_iter().flat_map(|(a, b)| [a, b]).collect();

        Mesh::new(
            // This tells wgpu that the positions are list of lines
            // where every pair is a start and end point
            PrimitiveTopology::LineList,
            // RenderAssetPersistencePolicy::Unload,
        )
            // Add the vertices positions as an attribute
            .with_inserted_attribute(Mesh::ATTRIBUTE_POSITION, vertices)
    }
}

/// A list of points that will have a line drawn between each consecutive points
#[derive(Debug, Clone)]
pub struct LineStrip {
    pub points: Vec<Vec3>,
}

impl From<LineStrip> for Mesh {
    fn from(line: LineStrip) -> Self {
        Mesh::new(
            // This tells wgpu that the positions are a list of points
            // where a line will be drawn between each consecutive point
            PrimitiveTopology::LineStrip,
            // RenderAssetPersistencePolicy::Unload,
        )
            // Add the point positions as an attribute
            .with_inserted_attribute(Mesh::ATTRIBUTE_POSITION, line.points)
    }
}

[nicopap]

Why am I getting this error message?

You deduced it right.

The default picking backend for bevy_mod_picking uses mesh collision to detect
what the mouse is hovering. To tell if the mouse is hovering over a mesh,
bevy_mod_picking reads the Mesh vertices and applies a collision
detection algorithm that is specific to meshes with a TriangleList topology,
it can't handle other kind of topologies.

This is a difficult problem to solve. I can't give you a ready-made solution. You
can do the following:

• Generate meshes with a TriangleList topology for your lines, and store them in RAM.
  This would also allow you to customize your line width!
• Have a bitmatrix that acts like a canvas, draw the lines into it, implement a
  picking backend for it.

Why are there gaps in the lines?

Your event handler is as follow:

On::<Pointer<Drag>>::commands_mut(|event, commands| {
    commands.entity(event.target).insert(CursorOnPage {
        entity: event.target,
        location: event.pointer_location.clone(),
        delta: event.delta.clone(),
    });
}),

It overwrites the CursorOnPage component each time it is called. Maybe the event
handler is called twice in the same frame? This could cause the gaps. I suggest pushing
an element to a Vec inside a resource.

#[derive(Resource)]
struct DragPoints(Vec<Vec2>);

struct AddDragPointCommand(Vec2)

impl Command for AddDragPointCommand {
    fn apply(self, world: &mut World) {
        let mut points = world.resource_mut::<DragPoints>();
        points.push(self.0);
    }
}

// ...
On::<Pointer<Drag>>::commands_mut(|event, commands| {
    commands.add(AddDragPointCommand(event.pointer_location.position));
}),
// ...

You could also run a system that sends an event. Then you could read all the events one
after another in a different system with the EventReader<MyCustomDragEvent> system param.

[wbpaley]

I followed both parts of the @nicopap advice: collected points in a resource and generated meshes for the lines. Fortunately lyon is compact and seems to do a good job. (Took days to first try making my own meshes, then trying to isolate code from Cairo and Skia. Grateful for lyon!..)

Here's the current state if it helps anyone. Remember, I've barely begun so I'm sure it could be done more concisely or with better form--but just in case it helps someone else I wanted to share it:

use bevy_mod_picking::{DefaultPickingPlugins, PickableBundle};
use std::ops::{Add};
use std::time::Duration;
use bevy::core_pipeline::clear_color::ClearColorConfig;
use bevy::ecs::system::Command;
use bevy::prelude::*;
use bevy::render::mesh::{Indices, PrimitiveTopology};
use bevy_mod_picking::prelude::{Drag, DragEnd, DragStart, Highlight, HighlightKind, On, Pickable, Pointer};
use bevy_prototype_lyon::prelude::{LineCap, LineJoin};
use lyon_algorithms::geom::point;
use lyon_algorithms::math::Point;
use lyon_tessellation::{StrokeOptions, VertexBuffers};

fn main() {
    App::new()
        .insert_resource(Msaa::Sample4)
        .insert_resource(DragPoints {
            0: vec![], })

        .add_plugins((DefaultPlugins.set(WindowPlugin {
            primary_window: Some(Window {
                resolution: (1920.0, 1080.0).into(),
                title: "theSame.page".to_string(),
                ..default()
            }),
            ..default()
        })))

        .add_plugins(DefaultPickingPlugins)

        .add_systems(Startup, (
            crate::setup,
            crate::setup_page,
        ))

        .add_systems(Update, (
            draw_polyline_system,
        ))

        .run();
}

#[derive(Component)]
struct DrawDragPoints {
    entity: Entity,
    done: bool,
}

//  use drag point list to draw a polyline (assumes single stroke input); todo: multi
struct DragPoint {
    position: Vec2,
    real_time_elapsed: Duration,
}

#[derive(Resource)]
struct DragPoints(Vec<(crate::DragPoint)>);

struct AddDragPointCommand(Vec2);
impl Command for crate::AddDragPointCommand {
    fn apply(self, world: &mut World) {
        let now = world.resource::<Time<Real>>().elapsed();
        let mut points = world.resource_mut::<crate::DragPoints>();
        points.0.push(crate::DragPoint {
            position: self.0,
            real_time_elapsed: now
        });
    }
}

#[derive(Component)]
struct CurrentStroke {
    entity: Entity,
}

pub fn draw_polyline_system(
    camera_query: Query<(&Camera, &GlobalTransform)>,
    ground_query: Query<&GlobalTransform, With<crate::Ground>>,

    mut commands: Commands,
    mut meshes: ResMut<Assets<Mesh>>,
    mut materials: ResMut<Assets<StandardMaterial>>,

    mut points: ResMut<crate::DragPoints>,
    mut draw_query: Query<&crate::DrawDragPoints>,
    mut current_stroke_query: Query<&crate::CurrentStroke>,
) {
    let mut done = false;
    for (draw) in draw_query.iter() {
        commands.entity(draw.entity).remove::<crate::DrawDragPoints>();

        for (current_stroke) in current_stroke_query.iter() {
            commands.entity(current_stroke.entity).despawn();
        }

        //  Create a simple path.
        let mut path_builder = lyon_tessellation::path::Path::builder();

        // Create the destination vertex and index buffers.
        let mut buffers: VertexBuffers<Point, u16> = VertexBuffers::new();

        // Create the vertex builder
        let mut vertex_builder = lyon_tessellation::geometry_builder::simple_builder(&mut buffers);

        // Create the tessellator.
        let mut tessellator = lyon_tessellation::StrokeTessellator::new();

        let mut this_point_index = 0;
        let mut here = Vec3::ZERO;
        for ((drag_point)) in points.0.iter() {
            let (camera, camera_transform) = camera_query.single();
            let ground = ground_query.single();
            //  project cursor pos through camera transform into world, then find where that ray hits GROUND
            let Some(ray) = camera.viewport_to_world(camera_transform, drag_point.position)
                else { return; };
            let Some(distance) = ray.intersect_plane(ground.translation(), ground.up())
                else { return; };
            here = ray.get_point(distance);

            if this_point_index == 0 {
                path_builder.begin(point(here.x, here.z));
            } else {
                path_builder.line_to(point(here.x, here.z));
            }

            this_point_index += 1;
        }

        if this_point_index > 0 {
            //  catch a click without a drag move: make a dot
            if this_point_index == 1 {
                path_builder.line_to(point(here.x, here.z));
            }

            path_builder.end(false);
            let path = path_builder.build();

            // Compute the tessellation.
            tessellator.tessellate(
                &path,
                //  join & cap resolutions seem to be tied to size; todo: make smoother
                &StrokeOptions::default()
                    .with_line_width(3.5)
                    .with_line_join(LineJoin::Round)
                    .with_start_cap(LineCap::Round)
                    .with_end_cap(LineCap::Round),
                &mut vertex_builder
            ).expect("can't tessellate line");

            let mut vv = Vec::new();
            for (v) in buffers.vertices {
                vv.push(Vec3::new(v.x, 0.0, v.y));
            }

            let mut indices = Vec::new();
            for (i) in buffers.indices {
                indices.push(i as u32);
            }
            let indices = Indices::U32(indices);

            let mut mesh = Mesh::new(PrimitiveTopology::TriangleList);
            mesh.set_indices(Some(indices));
            mesh.insert_attribute(Mesh::ATTRIBUTE_POSITION, vv);

            let id = commands.spawn((
                PbrBundle {
                    mesh: meshes.add(mesh),
                    material: materials.add(Color::rgba(0.5, 0.5, 0.8, 0.8).into()),
                    transform: Transform::from_xyz(0.0, 0.0001, 0.0),  //  just above ground
                    ..default()
                },
            )).id();

            commands.entity(id)
                .insert((
                    PickableBundle::default(),
                ));

            if !draw.done {
                commands.entity(id)
                    .insert((
                        crate::CurrentStroke { entity: id },
                    ));
            }

            done |= draw.done;
        }

        if done {
            &points.0.clear();
        }
    }
}

fn setup_page(
    mut commands: Commands,
    mut meshes: ResMut<Assets<Mesh>>,
    mut materials: ResMut<Assets<StandardMaterial>>,
) {
    // the page
    let m = materials.add(StandardMaterial {
        base_color: Color::rgba(1.0, 0.99, 0.98, 1.0),
        metallic: 0.0,
        perceptual_roughness: 1.0,
        ..default()
    });
    commands.spawn((
        PbrBundle {
            mesh: meshes.add(Mesh::from(shape::Plane::from_size(2000.0))),
            material: m.into(),
            ..default()
        },
        crate::Ground,
    )).insert((
        PickableBundle::default(),

        On::<Pointer<DragStart>>::target_insert(Pickable::IGNORE), // Disable picking
        On::<Pointer<Drag>>::commands_mut(|event, commands| {
            commands.add(crate::AddDragPointCommand(event.pointer_location.position));
            commands.entity(event.target).insert(crate::DrawDragPoints { entity:event.target, done: false });
        }),
        On::<Pointer<DragEnd>>::commands_mut(|event, commands| {
            commands.entity(event.target).insert(crate::DrawDragPoints { entity:event.target, done: true });
            commands.entity(event.target).insert(Pickable::default()); // Enable picking
        }),
    ));
}

#[derive(Component)]
struct Ground;

fn setup(
    mut commands: Commands,
) {
    commands.spawn((Camera3dBundle {
        projection: OrthographicProjection {
            ..default()
        }.into(),
        camera_3d: Camera3d {
            clear_color: ClearColorConfig::Custom(Color::rgb(0.95, 0.95, 0.95)),
            ..default()
        },
        transform: Transform::from_xyz(0.0, 10.0, 0.0).looking_at(Vec3::ZERO, Vec3::Z),
        ..default()
    },));

    commands.spawn(DirectionalLightBundle {
        directional_light: DirectionalLight {
            color: Color::rgb(0.95, 0.95, 0.95),
            illuminance: 18000.0,
            ..default()
        },
        transform: Transform::from_xyz(1.0, 10.0, 2.0).looking_at(Vec3::ZERO, Vec3::Y),
        ..default()
    });
}