1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
//! Handles picking in 2D and 3D spaces.
use std::collections::HashSet;
use re_entity_db::InstancePathHash;
use re_log_types::Instance;
use re_renderer::PickingLayerProcessor;
use crate::eye::Eye;
use crate::PickableTexturedRect;
#[derive(Clone, PartialEq, Eq)]
pub enum PickingHitType {
/// The hit was a textured rect.
TexturedRect,
/// The result came from GPU based picking.
GpuPickingResult,
/// We hit a egui ui element, meaning that depth information is not usable.
GuiOverlay,
}
#[derive(Clone, PartialEq)]
pub struct PickingRayHit {
/// What entity or instance got hit by the picking ray.
///
/// The ray hit position may not actually be on this entity, as we allow snapping to closest entity!
pub instance_path_hash: InstancePathHash,
/// Where the ray hit the entity.
pub space_position: glam::Vec3,
pub depth_offset: re_renderer::DepthOffset,
/// Any additional information about the picking hit.
pub hit_type: PickingHitType,
}
#[derive(Clone, PartialEq)]
pub struct PickingResult {
/// Picking ray hits. NOT sorted by distance but rather by source of picking.
///
/// Typically there is only one hit, but there might be several if there are transparent objects
/// or "aggressive" objects like 2D images which we always want to pick, even if they're in the background.
/// (This is very useful for 2D scenes and so far we keep this behavior in 3D for simplicity)
pub hits: Vec<PickingRayHit>,
}
impl PickingResult {
pub fn space_position(&self) -> Option<glam::Vec3> {
// Use gpu hit if available as they are usually the position one expects.
// (other picking sources might be in here even if hidden!)
self.hits
.iter()
.find(|h| h.hit_type == PickingHitType::GpuPickingResult)
.or_else(|| self.hits.first())
.map(|hit| hit.space_position)
}
}
/// Picking context in which picking is performed.
pub struct PickingContext {
/// Cursor position in the UI coordinate system.
#[allow(unused)]
pub pointer_in_ui: glam::Vec2,
/// Cursor position on the renderer canvas in pixels.
pub pointer_in_pixel: glam::Vec2,
/// Cursor position in the UI coordinates after panning & zooming.
///
/// As of writing, for 3D spaces this is equal to [`Self::pointer_in_ui`],
/// since we don't allow panning & zooming after perspective projection.
pub pointer_in_camera_plane: glam::Vec2,
/// Transforms ui coordinates to "ui-camera-plane-coordinates"
/// Ui camera plane coordinates are ui coordinates that have been panned and zoomed.
///
/// See also [`Self::pointer_in_camera_plane`].
pub camera_plane_from_ui: egui::emath::RectTransform,
/// The picking ray used. Given in the coordinates of the space the picking is performed in.
pub ray_in_world: re_math::Ray3,
}
impl PickingContext {
/// Radius in which cursor interactions may snap to the nearest object even if the cursor
/// does not hover it directly.
///
/// Note that this needs to be scaled when zooming is applied by the virtual->visible ui rect transform.
pub const UI_INTERACTION_RADIUS: f32 = 5.0;
/// Creates a new [`PickingContext`] for executing picking operations and providing
/// information about the picking ray & general circumstances.
pub fn new(
pointer_in_ui: egui::Pos2,
camera_plane_from_ui: egui::emath::RectTransform,
pixels_per_point: f32,
eye: &Eye,
) -> Self {
let pointer_in_camera_plane = camera_plane_from_ui.transform_pos(pointer_in_ui);
let pointer_in_camera_plane =
glam::vec2(pointer_in_camera_plane.x, pointer_in_camera_plane.y);
let pointer_in_pixel =
(pointer_in_ui - camera_plane_from_ui.from().left_top()) * pixels_per_point;
Self {
pointer_in_camera_plane,
pointer_in_pixel: glam::vec2(pointer_in_pixel.x, pointer_in_pixel.y),
pointer_in_ui: glam::vec2(pointer_in_ui.x, pointer_in_ui.y),
camera_plane_from_ui,
ray_in_world: eye.picking_ray(*camera_plane_from_ui.to(), pointer_in_camera_plane),
}
}
/// Performs picking for a given scene.
pub fn pick<'a>(
&self,
render_ctx: &re_renderer::RenderContext,
gpu_readback_identifier: re_renderer::GpuReadbackIdentifier,
previous_picking_result: &Option<PickingResult>,
images: impl Iterator<Item = &'a PickableTexturedRect>,
ui_rects: &[PickableUiRect],
) -> PickingResult {
re_tracing::profile_function!();
// Gather picking results from different sources.
let gpu_pick = picking_gpu(
render_ctx,
gpu_readback_identifier,
self,
previous_picking_result,
);
let mut rect_hits = picking_textured_rects(self, images);
rect_hits.sort_by(|a, b| b.depth_offset.cmp(&a.depth_offset));
let ui_rect_hits = picking_ui_rects(self, ui_rects);
let mut hits = Vec::new();
// Start with gpu based picking as baseline. This is our prime source of picking information.
//
// ..unless the same object got also picked as part of a textured rect.
// Textured rect picks also know where on the rect, making this the better source!
// Note that whenever this happens, it means that the same object path has a textured rect and something else
// e.g. a camera.
if let Some(gpu_pick) = gpu_pick {
if rect_hits.iter().all(|rect_hit| {
rect_hit.instance_path_hash.entity_path_hash
!= gpu_pick.instance_path_hash.entity_path_hash
}) {
hits.push(gpu_pick);
}
}
// We never throw away any textured rects, even if they're behind other objects.
hits.extend(rect_hits);
// UI rects are overlaid on top, but we don't let them hide other picking results either.
// Give any other previous hits precedence.
let previously_hit_objects: HashSet<_> = hits
.iter()
.map(|prev_hit| prev_hit.instance_path_hash)
.collect();
hits.extend(
ui_rect_hits
.into_iter()
.filter(|ui_hit| !previously_hit_objects.contains(&ui_hit.instance_path_hash)),
);
PickingResult { hits }
}
}
fn picking_gpu(
render_ctx: &re_renderer::RenderContext,
gpu_readback_identifier: u64,
context: &PickingContext,
previous_picking_result: &Option<PickingResult>,
) -> Option<PickingRayHit> {
re_tracing::profile_function!();
// Only look at newest available result, discard everything else.
let mut gpu_picking_result = None;
while let Some(picking_result) =
PickingLayerProcessor::next_readback_result::<()>(render_ctx, gpu_readback_identifier)
{
gpu_picking_result = Some(picking_result);
}
if let Some(gpu_picking_result) = gpu_picking_result {
// First, figure out where on the rect the cursor is by now.
// (for simplicity, we assume the screen hasn't been resized)
let pointer_on_picking_rect =
context.pointer_in_pixel - gpu_picking_result.rect.min.as_vec2();
// The cursor might have moved outside of the rect. Clamp it back in.
let pointer_on_picking_rect = pointer_on_picking_rect.clamp(
glam::Vec2::ZERO,
(gpu_picking_result.rect.extent - glam::UVec2::ONE).as_vec2(),
);
// Find closest non-zero pixel to the cursor.
let mut picked_id = re_renderer::PickingLayerId::default();
let mut picked_on_picking_rect = glam::Vec2::ZERO;
let mut closest_rect_distance_sq = f32::INFINITY;
for (i, id) in gpu_picking_result.picking_id_data.iter().enumerate() {
if id.object.0 != 0 {
let current_pos_on_picking_rect = glam::uvec2(
i as u32 % gpu_picking_result.rect.extent.x,
i as u32 / gpu_picking_result.rect.extent.x,
)
.as_vec2()
+ glam::vec2(0.5, 0.5); // Use pixel center for distances.
let distance_sq =
current_pos_on_picking_rect.distance_squared(pointer_on_picking_rect);
if distance_sq < closest_rect_distance_sq {
picked_on_picking_rect = current_pos_on_picking_rect;
closest_rect_distance_sq = distance_sq;
picked_id = *id;
}
}
}
if picked_id == re_renderer::PickingLayerId::default() {
// Nothing found.
return None;
}
let picked_world_position =
gpu_picking_result.picked_world_position(picked_on_picking_rect.as_uvec2());
Some(PickingRayHit {
instance_path_hash: re_space_view::instance_path_hash_from_picking_layer_id(picked_id),
space_position: picked_world_position,
depth_offset: 1,
hit_type: PickingHitType::GpuPickingResult,
})
} else {
// It is possible that some frames we don't get a picking result and the frame after we get several.
// We need to cache the last picking result and use it until we get a new one or the mouse leaves the screen.
// (Andreas: On my mac this *actually* happens in very simple scenes, I get occasional frames with 0 and then with 2 picking results!)
if let Some(PickingResult { hits }) = previous_picking_result {
for previous_opaque_hit in hits {
if matches!(
previous_opaque_hit.hit_type,
PickingHitType::GpuPickingResult
) {
return Some(previous_opaque_hit.clone());
}
}
}
None
}
}
fn picking_textured_rects<'a>(
context: &PickingContext,
images: impl Iterator<Item = &'a PickableTexturedRect>,
) -> Vec<PickingRayHit> {
re_tracing::profile_function!();
let mut hits = Vec::new();
let mut hit_image_rect_entities = HashSet::new();
for image in images {
let rect = &image.textured_rect;
let Some(normal) = rect.extent_u.cross(rect.extent_v).try_normalize() else {
continue; // extent_u and extent_v are parallel. Shouldn't happen.
};
let rect_plane = re_math::Plane3::from_normal_point(normal, rect.top_left_corner_position);
// TODO(andreas): Interaction radius is currently ignored for rects.
let (intersect, t) =
rect_plane.intersect_ray(context.ray_in_world.origin, context.ray_in_world.dir);
if !intersect {
continue;
}
let intersection_world = context.ray_in_world.point_along(t);
let dir_from_rect_top_left = intersection_world - rect.top_left_corner_position;
let u = dir_from_rect_top_left.dot(rect.extent_u) / rect.extent_u.length_squared();
let v = dir_from_rect_top_left.dot(rect.extent_v) / rect.extent_v.length_squared();
if (0.0..=1.0).contains(&u) && (0.0..=1.0).contains(&v) {
let [width, height] = rect.colormapped_texture.width_height();
// Ignore the image if we hit the same entity already as an image.
// This happens if the same entity has multiple textured rects.
let entity_path_hash = image.ent_path.hash();
if hit_image_rect_entities.insert(entity_path_hash) {
hits.push(PickingRayHit {
instance_path_hash: InstancePathHash {
entity_path_hash,
instance: Instance::from_2d_image_coordinate(
[(u * width as f32) as u32, (v * height as f32) as u32],
width as u64,
),
},
space_position: intersection_world,
hit_type: PickingHitType::TexturedRect,
depth_offset: rect.options.depth_offset,
});
}
}
}
hits
}
pub struct PickableUiRect {
pub rect: egui::Rect,
pub instance_hash: InstancePathHash,
}
fn picking_ui_rects(
context: &PickingContext,
ui_rects: &[PickableUiRect],
) -> Option<PickingRayHit> {
re_tracing::profile_function!();
let egui_pos = egui::pos2(
context.pointer_in_camera_plane.x,
context.pointer_in_camera_plane.y,
);
for ui_rect in ui_rects {
if ui_rect.rect.contains(egui_pos) {
// Handle only a single ui rectangle (exit right away, ignore potential overlaps)
return Some(PickingRayHit {
instance_path_hash: ui_rect.instance_hash,
space_position: context.ray_in_world.origin,
hit_type: PickingHitType::GuiOverlay,
depth_offset: 0,
});
}
}
None
}