Viewer: 3D layer outlines and the axis gizmo clip at the camera instead of drawing long lines (#572)

The viewer projected a 3D layer's box, its axis gizmo and its Extended Viewer outline
through the camera with a plain perspective divide. A point at or behind the camera
(w ≤ 0) flips through infinity, so a layer turned or moved so that part of it reaches past
the camera drew long lines across the viewer, and its handles and hit area flipped too.

- `layer_quad` (handles, hit tests, marquee) is None for a layer reaching past the near
  plane, so nothing is placed or picked on a flipped quad.
- `layer_outline` clips the box to the part in front of the near plane (Sutherland-Hodgman
  in homogeneous coordinates); the selection outline and the Extended Viewer use it.
- The 3D axis gizmo's arrows are clipped the same way (`project_segment`).

Tests: a box crossing the near plane keeps only its front part, one wholly behind draws
nothing; gizmo segments are cut at the plane and never flip.

Note: the reporter's exact setup (ellipse, X/Y Rotation, Orientation) wasn't reproduced
in a headless probe with the default camera, where the layer stays in front of it; this
fixes the projection fault that draws such lines.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01QiAJeFttZp83Fh7YuzAR5f

crates/ui-egui/src/panels/viewer.rs

@@ -258,17 +258,87 @@ pub(crate) fn fx2c(ctx: &EvalCtx, layer: &Layer) -> Mat3 {
     from_effect_space(ctx, layer, l2c(ctx, layer).0)
 }
 
-/// Layer → comp matrix and its bounds quad (in comp pixels).
+/// The least homogeneous depth (`w`: the distance in front of a perspective camera in pixels, 1
+/// for 2D layers and orthographic views) a viewer outline is drawn at: points at or behind the
+/// camera would flip through infinity and draw long lines across the viewer (#572).
+const NEAR_W: f64 = 0.5;
+
+/// `p` through the homography `m` before the perspective divide: (x, y, w).
+fn homogeneous(m: &Mat3, p: [f64; 2]) -> [f64; 3] {
+    let r = |i: usize| m.0.get(i).map_or(0.0, |row| row[0] * p[0] + row[1] * p[1] + row[2]);
+    [r(0), r(1), r(2)]
+}
+
+/// The closed polygon `poly` (layer pixels) through `m`, clipped to the part in front of the
+/// camera's near plane (comp pixels; empty when none of it is).
+fn project_clipped(m: &Mat3, poly: &[[f64; 2]]) -> Vec<[f64; 2]> {
+    let h: Vec<[f64; 3]> = poly.iter().map(|p| homogeneous(m, *p)).collect();
+    let mut out = vec![];
+    for (i, a) in h.iter().enumerate() {
+        let Some(b) = h.get((i + 1) % h.len()) else { continue };
+        let (ina, inb) = (a[2] >= NEAR_W, b[2] >= NEAR_W);
+        if ina {
+            out.push(*a);
+        }
+        if ina != inb {
+            let t = (NEAR_W - a[2]) / (b[2] - a[2]);
+            out.push([a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, NEAR_W]);
+        }
+    }
+    out.iter().map(|q| [q[0] / q[2], q[1] / q[2]]).filter(|q| q[0].is_finite() && q[1].is_finite()).collect()
+}
+
+/// The segment a → b (world space) through the projection `pc`, clipped at the near plane.
+fn project_segment(pc: &effectcraft_engine::geom::Mat4, a: effectcraft_engine::geom::Vec3, b: effectcraft_engine::geom::Vec3) -> Option<[[f64; 2]; 2]> {
+    let h = |p: effectcraft_engine::geom::Vec3| {
+        let r = |i: usize| pc.0.get(i).map_or(0.0, |row| row[0] * p.x + row[1] * p.y + row[2] * p.z + row[3]);
+        [r(0), r(1), r(3)]
+    };
+    let (mut ha, mut hb) = (h(a), h(b));
+    if ha[2] < NEAR_W && hb[2] < NEAR_W {
+        return None;
+    }
+    let cut = |p: [f64; 3], q: [f64; 3]| {
+        let t = (NEAR_W - p[2]) / (q[2] - p[2]);
+        [p[0] + (q[0] - p[0]) * t, p[1] + (q[1] - p[1]) * t, NEAR_W]
+    };
+    if ha[2] < NEAR_W {
+        ha = cut(ha, hb);
+    } else if hb[2] < NEAR_W {
+        hb = cut(hb, ha);
+    }
+    let d = |q: [f64; 3]| [q[0] / q[2], q[1] / q[2]];
+    let s = [d(ha), d(hb)];
+    s.iter().flatten().all(|v| v.is_finite()).then_some(s)
+}
+
+/// The content box corners (layer pixels) of `layer`.
+fn box_corners(b: [f64; 4]) -> [[f64; 2]; 4] {
+    [[b[0], b[1]], [b[2], b[1]], [b[2], b[3]], [b[0], b[3]]]
+}
+
+/// Layer → comp matrix and its bounds quad (in comp pixels). None for a 3D layer reaching to or
+/// behind the camera (its quad would flip; [`layer_outline`] draws the part in front).
 pub(crate) fn layer_quad(ctx: &EvalCtx, layer: &Layer) -> Option<(Mat3, [[f64; 2]; 4], [f64; 4])> {
     let b = effectcraft_engine::render::content_bounds(ctx, layer)?;
     let (m, _) = l2c(ctx, layer);
-    let pts = [[b[0], b[1]], [b[2], b[1]], [b[2], b[3]], [b[0], b[3]]].map(|p| {
+    let corners = box_corners(b);
+    if corners.iter().any(|p| homogeneous(&m, *p)[2] < NEAR_W) {
+        return None;
+    }
+    let pts = corners.map(|p| {
         let q = m.apply(gv2(p[0], p[1]));
         [q.x, q.y]
     });
     Some((m, pts, b))
 }
 
+/// The outline of a layer's content box in comp pixels, clipped at the camera's near plane.
+pub(crate) fn layer_outline(ctx: &EvalCtx, layer: &Layer) -> Vec<[f64; 2]> {
+    let Some(b) = effectcraft_engine::render::content_bounds(ctx, layer) else { return vec![] };
+    project_clipped(&l2c(ctx, layer).0, &box_corners(b))
+}
+
 /// Which axes a bounding-box handle scales: corners (0–3) both; the top and bottom edges (4, 6)
 /// y; the right and left edges (5, 7) x.
 fn handle_axes(handle: usize) -> [bool; 2] {
@@ -836,6 +906,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
                 }
             }
             let Some((m, q, _)) = layer_quad(&ectx, l) else {
+                // A 3D layer reaching past the camera: the part of its box in front of it.
+                let outline = layer_outline(&ectx, l);
+                if outline.len() > 2 {
+                    painter.add(egui::Shape::closed_line(outline.iter().map(|p| map.to_screen(*p)).collect(), Stroke::new(1.0, col)));
+                    continue;
+                }
                 // Cameras, lights, empty layers: just the anchor.
                 if let Some(tr) = l.transform() {
                     let pos = ectx.v3(l, tr, "position", [0.0; 3]);
@@ -868,15 +944,16 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
                     let w = ectx.world_matrix(l);
                     let pc = cam_state(&ectx).projection(cw as f64, ch as f64) * w;
                     let a3 = effectcraft_engine::geom::vec3(a[0], a[1], a[2]);
-                    let o = pc.apply(a3);
                     let len = 60.0 / zoom as f64;
                     for (axis, col) in [
                         (effectcraft_engine::geom::vec3(len, 0.0, 0.0), Color32::from_rgb(0xe0, 0x50, 0x50)),
                         (effectcraft_engine::geom::vec3(0.0, len, 0.0), Color32::from_rgb(0x60, 0xd0, 0x60)),
                         (effectcraft_engine::geom::vec3(0.0, 0.0, len), Color32::from_rgb(0x50, 0x8c, 0xf0)),
                     ] {
-                        let e = pc.apply(a3 + axis);
-                        painter.arrow(map.to_screen([o.x, o.y]), map.to_screen([e.x, e.y]) - map.to_screen([o.x, o.y]), Stroke::new(2.0, col));
+                        // (clipped at the camera's near plane, #572)
+                        if let Some([o, e]) = project_segment(&pc, a3, a3 + axis) {
+                            painter.arrow(map.to_screen(o), map.to_screen(e) - map.to_screen(o), Stroke::new(2.0, col));
+                        }
                     }
                 }
             }
@@ -888,7 +965,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
     let has_3d = comp.layers.iter().any(|l| l.is_3d() && l.is_active_at(time));
     if has_3d && app.session.prefs.three_d.extended_viewer {
         for l in comp.layers.iter().filter(|l| l.is_3d() && l.is_active_at(time) && l.has_video()) {
-            if let Some((_, q, _)) = layer_quad(&ectx, l) {
+            let q = layer_outline(&ectx, l);
+            if q.len() > 2 {
                 let sq: Vec<Pos2> = q.iter().map(|p| map.to_screen(*p)).collect();
                 let outside = sq.iter().any(|p| !comp_rect.contains(*p));
                 if outside {
@@ -2228,3 +2306,38 @@ fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames:
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use effectcraft_engine::geom::{Mat4, vec3};
+
+    /// A perspective-like homography: w = 100 + y (the plane crosses the camera at y = −100).
+    fn tilted() -> Mat3 {
+        Mat3([[100.0, 0.0, 0.0], [0.0, 100.0, 0.0], [0.0, 1.0, 100.0]])
+    }
+
+    /// #572: a box reaching behind the camera is clipped at the near plane instead of flipping
+    /// through infinity into long lines.
+    #[test]
+    fn outlines_clip_at_the_near_plane() {
+        let m = tilted();
+        let front = project_clipped(&m, &box_corners([-50.0, -50.0, 50.0, 50.0]));
+        assert_eq!(front.len(), 4);
+        let crossing = project_clipped(&m, &box_corners([-50.0, -150.0, 50.0, 50.0]));
+        assert_eq!(crossing.len(), 4, "{crossing:?}");
+        // Every point is in front: none flipped to the other side (y < 0 here means behind).
+        assert!(crossing.iter().all(|p| p[0].is_finite() && p[1].abs() < 1e4 && p[0].abs() < 1e4), "{crossing:?}");
+        assert!(project_clipped(&m, &box_corners([-50.0, -400.0, 50.0, -200.0])).is_empty(), "wholly behind");
+    }
+
+    #[test]
+    fn gizmo_segments_clip_at_the_near_plane() {
+        // w = 100 + z.
+        let pc = Mat4([[100.0, 0.0, 0.0, 0.0], [0.0, 100.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 1.0, 100.0]]);
+        let s = project_segment(&pc, vec3(10.0, 0.0, 0.0), vec3(10.0, 0.0, -300.0)).unwrap();
+        assert_eq!(s[0], [10.0, 0.0]);
+        assert!(s[1][0] > 10.0 && s[1][0].is_finite(), "toward the camera it grows, never flips: {s:?}");
+        assert!(project_segment(&pc, vec3(0.0, 0.0, -200.0), vec3(0.0, 0.0, -300.0)).is_none());
+    }
+}