fix(warp): Perspective Warp moves only the selected part of a linked mask (#3083) (#3148)

With a selection, Perspective Warp moved only the selected pixels but warped
the whole linked layer mask, so unselected pixels changed visibility. The mask
now follows the same split as Free Transform (#205): the selected region of a
linked mask moves with the pixels and the rest stays put.


Claude-Session: https://claude.ai/code/session_015Vepmc5pt6MikxbsDj21Bu

Co-authored-by: Leel Wasanatha <[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/engine/src/distort_cmds.rs

@@ -232,7 +232,8 @@ fn check_locks(l: &Layer, group: Locks, position: bool) -> Result<()> {
 
 /// Shared driver: a smart object records `cmd` as a smart filter; a pixel layer is edited in
 /// place by `f(surface, selection)`.
-type MaskFn<'a> = Option<&'a dyn Fn(&Surface) -> Surface>;
+/// Deforms a linked layer mask the same way, limited to the selection when there is one.
+type MaskFn<'a> = Option<&'a dyn Fn(&Surface, Option<&Surface>) -> Result<Surface>>;
 /// Deforms a layer's pixels (with the selection and canvas); an `Err` leaves the layer as it was.
 type LayerFn<'a> = &'a dyn Fn(&Surface, Option<&Surface>, Rect) -> Result<Surface>;
 
@@ -259,11 +260,11 @@ fn run_on_layer(s: &mut Session, cmd: &str, label: &str, p: &Value, moves: bool,
             return Err(EngineError::Other(format!("{label} needs a pixel layer (rasterize it first)")));
         };
         *surf = f(surf, selection.as_ref(), canvas)?;
-        // A linked layer mask follows a geometric warp.
+        // A linked layer mask follows a geometric warp, only where the pixels moved (#3083).
         if let (Some(mf), Some(m)) = (mask, l.mask.as_mut())
             && m.linked
         {
-            m.surface = mf(&m.surface);
+            m.surface = mf(&m.surface, selection.as_ref())?;
         }
         Ok(())
     })?;
@@ -344,9 +345,14 @@ fn perspective(s: &mut Session, p: &Value, cmd: &str) -> Result<Value> {
     }
     let it = interp(p);
     let map = PerspectiveMap::new(&planes);
-    let mask_fn = |m: &Surface| match &map {
-        Some(map) => warp_mesh_gray(m, &|x, y| map.map(x, y), Interp::Bilinear),
-        None => m.clone(),
+    let mask_fn = |m: &Surface, sel: Option<&Surface>| {
+        let Some(map) = &map else { return Ok(m.clone()) };
+        let f = |x, y| map.map(x, y);
+        let whole = |m: &Surface| Ok(warp_mesh_gray(m, &f, Interp::Bilinear));
+        match sel {
+            Some(sel) => crate::transform_cmds::move_gray_selected(m, sel, &|lifted, src| Ok(warp_mesh_surface(lifted, src, &f, Interp::Bilinear)), &whole),
+            None => whole(m),
+        }
     };
     let mut r = run_on_layer(
         s,

crates/engine/src/distort_cmds/tests.rs

@@ -215,6 +215,42 @@ fn perspective_identity_known_homography_and_mask() {
     assert!(s.execute(PERSPECTIVE, json!({"planes": [{"src": q, "dst": d}], "straighten": "sideways"})).is_err());
 }
 
+#[test]
+fn perspective_with_a_selection_moves_only_the_selected_part_of_a_linked_mask() {
+    // #3083: the mask hides a stripe inside the selection (x 12..16) and one outside it (x 40..44).
+    let mask_at = |s: &Session, x, y| {
+        let st = s.active().unwrap();
+        st.doc.layer(st.active_layer.unwrap()).unwrap().mask.as_ref().unwrap().surface.sample_channel(x, y, 0)
+    };
+    for selected in [true, false] {
+        let mut s = session(8);
+        s.execute("layer.layerMask.revealAll", json!({})).unwrap();
+        s.edit("mask", |doc, active| {
+            let m = doc.layer_mut(active.unwrap()).unwrap().mask.as_mut().unwrap();
+            m.surface.fill_rect(Rect::new(12, 10, 16, 40), &[0.0]);
+            m.surface.fill_rect(Rect::new(40, 10, 44, 40), &[0.0]);
+            Ok(())
+        })
+        .unwrap();
+        if selected {
+            s.execute("select.rect", json!({"x": 0, "y": 0, "width": 30, "height": 60})).unwrap();
+        }
+        let q = [[0.0, 0.0], [30.0, 0.0], [30.0, 60.0], [0.0, 60.0]];
+        let d = q.map(|p| [p[0] + 4.0, p[1]]);
+        s.execute(PERSPECTIVE, json!({"planes": [{"src": q, "dst": d}], "interpolation": "bilinear"})).unwrap();
+        // The selected stripe moves with the pixels either way.
+        assert!(mask_at(&s, 17, 25) < 0.01 && mask_at(&s, 13, 25) > 0.99, "selected={selected}");
+        if selected {
+            // The unselected stripe stays over the unselected pixels.
+            assert!(mask_at(&s, 43, 25) < 0.01 && mask_at(&s, 47, 25) > 0.99);
+            assert_eq!(active_surface(&s).pixel(43, 25)[3], 1.0);
+        } else {
+            // Without a selection the whole mask moves with the layer.
+            assert!(mask_at(&s, 47, 25) < 0.01 && mask_at(&s, 43, 25) > 0.99);
+        }
+    }
+}
+
 #[test]
 fn smart_objects_store_distortions_as_smart_filters_losslessly() {
     for depth in [8u64, 16, 32] {

crates/engine/src/transform_cmds.rs

@@ -123,12 +123,23 @@ pub fn split_gray_selected(s: &Surface, sel: &Surface) -> Result<Option<(Surface
 /// [`warp_gray`] limited to a selection: only the selected values move (the vacated area reads as
 /// the default), as the selected pixels of the layer do.
 pub(crate) fn warp_gray_selected(s: &Surface, sel: &Surface, h: &Homography, interp: Interp) -> Result<Surface> {
-    let Some((lifted, mut out)) = split_gray_selected(s, sel)? else { return warp_gray(s, h, interp) };
+    move_gray_selected(s, sel, &|lifted, src| Ok(warp_surface(lifted, src, h, interp)?), &|s| warp_gray(s, h, interp))
+}
+
+/// [`warp_gray_selected`] with any geometric warp: `warp` moves the lifted grey + alpha values
+/// (given their bounds), `whole` moves `s` when it isn't a single grey channel.
+pub(crate) fn move_gray_selected(
+    s: &Surface,
+    sel: &Surface,
+    warp: &dyn Fn(&Surface, Rect) -> Result<Surface>,
+    whole: &dyn Fn(&Surface) -> Result<Surface>,
+) -> Result<Surface> {
+    let Some((lifted, mut out)) = split_gray_selected(s, sel)? else { return whole(s) };
     let src = lifted.content_bounds();
     if src.is_empty() {
         return Ok(out);
     }
-    let w = warp_surface(&lifted, src, h, interp)?;
+    let w = warp(&lifted, src)?;
     let b = w.content_bounds();
     if !b.is_empty() {
         let moved = crate::allocation::read_region(&w, b, "compositing transformed mask pixels")?;