perf(engine): lasso and polygon selections rasterise only the rows they span (#4532) (#4773)

polygon_selection rasterised every lasso over the whole canvas (a
w×h f32 mask) and combined it over the whole canvas, whatever the
polygon's size.

Without feather it now rasterises the full-width band of rows the
polygon spans and combines over that band; Add and Subtract keep the
old selection outside it, Replace and Intersect leave it empty there,
exactly as the whole-canvas combine does. Full-width rows keep
`polygon`'s coverage bit-identical (its x arithmetic is relative to the
area's left edge, its y samples whole rows plus the same fractions);
a feathered lasso keeps the whole-canvas path, whose blur reaches past
the polygon. A test compares 24 random polygons, partly off the canvas,
in every mode, with and without anti-aliasing, against the old path.

Measured, a 2000-point lasso of a 400 px circle on 5472×3648: Replace
15–16 ms → 5 ms; Add to a selection unchanged (about 30 ms).

Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/engine/src/selection_cmds.rs

@@ -5,6 +5,7 @@
 use photocraft_algo::selection::{self as sel, SelectionMode};
 use photocraft_doc::Document;
 use photocraft_geom::Rect;
+use photocraft_raster::Surface;
 use serde_json::{Value, json};
 
 use crate::commands::CommandSpec;
@@ -603,11 +604,51 @@ pub(crate) fn polygon_selection(s: &mut Session, cmd: &str, label: &str, pts: &[
         return Err(EngineError::BadParams { cmd: cmd.into(), msg: "feather must be a number in 0..1000".into() });
     }
     let area = s.active().ok_or(EngineError::NoDocument)?.doc.bounds();
-    let mut mask = sel::polygon(pts, area, b(p, "antiAlias", true));
     if feather > 0.0 {
+        // The blur spreads past the polygon and isn't bit-identical over a smaller window.
+        let mut mask = sel::polygon(pts, area, b(p, "antiAlias", true));
         mask = sel::feather(&mask, area.width() as usize, area.height() as usize, feather);
+        return set_selection(s, label, area, mask, mode(p));
     }
-    set_selection(s, label, area, mask, mode(p))
+    // Only the rows the polygon spans (#4532): rows outside it are empty, and full-width rows
+    // keep `polygon`'s coverage bit-identical (its x arithmetic is relative to the area's left
+    // edge; its y samples are whole rows plus the same sub-row fractions).
+    let band = polygon_rows(pts, area);
+    let mask = sel::polygon(pts, band, b(p, "antiAlias", true));
+    let m = mode(p);
+    let selected = s.edit(label, |doc, _| {
+        doc.selection = combine_band(doc.selection.as_ref(), &mask, band, m);
+        Ok(doc.selection.is_some())
+    })?;
+    Ok(json!({ "selected": selected }))
+}
+
+/// The full-width rows of `area` that points `pts` span (empty when none are finite).
+fn polygon_rows(pts: &[(f32, f32)], area: Rect) -> Rect {
+    let ys = pts.iter().map(|p| p.1).filter(|y| y.is_finite());
+    let (lo, hi) = ys.fold((f32::INFINITY, f32::NEG_INFINITY), |(lo, hi), y| (lo.min(y), hi.max(y)));
+    if lo > hi {
+        return Rect::EMPTY;
+    }
+    let clamp = |v: f32| v.clamp(area.y0 as f32, area.y1 as f32) as i32;
+    let (y0, y1) = (clamp(lo.floor()), clamp(hi.ceil() + 1.0));
+    if y0 >= y1 { Rect::EMPTY } else { Rect::new(area.x0, y0, area.x1, y1) }
+}
+
+/// [`sel::combine`] of a mask covering only `band` (zero elsewhere), giving exactly what the
+/// whole-canvas combine gives: Replace and Intersect are empty outside the band, and Add and
+/// Subtract keep the old selection there.
+fn combine_band(old: Option<&Surface>, mask: &[f32], band: Rect, m: SelectionMode) -> Option<Surface> {
+    let inside = sel::combine(old, mask, band, m);
+    let Some(old) = old.filter(|_| matches!(m, SelectionMode::Add | SelectionMode::Subtract)) else { return inside };
+    if band.is_empty() {
+        return Some(old.clone());
+    }
+    let values = inside.map_or_else(|| vec![0.0; band.width() as usize * band.height() as usize], |w| w.read_region(band));
+    let mut s = old.clone();
+    s.write_region(band, &values);
+    s.prune();
+    (s.tile_count() > 0).then_some(s)
 }
 
 macro_rules! spec {
@@ -1270,4 +1311,35 @@ mod tests {
             }
         }
     }
+
+    /// The lasso over the polygon's rows gives exactly the whole-canvas result (#4532): random
+    /// polygons, partly off the canvas, with and without anti-aliasing, in every mode over an
+    /// existing selection.
+    #[test]
+    fn lasso_over_its_rows_matches_the_whole_canvas() {
+        let mut seed = 0x2545_f491_u32;
+        let mut rnd = |lo: f32, hi: f32| {
+            seed ^= seed << 13;
+            seed ^= seed >> 17;
+            seed ^= seed << 5;
+            lo + (seed % 10_000) as f32 / 10_000.0 * (hi - lo)
+        };
+        for case in 0..24 {
+            let mut s = Session::new();
+            s.execute("file.new", json!({"width": 300, "height": 520})).unwrap();
+            s.execute("select.rect", json!({"x": 40, "y": 60, "width": 200, "height": 300})).unwrap();
+            let n = 3 + case % 7;
+            let pts: Vec<(f32, f32)> = (0..n).map(|_| (rnd(-30.0, 330.0), rnd(100.0, 420.0))).collect();
+            let aa = case % 2 == 0;
+            let m = ["replace", "add", "subtract", "intersect"][case % 4];
+            let doc = s.active().unwrap().doc.clone();
+            let area = doc.bounds();
+            let want = sel::combine(doc.selection.as_ref(), &sel::polygon(&pts, area, aa), area, SelectionMode::parse(m));
+            let points: Vec<[f32; 2]> = pts.iter().map(|p| [p.0, p.1]).collect();
+            s.execute("select.lasso", json!({"points": points, "antiAlias": aa, "mode": m})).unwrap();
+            let got = s.active().unwrap().doc.selection.clone();
+            let read = |x: &Option<Surface>| x.as_ref().map(|s| s.read_region(area));
+            assert_eq!(read(&got), read(&want), "case {case}: {m}, anti-alias {aa}, {pts:?}");
+        }
+    }
 }