Never crash on hostile numbers: LUT sizes, Quick Selection points, link group ids

- Color Lookup: a stored document's `size` is untrusted. `size³ × 3` overflowed for sizes of
  2²² and up: debug builds panicked, release builds wrapped past the table-length guard and
  indexed out of bounds. One checked `photocraft_doc::adjust::lut3d_len` now backs the CPU
  compositor, the GPU planner and the PSD writer.
- Quick Selection: a stroke point far off the canvas (`select.quick {"points":[[3e9,0]]}`)
  saturated to i32::MAX and then overflowed on `+ 1`.
- Link Layers: `max + 1` over stored link group ids overflowed at u64::MAX (debug panic, release
  wrap that could merge unrelated layers). A fresh id is now the smallest unused one when the
  largest is taken; a selection's first group is read with `first()`.
- Vanishing Point clone footprint: the same `as i32 + 1` saturates (hardening; no reproducer).

crates/algo/src/segment/quick.rs

@@ -51,7 +51,7 @@ pub fn quick_select(sampler: &dyn Sampler, canvas: Rect, points: &[(f32, f32)],
         x1 = x1.max(x + r);
         y1 = y1.max(y + r);
     }
-    let bbox = Rect::new(x0.floor() as i32, y0.floor() as i32, x1.ceil() as i32 + 1, y1.ceil() as i32 + 1).intersect(&canvas);
+    let bbox = Rect::new(x0.floor() as i32, y0.floor() as i32, (x1.ceil() as i32).saturating_add(1), (y1.ceil() as i32).saturating_add(1)).intersect(&canvas);
     if bbox.is_empty() {
         return None;
     }

crates/algo/src/segment/tests.rs

@@ -304,6 +304,17 @@ fn two_regions(w: usize, h: usize) -> RgbImage {
     })
 }
 
+#[test]
+fn quick_select_with_points_far_off_canvas_selects_nothing() {
+    // A cast saturates to i32::MAX; adding one to it once overflowed.
+    let img = two_regions(140, 100);
+    let s = ImageSampler { img: &img, origin: (0, 0) };
+    let canvas = Rect::new(0, 0, 140, 100);
+    for p in [(3e9, 0.0), (0.0, 3e9), (f32::MAX, f32::MAX)] {
+        assert!(quick::quick_select(&s, canvas, &[p], 8.0, quick::WORK_PX).is_none(), "{p:?}");
+    }
+}
+
 #[test]
 fn quick_select_grows_without_leaking() {
     let img = two_regions(140, 100);

crates/algo/src/vanishing.rs

@@ -289,7 +289,7 @@ pub fn clone_stroke(scene: &Scene, surf: &mut Surface, source: [f64; 2], points:
             let (x, y) = hd.apply(u, v);
             bb = [bb[0].min(x), bb[1].min(y), bb[2].max(x), bb[3].max(y)];
         }
-        let area = Rect::new(bb[0].floor() as i32, bb[1].floor() as i32, bb[2].ceil() as i32 + 1, bb[3].ceil() as i32 + 1);
+        let area = Rect::new(bb[0].floor() as i32, bb[1].floor() as i32, (bb[2].ceil() as i32).saturating_add(1), (bb[3].ceil() as i32).saturating_add(1));
         if area.is_empty() || area.width() > 4096 || area.height() > 4096 {
             continue;
         }

crates/compose/src/adjust.rs

@@ -6,7 +6,7 @@
 use photocraft_color::SampleType;
 use photocraft_color::convert::{D50, SRGB_TO_XYZ_D50, XYZ_D50_TO_SRGB, mat3_mul, rgb_to_gray, srgb_to_linear};
 use photocraft_doc::Adjustment;
-use photocraft_doc::adjust::{CurvePoint, HueRange, LevelsChannel, ToneSpace};
+use photocraft_doc::adjust::{CurvePoint, HueRange, LevelsChannel, ToneSpace, lut3d_len};
 
 use crate::Buffer;
 
@@ -203,7 +203,7 @@ pub fn apply_depth(adj: &Adjustment, buf: &mut Buffer, transfer: Transfer, depth
             }
         }),
         Adjustment::SelectiveColor { relative, adjustments } => map_rgb(buf, |c| selective_color(c, *relative, adjustments)),
-        Adjustment::ColorLookup { lut: Some(table), size, tetrahedral, dither, .. } if *size >= 2 && table.len() >= (*size as usize).pow(3) * 3 => {
+        Adjustment::ColorLookup { lut: Some(table), size, tetrahedral, dither, .. } if lut3d_len(*size).is_some_and(|len| table.len() >= len) => {
             let (n, w, x0, y0) = (*size as usize, buf.rect.width().max(1) as usize, buf.rect.x0, buf.rect.y0);
             for (i, p) in buf.px.iter_mut().enumerate() {
                 if p[3] <= 0.0 {
@@ -740,6 +740,18 @@ mod tone_tests {
         Buffer { rect, px }
     }
 
+    #[test]
+    fn color_lookup_with_an_overflowing_stored_size_is_the_identity() {
+        // A stored document's size is untrusted: (2²²)³ × 3 overflows, which once wrapped past
+        // the table-length guard and indexed out of bounds.
+        for size in [1 << 22, u32::MAX] {
+            let a = Adjustment::ColorLookup { name: "x".into(), lut: Some(std::sync::Arc::new(vec![0.5; 24])), size, tetrahedral: false, dither: false };
+            let mut b = ramp();
+            apply(&a, &mut b);
+            assert_eq!(b.px, ramp().px, "size {size}");
+        }
+    }
+
     #[test]
     fn identity_tone_in_every_space_is_identity() {
         for space in [ToneSpace::Rgb, ToneSpace::Cmyk, ToneSpace::Lab] {

crates/doc/src/adjust.rs

@@ -17,6 +17,15 @@ pub fn is_identity_curve(points: &[CurvePoint]) -> bool {
     }
 }
 
+/// Number of values in the table of a `size`³ Color Lookup (`size³ × 3`). `None` for a size
+/// below 2 or one whose table length overflows: a stored document's size is untrusted.
+pub fn lut3d_len(size: u32) -> Option<usize> {
+    if size < 2 {
+        return None;
+    }
+    usize::try_from(size).ok()?.checked_pow(3)?.checked_mul(3)
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct LevelsChannel {
     pub in_black: f32,
@@ -293,4 +302,14 @@ mod tests {
         let g: Adjustment = serde_json::from_str(r#"{"GradientMap":{"stops":[],"reverse":true}}"#).unwrap();
         assert!(matches!(g, Adjustment::GradientMap { dither: false, .. }));
     }
+
+    #[test]
+    fn lut3d_len_rejects_tiny_and_overflowing_sizes() {
+        assert_eq!(lut3d_len(0), None);
+        assert_eq!(lut3d_len(1), None);
+        assert_eq!(lut3d_len(2), Some(24));
+        assert_eq!(lut3d_len(33), Some(33 * 33 * 33 * 3));
+        assert_eq!(lut3d_len(1 << 22), None, "(2²²)³ overflows a 64-bit length");
+        assert_eq!(lut3d_len(u32::MAX), None);
+    }
 }

crates/engine/src/layer_multi_cmds.rs

@@ -535,8 +535,12 @@ fn link_layers(s: &mut Session) -> Result<Value> {
     let linked = s.edit("Link Layers", |doc, _| {
         let groups: Vec<Option<u64>> = sel.iter().map(|id| doc.layer(*id).and_then(|l| l.link_group)).collect();
         // Already one link group (or a single linked layer): unlink, like Photoshop's toggle.
-        let unlink = groups[0].is_some() && groups.iter().all(|g| *g == groups[0]);
-        let next = doc.walk().iter().filter_map(|(_, _, l)| l.link_group).max().unwrap_or(0) + 1;
+        let first = groups.first().copied().flatten();
+        let unlink = first.is_some() && groups.iter().all(|g| *g == first);
+        // A fresh id: one past the largest, or (a stored document already uses u64::MAX) the
+        // smallest unused one, never an existing group that would join unrelated layers.
+        let used: std::collections::HashSet<u64> = doc.walk().iter().filter_map(|(_, _, l)| l.link_group).collect();
+        let next = used.iter().max().map_or(Some(1), |m| m.checked_add(1)).or_else(|| (1..=u64::MAX).find(|g| !used.contains(g))).unwrap_or(1);
         for id in &sel {
             let l = doc.layer_mut(*id).ok_or(EngineError::NoLayer(*id))?;
             l.link_group = if unlink { None } else { Some(next) };
@@ -1290,6 +1294,28 @@ mod tests {
         s.execute("layer.translate", json!({"dx": -200_000, "dy": 200_000})).unwrap();
     }
 
+    #[test]
+    fn link_layers_never_reuses_a_group_when_the_largest_id_is_u64_max() {
+        let mut s = session(8);
+        let a = rect_layer(&mut s, Rect::new(0, 0, 10, 10));
+        let b = rect_layer(&mut s, Rect::new(20, 20, 30, 30));
+        let c = rect_layer(&mut s, Rect::new(50, 50, 60, 60));
+        let d = rect_layer(&mut s, Rect::new(70, 70, 80, 80));
+        // A stored document can carry any id; `max + 1` once overflowed here.
+        let st = s.active_mut().unwrap();
+        for id in [c, d] {
+            std::sync::Arc::make_mut(&mut st.doc).layer_mut(id).unwrap().link_group = Some(u64::MAX);
+        }
+        select_all(&mut s, &[a, b]);
+        assert_eq!(s.execute("layer.linkLayers", json!({})).unwrap()["linked"], true);
+        let g = doc(&s).layer(a).unwrap().link_group;
+        assert_eq!(g, Some(1));
+        assert_eq!(doc(&s).layer(b).unwrap().link_group, g);
+        for id in [c, d] {
+            assert_eq!(doc(&s).layer(id).unwrap().link_group, Some(u64::MAX), "the existing group is untouched");
+        }
+    }
+
     #[test]
     fn linked_layers_move_together_and_undo() {
         for depth in [8, 16, 32] {

crates/gpu/src/plan.rs

@@ -15,7 +15,7 @@
 use photocraft_color::BlendMode;
 use photocraft_compose::adjust::{self, Transfer};
 use photocraft_compose::effects::has_effects;
-use photocraft_doc::adjust::ToneSpace;
+use photocraft_doc::adjust::{ToneSpace, lut3d_len};
 use photocraft_doc::{Adjustment, Document, Effect, Fill, FxPaint, GlobalLight, Glow, Gradient, Layer, LayerContent, LayerId, Pattern, StrokePosition};
 use photocraft_geom::Rect;
 use photocraft_raster::Surface;
@@ -1537,11 +1537,12 @@ pub fn adjustment_program(adj: &Adjustment, transfer: Transfer, depth: photocraf
             }
             (15, p, Some(vec![row]))
         }
-        Adjustment::ColorLookup { lut: Some(table), size, tetrahedral, dither, .. } if *size >= 2 && table.len() >= (*size as usize).pow(3) * 3 => {
+        Adjustment::ColorLookup { lut: Some(table), size, tetrahedral, dither, .. } if lut3d_len(*size).is_some_and(|len| table.len() >= len) => {
             // The flattened table (n³ RGB triplets) spans as many 4096-wide rows as it needs.
             let n = *size as usize;
-            let len = n * n * n * 3;
-            let rows = table[..len]
+            let rows = lut3d_len(*size)
+                .and_then(|len| table.get(..len))
+                .unwrap_or_default()
                 .chunks(4096)
                 .map(|c| {
                     let mut row = [0.0f32; 4096];
@@ -1609,6 +1610,15 @@ mod tests {
     use photocraft_color::{Color, ColorMode, SampleType};
     use photocraft_geom::Size;
 
+    #[test]
+    fn color_lookup_with_an_overflowing_stored_size_uploads_no_table() {
+        for size in [1 << 22, u32::MAX] {
+            let a = Adjustment::ColorLookup { name: "x".into(), lut: Some(std::sync::Arc::new(vec![0.5; 24])), size, tetrahedral: false, dither: false };
+            let (kind, _, rows) = adjustment_program(&a, Transfer::Srgb, SampleType::U8);
+            assert_eq!((kind, rows.is_none()), (0, true), "size {size}");
+        }
+    }
+
     #[test]
     fn slots_are_recycled() {
         let mut d = Document::with_background("t", Size::new(8, 8), ColorMode::Rgb, SampleType::U8, Color::WHITE);

crates/io/src/adjust_map.rs

@@ -605,10 +605,9 @@ pub fn write(adj: &Adjustment) -> Vec<([u8; 4], Vec<u8>)> {
             // A lookup without a usable table (none chosen yet, or a bad size) renders as the
             // identity, so it is written as a 2³ identity cube rather than dropped.
             let n = *size as usize;
-            let (file, dither) = match lut {
-                Some(table) if (2..=256).contains(&n) && table.len() >= n * n * n * 3 => {
-                    (photocraft_cms::lutfile::LutFile { title: String::new(), size: n, data: table[..n * n * n * 3].to_vec() }, *dither)
-                }
+            let cube = lut.as_deref().zip(photocraft_doc::adjust::lut3d_len(*size)).and_then(|(table, len)| table.get(..len)).filter(|_| n <= 256);
+            let (file, dither) = match cube {
+                Some(cube) => (photocraft_cms::lutfile::LutFile { title: String::new(), size: n, data: cube.to_vec() }, *dither),
                 _ => (photocraft_cms::lutfile::LutFile { title: String::new(), ..photocraft_cms::lutfile::LutFile::identity(2) }, false),
             };
             let en =