M12: SAM 3 Object and Describe masks (pure Rust, Metal)

AI masks from Meta's SAM 3, run in LightCraft itself: a new
`lightcraft-segment` crate ports the Hugging Face transformers
implementation (Apache-2.0) to candle, on the GPU through Metal on macOS
(CPU elsewhere), with no C/C++ and no `unsafe`.

- Object: click to include, ⌥-click to leave a part out (SAM 3's
  interactive prompt encoder + two-way mask decoder).
- Describe: a text prompt selects every instance ("sky", "the red car";
  "car, road" merges several) via the CLIP text encoder and DETR detector.
- Both as new masks and as Add / Subtract / Intersect components; + / −
  buttons under the selected mask combine selections.
- The segmentation is stored with the mask (288² quantized logits, ~10-30
  KB), so renders, exports and the web build never need the model.
- A zoomed-in detail pass (background, ~5 s on an M4 Pro) re-segments the
  area around a selection from a sharper source: 5-10x finer edges on
  small objects.
- Per-selection Edge slider (hard ↔ soft); hovering a mask in the list
  shows it in red (the always-on overlay is now off by default, O toggles).
- The engine feature `sam` (on in the desktop app) gates the model; the
  image is encoded once per photo in the background, clicks take ~35 ms.
- Weights are not bundled (SAM License): tools/install-sam3.sh / .ps1
  download and verify facebook/sam3; docs/ai-masks.md.
- Clearer errors (missing file, missing model) and a stderr logger in the
  app (LIGHTCRAFT_LOG=info).

candle is pinned at 0.9.2: later releases make candle-core depend on
tokenizers with the Oniguruma C library. Every stage is checked against
the Python reference (tests/reference.rs, tools/sam3_reference.py): on
Metal the backbone matches to 7e-5 relative error, masks to ~5e-5.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01C8BkS7JDqa1EMcjW6spDGA

crates/develop/Cargo.toml

@@ -13,6 +13,7 @@ lightcraft-geom = { workspace = true }
 lightcraft-color = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+miniz_oxide = { workspace = true }
 
 [dev-dependencies]
 proptest = { workspace = true }

crates/develop/src/lib.rs

@@ -8,10 +8,12 @@
 
 pub mod controls;
 pub mod presets;
+pub mod segmask;
 pub mod settings;
 
 pub use controls::{CONTROLS, ControlSpec, Section, Track};
 pub use presets::{Preset, SettingsGroup, apply_partial, extract_groups};
+pub use segmask::SegMask;
 pub use settings::*;
 
 use serde_json::Value;

crates/develop/src/segmask.rs

@@ -0,0 +1,131 @@
+//! Model segmentations stored with a mask ([`SegMask`]), so rendering never needs the model.
+
+use serde::{Deserialize, Serialize};
+
+/// Logits are stored in steps of 1/`SCALE`, clamped to ±127/`SCALE` (≈ ±15.9: a sigmoid of
+/// 1 − 1.2e-7, i.e. fully in or out).
+const SCALE: f32 = 8.0;
+/// Largest grid a stored segmentation may claim (hostile input caps the allocation).
+pub const MAX_SIDE: usize = 1024;
+
+/// A segmentation computed by a model (SAM 3 in `lightcraft-segment`): `side × side` logits
+/// over the uncropped, oriented image stretched to a square (row `y`, column `x` covers
+/// normalized `((x + 0.5) / side, (y + 0.5) / side)`), quantized to `i8`, deflated and base64
+/// encoded. Coverage at a point is the sigmoid of the bilinearly sampled logit, so edges stay
+/// smooth at any output size.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct SegMask {
+    pub side: u32,
+    pub data: String,
+    /// The part of the image the grid covers, normalized `[x0, y0, x1, y1]` (`None` = all of
+    /// it). A zoomed-in pass over one object covers just its surroundings, at a higher
+    /// resolution than a pass over the whole photo.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rect: Option<[f64; 4]>,
+}
+
+impl SegMask {
+    /// From `side × side` logits over the whole image.
+    pub fn from_logits(side: usize, logits: &[f32]) -> SegMask {
+        let q: Vec<u8> = logits.iter().take(side * side).map(|l| ((l * SCALE).round().clamp(-127.0, 127.0) as i8) as u8).collect();
+        let z = miniz_oxide::deflate::compress_to_vec(&q, 8);
+        SegMask { side: side as u32, data: base64_encode(&z), rect: None }
+    }
+
+    /// From `side × side` logits over the part `rect` of the image.
+    pub fn from_logits_in(side: usize, logits: &[f32], rect: [f64; 4]) -> SegMask {
+        SegMask { rect: Some(rect), ..SegMask::from_logits(side, logits) }
+    }
+
+    /// The covered part, normalized (the whole image without a `rect`); `None` when damaged.
+    pub fn bounds(&self) -> Option<[f64; 4]> {
+        let r = self.rect.unwrap_or([0.0, 0.0, 1.0, 1.0]);
+        (r.iter().all(|v| v.is_finite()) && r[2] > r[0] && r[3] > r[1]).then_some(r)
+    }
+
+    /// The logits, or `None` when the data is damaged or claims an absurd size.
+    pub fn logits(&self) -> Option<Vec<f32>> {
+        let side = self.side as usize;
+        if side == 0 || side > MAX_SIDE {
+            return None;
+        }
+        let z = base64_decode(&self.data)?;
+        let q = miniz_oxide::inflate::decompress_to_vec_with_limit(&z, side * side).ok()?;
+        (q.len() == side * side).then(|| q.iter().map(|b| f32::from(*b as i8) / SCALE).collect())
+    }
+}
+
+const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+fn base64_encode(bytes: &[u8]) -> String {
+    let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
+    for chunk in bytes.chunks(3) {
+        let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
+        let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
+        for i in 0..4 {
+            if i <= chunk.len() {
+                out.push(char::from(B64[((n >> (18 - 6 * i)) & 63) as usize]));
+            } else {
+                out.push('=');
+            }
+        }
+    }
+    out
+}
+
+fn base64_decode(s: &str) -> Option<Vec<u8>> {
+    let mut out = Vec::with_capacity(s.len() / 4 * 3);
+    let (mut acc, mut bits) = (0u32, 0u32);
+    for c in s.bytes().filter(|c| !c.is_ascii_whitespace()) {
+        if c == b'=' {
+            break;
+        }
+        let v = B64.iter().position(|b| *b == c)? as u32;
+        acc = (acc << 6) | v;
+        bits += 6;
+        if bits >= 8 {
+            bits -= 8;
+            out.push((acc >> bits) as u8);
+            acc &= (1 << bits) - 1;
+        }
+    }
+    Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn base64_round_trips() {
+        for n in 0..10 {
+            let v: Vec<u8> = (0..n).map(|i| (i * 37 + 11) as u8).collect();
+            assert_eq!(base64_decode(&base64_encode(&v)).unwrap(), v);
+        }
+        assert_eq!(base64_encode(b"Man"), "TWFu");
+        assert_eq!(base64_encode(b"Ma"), "TWE=");
+        assert!(base64_decode("TW!u").is_none());
+    }
+
+    #[test]
+    fn logits_round_trip_quantized() {
+        let side = 16;
+        let l: Vec<f32> = (0..side * side).map(|i| (i as f32 - 128.0) / 4.0).collect();
+        let m = SegMask::from_logits(side, &l);
+        let back = m.logits().unwrap();
+        for (a, b) in l.iter().zip(&back) {
+            assert!((a.clamp(-127.0 / SCALE, 127.0 / SCALE) - b).abs() <= 0.5 / SCALE + 1e-6);
+        }
+    }
+
+    #[test]
+    fn damaged_data_is_none() {
+        let mut m = SegMask::from_logits(8, &[1.0; 64]);
+        m.side = 9;
+        assert!(m.logits().is_none(), "size mismatch");
+        m.side = 100_000;
+        assert!(m.logits().is_none(), "absurd size");
+        assert!(SegMask { side: 8, data: "not base64 !".into(), rect: None }.logits().is_none());
+        assert!(SegMask { rect: Some([0.5, 0.0, 0.2, 1.0]), ..SegMask::from_logits(2, &[0.0; 4]) }.bounds().is_none());
+    }
+}

crates/develop/src/settings.rs

@@ -772,8 +772,33 @@ pub enum MaskShape {
     Subject,
     Sky,
     Background,
+    /// One object picked by clicks (SAM 3 point prompts): `hint` holds the clicks that include,
+    /// `exclude` the ones that exclude; `seg` the segmentation computed from them.
     Object {
         hint: Vec<Point>,
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        exclude: Vec<Point>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        seg: Option<crate::SegMask>,
+        /// Zoomed-in passes over parts of the image (higher resolution than `seg`), used
+        /// inside their rectangles.
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        detail: Vec<crate::SegMask>,
+        /// Edge −100..100: below 0 harder (a steeper transition), above 0 softer (feathered).
+        #[serde(default, skip_serializing_if = "is_zero")]
+        edge: f64,
+    },
+    /// Everything a description names ("sky", "the red car": SAM 3 concept prompts); `seg` is
+    /// the segmentation computed for it.
+    Prompt {
+        text: String,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        seg: Option<crate::SegMask>,
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        detail: Vec<crate::SegMask>,
+        /// Edge −100..100, as for `Object`.
+        #[serde(default, skip_serializing_if = "is_zero")]
+        edge: f64,
     },
     People {
         person: u32,