#783: Object Matte passes its box to the model as a box prompt

Problem: with a trained model installed, Object Matte's optional box reached
MobileSAM only as background points along the box's edges (an approximation),
although the model's prompt encoder takes a box prompt directly.

Fix: a new Roto Brush stroke kind, Box (two opposite corners), that
stroke_prompt turns into the model prompt's bbox (scaled to the matte grid,
corners in any order, degenerate or non-finite boxes ignored). The graph cut
never rasterizes it. Object Matte emits it instead of the edge strokes when a
model segments; the classical fallback keeps the edge strokes. A prior matte's
box no longer replaces a box the user drew on that frame.

Tests: effects object_matte::the_box_reaches_the_model_as_a_box_prompt (a
stand-in model sees the box, at full and half resolution, and no edge points);
track roto::tests::box_stroke_is_the_model_box_prompt.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/effects/src/object_matte.rs

@@ -5,8 +5,9 @@
 //! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the
 //! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the
 //! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is
-//! installed, the classical graph cut otherwise (the point is a foreground stroke, the box's
-//! edges, or without a box the layer's border, background strokes). With **Track Object** the
+//! installed, the classical graph cut otherwise. The model gets the point as a foreground point
+//! and the box as its box prompt; the graph cut gets the point as a foreground stroke and the
+//! box's edges (without a box, the layer's border) as background strokes. With **Track Object** the
 //! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical
 //! flow, re-segmented by the model prompted with the carried matte); without it every frame is
 //! segmented on its own from the prompt (useful when the point is animated).
@@ -119,9 +120,10 @@ pub fn tracks(params: &Params) -> bool {
 }
 
 /// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground
-/// stroke; the box's edges as background strokes; without a box, the layer's border when there
-/// is no model (the graph cut needs background samples; a model does not, and a border stroke
-/// would cut objects touching the frame edge). `None` when the point is unusable.
+/// stroke; the box as a box stroke (the model's box prompt) with a model, its edges as
+/// background strokes for the graph cut without one; without a box, the layer's border when
+/// there is no model (the graph cut needs background samples; a model does not, and a border
+/// stroke would cut objects touching the frame edge). `None` when the point is unusable.
 pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option<Vec<Stroke>> {
     let [w, h] = layer_size;
     if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) {
@@ -144,7 +146,12 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_mo
             if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 {
                 return None;
             }
-            out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) });
+            out.push(if with_model {
+                // The model takes the box itself as its box prompt.
+                Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]] }
+            } else {
+                Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }
+            });
         }
         None if !with_model => {
             let i = EDGE_RADIUS;
@@ -438,6 +445,8 @@ mod tests {
     #[derive(Default)]
     struct Stub {
         calls: std::sync::atomic::AtomicUsize,
+        /// The box prompt of the last call.
+        bbox: std::sync::Mutex<Option<[f32; 4]>>,
     }
     impl MaskModel for Stub {
         fn info(&self) -> &'static effectcraft_segment::ModelInfo {
@@ -445,6 +454,7 @@ mod tests {
         }
         fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {
             self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            *self.bbox.lock().unwrap() = prompt.bbox;
             let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) };
             Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect())
         }
@@ -464,5 +474,39 @@ mod tests {
         let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") };
         assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5);
         assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1);
+        assert_eq!(*stub.bbox.lock().unwrap(), None);
+    }
+
+    #[test]
+    fn the_box_reaches_the_model_as_a_box_prompt() {
+        let p = params(&[
+            ("objectPoint", Value::Vec2([90.0, 70.0])),
+            ("useBox", Value::Bool(true)),
+            ("boxTopLeft", Value::Vec2([100.0, 80.0])),
+            ("boxBottomRight", Value::Vec2([74.0, 54.0])),
+        ]);
+        // With a model: the point and a box stroke (corners ordered); without: the box's edges
+        // as a background stroke, as before.
+        let with = prompt_strokes(&p, [100.0, 80.0], 0, true).unwrap();
+        assert_eq!(with.iter().map(|s| s.kind).collect::<Vec<_>>(), [StrokeKind::Fg, StrokeKind::Box]);
+        assert_eq!(with[1].points, [[74.0, 54.0], [100.0, 80.0]]);
+        let without = prompt_strokes(&p, [100.0, 80.0], 0, false).unwrap();
+        assert_eq!(without.iter().map(|s| s.kind).collect::<Vec<_>>(), [StrokeKind::Fg, StrokeKind::Bg]);
+        // The model is called with the box itself (matte pixels at scale 1) and only one
+        // foreground point: no background points along the edges.
+        let stub = Arc::new(Stub::default());
+        let img = frame_at(0);
+        let mut frame = |g: i64| Some(Arc::new(frame_at(g)));
+        let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") };
+        assert_eq!(*stub.bbox.lock().unwrap(), Some([74.0, 54.0, 100.0, 80.0]));
+        assert!(a[70 * 100 + 90] > 0.5 && a[40 * 100 + 40] < 0.5);
+        // At half resolution the box is scaled with the frame (another Input Key, so the
+        // full-resolution result above is not reused).
+        let mut p = p;
+        p.values.insert(INPUT_KEY.into(), Value::Str("half".into()));
+        let half = effectcraft_raster::resample(&img, 50, 40);
+        let mut frame = |g: i64| Some(Arc::new(effectcraft_raster::resample(&frame_at(g), 50, 40)));
+        let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false);
+        assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0]));
     }
 }

crates/track/src/roto/mod.rs

@@ -59,6 +59,9 @@ pub enum StrokeKind {
     Refine,
     /// Refine Edge tool with Alt: remove from the band.
     RefineErase,
+    /// A box around the object (Object Matte): two opposite corners in `points`. Trained models
+    /// get it as their box prompt; the graph cut ignores it.
+    Box,
 }
 
 impl StrokeKind {
@@ -830,8 +833,9 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> {
     out
 }
 
-/// A model's prompt from the strokes: points along the foreground and background strokes.
-fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
+/// A model's prompt from the strokes: points along the foreground and background strokes, and
+/// the box of a [`StrokeKind::Box`] stroke (matte pixels).
+pub fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
     let mut points = vec![];
     for (kind, fg) in [(StrokeKind::Fg, true), (StrokeKind::Bg, false)] {
         let ss: Vec<&&Stroke> = strokes.iter().filter(|s| s.kind == kind).collect();
@@ -840,11 +844,20 @@ fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
             points.extend(points_along(s, scale, per).into_iter().map(|p| (p, fg)));
         }
     }
-    Prompt { points, ..Default::default() }
+    let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale));
+    Prompt { points, bbox, ..Default::default() }
 }
 
-/// Add a prior matte to a prompt: its box (a little larger), up to four points spread through
-/// its core, and the matte itself.
+/// The box `[x0, y0, x1, y1]` (matte pixels) of a box stroke: its first two points as opposite
+/// corners; `None` when they are missing, not finite or the box is empty.
+fn box_of(s: &Stroke, scale: f64) -> Option<[f32; 4]> {
+    let (a, c) = (s.points.first()?, s.points.get(1)?);
+    let v = [a[0].min(c[0]), a[1].min(c[1]), a[0].max(c[0]), a[1].max(c[1])].map(|v| (v * scale) as f32);
+    (v.iter().all(|x| x.is_finite()) && v[2] > v[0] && v[3] > v[1]).then_some(v)
+}
+
+/// Add a prior matte to a prompt: its box (a little larger; unless the prompt has one), up to
+/// four points spread through its core, and the matte itself.
 fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bool {
     let bin: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();
     let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
@@ -858,7 +871,10 @@ fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bo
         return false;
     }
     let (mx, my) = (((x1 - x0) as f32 * 0.05).max(4.0), ((y1 - y0) as f32 * 0.05).max(4.0));
-    prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]);
+    // A box the user drew on this frame wins over the prior's.
+    if prompt.bbox.is_none() {
+        prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]);
+    }
     // Farthest-point picks among the core (deep inside), starting from the deepest pixel.
     let sd = matting::signed_distance(&bin, w, h);
     let deep = sd.iter().copied().fold(0.0f32, f32::max);

crates/track/src/roto/tests.rs

@@ -428,3 +428,25 @@ fn mobilesam_segments_and_propagates_the_disk() {
     eprintln!("MobileSAM: base IoU {jb:.4}, worst propagated {worst:.4} (classic {cworst:.4}), {:.0} ms/frame", t.elapsed().as_secs_f64() * 1000.0 / 20.0);
     assert!(jb > 0.95 && worst > 0.93 && worst >= cworst - 0.01, "base {jb}, worst {worst} vs classic {cworst}");
 }
+
+/// A box stroke becomes the model's box prompt (scaled, corners in any order); the graph cut
+/// ignores it; a prior's box does not replace it.
+#[test]
+fn box_stroke_is_the_model_box_prompt() {
+    let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]] };
+    let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]] };
+    let p = stroke_prompt(&[&fg, &bx], 0.5);
+    assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0]));
+    assert_eq!(p.points, vec![([25.0, 20.0], true)]);
+    // Degenerate or non-finite boxes are no prompt.
+    for pts in [vec![[20.0, 10.0]], vec![[20.0, 10.0], [20.0, 50.0]], vec![[f64::NAN, 10.0], [80.0, 60.0]]] {
+        let b = Stroke { points: pts, ..bx.clone() };
+        assert_eq!(stroke_prompt(&[&fg, &b], 1.0).bbox, None);
+    }
+    assert!(!StrokeKind::Box.is_segmentation());
+    assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v));
+    let mut q = stroke_prompt(&[&fg, &bx], 1.0);
+    let soft: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect();
+    assert!(add_prior_prompt(&mut q, &soft, 100, 80));
+    assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0]));
+}