Masks: an open mask leaves its layer whole until it is closed (#290)

While a mask was being drawn with the Pen, its open path was filled as
if closed, so the layer went black except inside the points placed so
far, which made rotoscoping by eye impossible. As in After Effects, an
open mask makes no transparency: the layer shows whole until the path
is closed. Effects that take a mask as a path (Stroke and the others)
still get open masks.

Test: open_masks_make_no_transparency (render).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/render/src/masks.rs

@@ -134,7 +134,10 @@ pub fn shapes(ctx: &EvalCtx, layer: &Layer) -> Vec<effectcraft_effects::MaskShap
 /// Returns whether any mask was applied.
 pub fn apply(ctx: &EvalCtx, layer: &Layer, buf: &mut Buf, blur: MaskBlur) -> bool {
     let Some(masks) = layer.masks() else { return false };
-    let list: Vec<_> = masks.groups().filter(|g| g.enabled && matches!(g.kind, GroupKind::Mask { mode, .. } if mode != MaskMode::None)).collect();
+    // An open mask (a path the Pen is still drawing) makes no transparency: the layer shows
+    // whole until the path is closed (After Effects; effects still use it as a path).
+    let closed = |g: &effectcraft_project::PropGroup| matches!(ctx.group_value(layer, g, "path"), Some(Value::Path(sp)) if sp.closed);
+    let list: Vec<_> = masks.groups().filter(|g| g.enabled && matches!(g.kind, GroupKind::Mask { mode, .. } if mode != MaskMode::None) && closed(g)).collect();
     if list.is_empty() {
         return false;
     }

crates/render/src/tests.rs

@@ -368,6 +368,26 @@ fn effects_see_layer_masks() {
     assert!(img.get(150, 50)[3] > 0.99, "mode None mask does not cut the layer");
 }
 
+/// An open mask (the Pen still drawing it) leaves the layer whole; closed, it masks (#290).
+#[test]
+fn open_masks_make_no_transparency() {
+    let render = |closed: bool| {
+        let (mut p, cid, comp) = setup();
+        let mut l = solid(&mut p, &comp, [1.0, 0.0, 0.0], 200, 100);
+        let path = ShapePath { closed, ..ShapePath::rect([50.0, 50.0], 40.0, 40.0) };
+        let mut next = p.next_id;
+        let m = build::mask(&mut Ids(&mut next), "Mask 1", path, MaskMode::Add, [255, 255, 0]);
+        p.next_id = next;
+        l.props.sub_mut("masks").unwrap().children.push(m.into());
+        p.comp_mut(cid).unwrap().layers.push(l);
+        render_frame(&p, cid, Tick::ZERO, 1.0)
+    };
+    let open = render(false);
+    assert!(open.get(50, 50)[3] > 0.99 && open.get(150, 50)[3] > 0.99, "the open mask cut the layer");
+    let closed = render(true);
+    assert!(closed.get(50, 50)[3] > 0.99 && closed.get(150, 50)[3] < 0.01, "the closed mask masks");
+}
+
 #[test]
 fn effects_read_layer_params() {
     let (mut p, cid, comp) = setup();