From fe6d7b0192675398b1eeca678041e89df08af258 Mon Sep 17 00:00:00 2001 From: Andrew Frahn Date: Sun, 11 Oct 2026 09:40:35 +1100 Subject: [PATCH 1/2] Content-aware Remove: inpaint crate (PatchMatch completion) and a Fill spot mode New `inpaint` crate (L1): multi-scale patch-based completion with a PatchMatch nearest-neighbour field, seeded by a membrane fill. Deterministic, no panics on hostile sizes, cancellable. Ported from PhotoCraft (MIT OR Apache-2.0). Remove tool: a fourth spot mode, `fill` ("Content-Aware"), rebuilds the spot from patches of its surroundings instead of one source; coarse-to-fine, so the structure is chosen at a scale set by the spot size and preview and export agree (tested). spot.add leaves its source unset. Sources: C. Barnes et al., PatchMatch (SIGGRAPH 2009); Y. Wexler, E. Shechtman, M. Irani, Space-Time Completion of Video (TPAMI 2007); P. Perez et al., Poisson Image Editing (SIGGRAPH 2003) for the membrane fill; code ported from PhotoCraft crates/algo/src/inpaint.rs + poisson.rs (a680ed22). Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 8 + Cargo.toml | 1 + crates/develop/src/settings.rs | 2 + crates/engine/src/cmd/masks.rs | 4 +- crates/engine/src/tests_spots.rs | 10 + crates/inpaint/Cargo.toml | 14 + crates/inpaint/src/lib.rs | 61 +++ crates/inpaint/src/membrane.rs | 197 ++++++++++ crates/inpaint/src/patchmatch.rs | 587 ++++++++++++++++++++++++++++ crates/pipeline/Cargo.toml | 1 + crates/pipeline/src/spots.rs | 100 ++++- crates/ui-egui/locales/de.json | 3 +- crates/ui-egui/locales/es.json | 3 +- crates/ui-egui/locales/fr.json | 3 +- crates/ui-egui/locales/ja.json | 3 +- crates/ui-egui/locales/pt-br.json | 3 +- crates/ui-egui/locales/ru.json | 3 +- crates/ui-egui/locales/uk.json | 3 +- crates/ui-egui/locales/zh-hans.json | 3 +- crates/ui-egui/locales/zh-hant.json | 3 +- crates/ui-egui/src/panels/detail.rs | 1 + crates/ui-egui/src/panels/right.rs | 3 +- xtask/src/layers.rs | 1 + 23 files changed, 1003 insertions(+), 14 deletions(-) create mode 100644 crates/inpaint/Cargo.toml create mode 100644 crates/inpaint/src/lib.rs create mode 100644 crates/inpaint/src/membrane.rs create mode 100644 crates/inpaint/src/patchmatch.rs diff --git a/Cargo.lock b/Cargo.lock index 700f386fb..40a9988fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3291,6 +3291,13 @@ dependencies = [ "rayon", ] +[[package]] +name = "lightcraft-inpaint" +version = "0.5.0" +dependencies = [ + "rayon", +] + [[package]] name = "lightcraft-mcp" version = "0.6.0" @@ -3338,6 +3345,7 @@ dependencies = [ "lightcraft-color", "lightcraft-develop", "lightcraft-geom", + "lightcraft-inpaint", "lightcraft-raster", "lightcraft-raw", "lightcraft-scenes", diff --git a/Cargo.toml b/Cargo.toml index b9a511e26..f5278446a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ lightcraft-raw = { path = "crates/raw" } lightcraft-codecs = { path = "crates/codecs" } lightcraft-meta = { path = "crates/meta" } lightcraft-develop = { path = "crates/develop" } +lightcraft-inpaint = { path = "crates/inpaint" } lightcraft-scenes = { path = "crates/scenes" } lightcraft-pipeline = { path = "crates/pipeline" } lightcraft-gpu = { path = "crates/gpu" } diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs index 54577bffc..8ff0c1316 100644 --- a/crates/develop/src/settings.rs +++ b/crates/develop/src/settings.rs @@ -963,6 +963,8 @@ pub enum SpotMode { Remove, Heal, Clone, + /// Content-aware: the spot is rebuilt from patches of its surroundings (no single source). + Fill, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs index 24a94cbc8..c463cdeee 100644 --- a/crates/engine/src/cmd/masks.rs +++ b/crates/engine/src/cmd/masks.rs @@ -695,7 +695,7 @@ pub fn specs() -> Vec { "Add Remove Spot", [], None, - "{mode?: remove|heal|clone, points: [[x,y],…], size?: fraction of long edge, feather?, opacity?, source?: [dx,dy] (default: the best match nearby)} — selects the new spot; returns {index}", + "{mode?: remove|heal|clone|fill (content-aware), points: [[x,y],…], size?: fraction of long edge, feather?, opacity?, source?: [dx,dy] (default: the best match nearby)} — selects the new spot; returns {index}", has_active, |s, p| { let mode: SpotMode = @@ -719,7 +719,7 @@ pub fn specs() -> Vec { }; let id = s.active().ok_or_else(|| bad("spot.add", "no active photo"))?; let mut dd = (*s.develop_of(id).unwrap_or_default()).clone(); - if spot.source_offset.is_none() { + if spot.source_offset.is_none() && spot.mode != SpotMode::Fill { // resolve the automatic source now: it gets a pin and stays put at every size spot.source_offset = pick_source(s, id, &dd, &spot, None); } diff --git a/crates/engine/src/tests_spots.rs b/crates/engine/src/tests_spots.rs index 16bfe52b9..25fd482b7 100644 --- a/crates/engine/src/tests_spots.rs +++ b/crates/engine/src/tests_spots.rs @@ -55,3 +55,13 @@ fn spots_select_move_edit_refresh_and_delete() { s.execute("library.next", &json!({})).unwrap(); assert_eq!(s.active_spot, None); } + +#[test] +fn a_content_aware_spot_has_no_source() { + let mut s = Session::with_demo(); + s.execute("spot.add", &json!({"mode": "fill", "points": [[0.4, 0.5]], "size": 0.03})).unwrap(); + let sp = &spots(&s)[0]; + assert_eq!((sp.mode, sp.source_offset), (lightcraft_develop::SpotMode::Fill, None)); + assert_eq!(serde_json::to_value(sp).unwrap()["mode"], "fill"); + assert!(s.execute("spot.update", &json!({"mode": "nonsense"})).is_err()); +} diff --git a/crates/inpaint/Cargo.toml b/crates/inpaint/Cargo.toml new file mode 100644 index 000000000..b2d111fba --- /dev/null +++ b/crates/inpaint/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "lightcraft-inpaint" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Content-aware hole filling: PatchMatch-based patch completion and membrane fill (L1). Ported from PhotoCraft" + +[lints] +workspace = true + +[dependencies] + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rayon = { workspace = true } diff --git a/crates/inpaint/src/lib.rs b/crates/inpaint/src/lib.rs new file mode 100644 index 000000000..fb76304af --- /dev/null +++ b/crates/inpaint/src/lib.rs @@ -0,0 +1,61 @@ +//! Content-aware hole filling (inpainting) for the Remove tool. +//! +//! * [`complete`]: multi-scale patch-based completion (Wexler, Shechtman, Irani 2007) with the +//! nearest-neighbour field found by PatchMatch (Barnes et al. 2009). See [`patchmatch`]. +//! * [`membrane_fill`]: smooth harmonic fill, used to seed the coarsest level and as a fallback. +//! +//! Buffers are interleaved `w × h × ch` floats in any colour model plus a `w × h` hole mask. +//! Everything is deterministic (fixed-seed RNG, results independent of the thread count) and never +//! panics on mismatched sizes: such calls return `None` or the input unchanged. +//! +//! Sources: C. Barnes, E. Shechtman, A. Finkelstein, D. B. Goldman, *PatchMatch: A Randomized +//! Correspondence Algorithm for Structural Image Editing*, SIGGRAPH 2009; Y. Wexler, E. Shechtman, +//! M. Irani, *Space-Time Completion of Video*, PAMI 2007. +//! +//! Ported from PhotoCraft (`crates/algo/src/inpaint.rs`, `poisson.rs`, MIT OR Apache-2.0, same +//! organisation; see `NOTICE`). + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod membrane; +pub mod patchmatch; + +pub use membrane::membrane_fill; +pub use patchmatch::{CompleteParams, complete, complete_with}; + +/// A cancelled [`complete_with`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Cancelled; + +impl std::fmt::Display for Cancelled { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("cancelled") + } +} + +impl std::error::Error for Cancelled {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hostile_sizes_do_not_panic() { + assert_eq!(complete(4, 4, 3, &[0.0; 5], &[true; 16], &CompleteParams::default()), None); + assert_eq!(complete(4, 4, 0, &[], &[true; 16], &CompleteParams::default()), None); + assert_eq!(complete(usize::MAX, 2, 3, &[0.0; 6], &[true; 2], &CompleteParams::default()), None); + assert_eq!(membrane_fill(4, 4, 1, &[0.5; 3], &[true; 16]), vec![0.5; 3]); + // A hole covering everything: nothing to copy from. + assert_eq!(complete(16, 16, 1, &[0.5; 256], &[true; 256], &CompleteParams::default()), None); + // A tiny image smaller than a patch. + let _ = complete(2, 2, 1, &[0.5; 4], &[true, false, false, false], &CompleteParams::default()); + } + + #[test] + fn cancellation_stops() { + let img = vec![0.5f32; 64 * 64]; + let hole: Vec = (0..64 * 64usize).map(|i| (i % 64).abs_diff(32) < 4 && (i / 64).abs_diff(32) < 4).collect(); + assert_eq!(complete_with(64, 64, 1, &img, &hole, &CompleteParams::default(), &|| true), Err(Cancelled)); + } +} diff --git a/crates/inpaint/src/membrane.rs b/crates/inpaint/src/membrane.rs new file mode 100644 index 000000000..1f259b02f --- /dev/null +++ b/crates/inpaint/src/membrane.rs @@ -0,0 +1,197 @@ +//! Membrane (harmonic) fill: `Δv = 0` inside a hole with the known pixels as Dirichlet boundary. +//! Used to initialise the coarsest level of [`crate::complete`] and as its fallback. +//! +//! Sources: the membrane interpolant of P. Pérez, M. Gangnet, A. Blake, *Poisson Image Editing*, +//! ACM SIGGRAPH 2003 (§3), solved by successive over-relaxation on the 4-connected grid, accelerated +//! cascadically (solve on a half-resolution grid, prolong bilinearly, a few SOR sweeps per level). +//! Ported from PhotoCraft's `algo::poisson` (MIT OR Apache-2.0). + +/// Convergence threshold (max per-sweep update) for the SOR sweeps. +const TOL: f32 = 2e-5; +/// SOR sweeps (cap) and relaxation factor on every level above the coarsest: tuned so a 96×72 disc +/// is within ~1e-3 of a fully converged Gauss–Seidel solve (below one 8-bit step). +const FINE_ITERS: usize = 30; +const FINE_OMEGA: f32 = 1.7; + +/// Solve `Δv = 0` on `unknown` pixels with Dirichlet values from the known pixels of `v` +/// (out-of-grid neighbours are ignored, i.e. a Neumann boundary). `v` holds the known values on +/// entry and the solution on exit. Cascadic multigrid + SOR. +pub(crate) fn solve_membrane(w: usize, h: usize, unknown: &[bool], v: &mut [f32]) { + if unknown.len() != w * h || v.len() != w * h { + return; + } + solve_rec(w, h, unknown, v, 0, &|| false); +} + +fn solve_rec(w: usize, h: usize, unknown: &[bool], v: &mut [f32], depth: usize, stop: &(dyn Fn() -> bool + Sync)) { + if stop() { + return; + } + let n_unknown = unknown.iter().filter(|u| **u).count(); + if n_unknown == 0 { + return; + } + let known: Vec = v.iter().zip(unknown).filter(|(_, u)| !**u).map(|(x, _)| *x).collect(); + if known.is_empty() { + return; // nothing to interpolate from + } + if w >= 8 && h >= 8 && n_unknown > 64 && depth < 16 { + // Restrict: a coarse pixel is known if any of its children is (value = mean of known children). + let (cw, chh) = (w.div_ceil(2), h.div_ceil(2)); + let mut cu = vec![true; cw * chh]; + let mut cv = vec![0.0f32; cw * chh]; + for cy in 0..chh { + for cx in 0..cw { + let (mut s, mut n) = (0.0f32, 0u32); + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (x, y) = (cx * 2 + dx, cy * 2 + dy); + if x < w && y < h && !unknown[y * w + x] { + s += v[y * w + x]; + n += 1; + } + } + if n > 0 { + cu[cy * cw + cx] = false; + cv[cy * cw + cx] = s / n as f32; + } + } + } + solve_rec(cw, chh, &cu, &mut cv, depth + 1, stop); + // Prolong: bilinear sample of the coarse solution as the initial guess. + for y in 0..h { + let fy = ((y as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (chh - 1) as f32); + let y0 = fy.floor() as usize; + let y1 = (y0 + 1).min(chh - 1); + let ty = fy - y0 as f32; + for x in 0..w { + if !unknown[y * w + x] { + continue; + } + let fx = ((x as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (cw - 1) as f32); + let x0 = fx.floor() as usize; + let x1 = (x0 + 1).min(cw - 1); + let tx = fx - x0 as f32; + let a = cv[y0 * cw + x0] + (cv[y0 * cw + x1] - cv[y0 * cw + x0]) * tx; + let b = cv[y1 * cw + x0] + (cv[y1 * cw + x1] - cv[y1 * cw + x0]) * tx; + v[y * w + x] = a + (b - a) * ty; + } + } + sor(w, h, unknown, v, FINE_ITERS, FINE_OMEGA, stop); + } else { + let mean = known.iter().sum::() / known.len() as f32; + for (x, u) in v.iter_mut().zip(unknown) { + if *u { + *x = mean; + } + } + sor(w, h, unknown, v, 2000, 1.85, stop); + } +} + +/// SOR sweeps over the unknown pixels until the largest update drops below [`TOL`]. +fn sor(w: usize, h: usize, unknown: &[bool], v: &mut [f32], max_iter: usize, omega: f32, stop: &(dyn Fn() -> bool + Sync)) { + // Precompute the unknown pixels and which neighbours exist. + let cells: Vec<(u32, u8)> = (0..w * h) + .filter(|&i| unknown[i]) + .map(|i| { + let (x, y) = (i % w, i / w); + let m = u8::from(x > 0) | (u8::from(x + 1 < w) << 1) | (u8::from(y > 0) << 2) | (u8::from(y + 1 < h) << 3); + (i as u32, m) + }) + .collect(); + for _ in 0..max_iter { + if stop() { + return; + } + let mut max_d = 0.0f32; + for &(i, m) in &cells { + let i = i as usize; + let (s, n) = if m == 0b1111 { + (v[i - 1] + v[i + 1] + v[i - w] + v[i + w], 4.0) + } else { + let mut s = 0.0; + let mut n = 0.0; + if m & 1 != 0 { + s += v[i - 1]; + n += 1.0; + } + if m & 2 != 0 { + s += v[i + 1]; + n += 1.0; + } + if m & 4 != 0 { + s += v[i - w]; + n += 1.0; + } + if m & 8 != 0 { + s += v[i + w]; + n += 1.0; + } + if n == 0.0 { + continue; + } + (s, n) + }; + let d = omega * (s / n - v[i]); + v[i] += d; + max_d = max_d.max(d.abs()); + } + if max_d < TOL { + break; + } + } +} + +/// Map `f` over the channels of an interleaved buffer as planar slices (parallel on native). +fn per_channel(w: usize, h: usize, ch: usize, f: impl Fn(usize) -> Vec + Sync + Send) -> Vec { + #[cfg(not(target_arch = "wasm32"))] + let planes: Vec> = if w * h >= 64 * 64 { + use rayon::prelude::*; + (0..ch).into_par_iter().map(&f).collect() + } else { + (0..ch).map(&f).collect() + }; + #[cfg(target_arch = "wasm32")] + let planes: Vec> = (0..ch).map(&f).collect(); + let mut out = vec![0.0f32; w * h * ch]; + for (c, p) in planes.iter().enumerate() { + for (i, v) in p.iter().enumerate() { + out[i * ch + c] = *v; + } + } + out +} + +/// Membrane fill (no source texture): harmonic interpolation of the known pixels into `hole`. +/// `img` is interleaved `w × h × ch`, `hole` has `w × h` entries; on a size mismatch `img` comes back +/// unchanged. +pub fn membrane_fill(w: usize, h: usize, ch: usize, img: &[f32], hole: &[bool]) -> Vec { + if Some(hole.len()) != w.checked_mul(h) || Some(img.len()) != hole.len().checked_mul(ch) { + return img.to_vec(); + } + per_channel(w, h, ch, |c| { + let mut v: Vec = (0..w * h).map(|i| if hole[i] { 0.0 } else { img[i * ch + c] }).collect(); + solve_membrane(w, h, hole, &mut v); + v + }) +} +#[cfg(test)] +mod tests { + use super::*; + + fn disc(w: usize, h: usize, cx: f32, cy: f32, r: f32) -> Vec { + (0..w * h).map(|i| ((i % w) as f32 - cx).hypot((i / w) as f32 - cy) < r).collect() + } + + #[test] + fn membrane_reproduces_linear_boundary() { + // A linear function is harmonic: the solve must reproduce it exactly inside the hole. + let (w, h) = (80, 60); + let unknown = disc(w, h, 40.0, 30.0, 20.0); + let truth: Vec = (0..w * h).map(|i| 0.2 + 0.005 * (i % w) as f32 + 0.003 * (i / w) as f32).collect(); + let mut v: Vec = truth.iter().zip(&unknown).map(|(t, u)| if *u { 0.0 } else { *t }).collect(); + solve_membrane(w, h, &unknown, &mut v); + let err = v.iter().zip(&truth).map(|(a, b)| (a - b).abs()).fold(0.0, f32::max); + assert!(err < 2e-3, "{err}"); + } +} diff --git a/crates/inpaint/src/patchmatch.rs b/crates/inpaint/src/patchmatch.rs new file mode 100644 index 000000000..ea69f38d8 --- /dev/null +++ b/crates/inpaint/src/patchmatch.rs @@ -0,0 +1,587 @@ +//! Hole filling for Spot Healing: patch-based completion, proximity match and texture synthesis. +//! +//! * [`complete`], content-aware fill: the multi-scale "space-time completion" EM scheme of +//! Y. Wexler, E. Shechtman, M. Irani, *Space-Time Completion of Video*, IEEE TPAMI 2007, with the +//! nearest-neighbour field computed by C. Barnes, E. Shechtman, A. Finkelstein, D. B. Goldman, +//! *PatchMatch: A Randomized Correspondence Algorithm for Structural Image Editing*, SIGGRAPH 2009 +//! (random init, propagation, exponentially shrinking random search). Each EM iteration finds, for +//! every patch overlapping the hole, its most similar fully-known patch, then re-estimates hole +//! pixels by a similarity-weighted vote of the overlapping matches. Coarse-to-fine over an image +//! pyramid (the coarsest level is initialised by membrane interpolation), so large holes stay cheap. +//! * [`best_offset`], proximity match: the single displacement whose surrounding ring best matches the +//! ring around the hole (sum of squared differences), i.e. an automatic Healing Brush source. +//! * [`synthesize`], create texture: patch-based texture synthesis in the style of A. Efros, +//! W. Freeman, *Image Quilting for Texture Synthesis and Transfer*, SIGGRAPH 2001 (blocks picked from +//! random candidates by overlap error, feathered overlap instead of a min-cut seam). +//! +//! Ported from PhotoCraft's `algo::inpaint` (MIT OR Apache-2.0); its proximity match and texture +//! synthesis are left out. +//! +//! All functions take interleaved `w × h × ch` normalised floats (any colour model/depth) and a +//! `w × h` hole mask, and return a full buffer equal to the input outside the hole. They are +//! deterministic (fixed-seed hash RNG, results independent of thread count). + +use crate::Cancelled; +use crate::membrane::membrane_fill; + +/// Below this many grid cells a level runs single-threaded (thread hand-off costs more than it saves). +#[cfg(not(target_arch = "wasm32"))] +const PAR_MIN: usize = 128 * 128; + +/// SplitMix64: tiny deterministic RNG. +#[derive(Clone)] +struct Rng(u64); +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed ^ 0x9E37_79B9_7F4A_7C15) + } + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + /// Uniform integer in `lo..=hi`. + fn range(&mut self, lo: i32, hi: i32) -> i32 { + if hi <= lo { + return lo; + } + lo + (self.next() % (hi - lo + 1) as u64) as i32 + } +} + +/// Parameters for [`complete`]. +#[derive(Clone, Debug, PartialEq)] +pub struct CompleteParams { + /// Patch radius (patches are `2r+1` square). + pub patch_radius: usize, + /// EM iterations at the coarsest level (finer levels use fewer, down to `min_em_iters`). + pub em_iters: usize, + pub min_em_iters: usize, + /// PatchMatch passes per EM iteration. + pub pm_iters: usize, + /// Levels where the hole is wider than this (in that level's pixels) only *refine* the upsampled + /// match field (one local PatchMatch pass and one vote), which keeps big holes tractable: the + /// structure is decided at coarse scale, fine levels just add detail. + pub refine_extent: usize, + pub seed: u64, +} + +impl Default for CompleteParams { + fn default() -> Self { + Self { patch_radius: 3, em_iters: 8, min_em_iters: 2, pm_iters: 2, refine_extent: 96, seed: 1 } + } +} + +/// One pyramid level. +#[derive(Clone)] +struct Level { + w: usize, + h: usize, + img: Vec, + hole: Vec, +} + +fn downsample(l: &Level, ch: usize) -> Level { + let (w, h) = (l.w.div_ceil(2), l.h.div_ceil(2)); + let mut img = vec![0.0f32; w * h * ch]; + let mut hole = vec![false; w * h]; + for y in 0..h { + for x in 0..w { + let mut n = 0.0f32; + let o = (y * w + x) * ch; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (sx, sy) = (x * 2 + dx, y * 2 + dy); + if sx >= l.w || sy >= l.h { + continue; + } + let si = sy * l.w + sx; + if l.hole[si] { + hole[y * w + x] = true; + } else { + for c in 0..ch { + img[o + c] += l.img[si * ch + c]; + } + n += 1.0; + } + } + if n > 0.0 { + for c in 0..ch { + img[o + c] /= n; + } + } + } + } + Level { w, h, img, hole } +} + +/// Summed-area table of a boolean grid (`(w+1) × (h+1)`). +fn integral(w: usize, h: usize, m: &[bool]) -> Vec { + let mut s = vec![0u32; (w + 1) * (h + 1)]; + for y in 0..h { + let mut row = 0u32; + for x in 0..w { + row += u32::from(m[y * w + x]); + s[(y + 1) * (w + 1) + x + 1] = s[y * (w + 1) + x + 1] + row; + } + } + s +} + +/// Count of set cells in the clipped window `[x0, x1) × [y0, y1)`. +fn window_count(s: &[u32], gw: usize, gh: usize, x0: i32, y0: i32, x1: i32, y1: i32) -> u32 { + let (x0, y0) = (x0.max(0) as usize, y0.max(0) as usize); + let (x1, y1) = ((x1.max(0) as usize).min(gw), (y1.max(0) as usize).min(gh)); + if x1 <= x0 || y1 <= y0 { + return 0; + } + let w1 = gw + 1; + s[y1 * w1 + x1] + s[y0 * w1 + x0] - s[y0 * w1 + x1] - s[y1 * w1 + x0] +} + +fn hole_bbox(w: usize, h: usize, hole: &[bool]) -> Option<(usize, usize, usize, usize)> { + let mut b: Option<(usize, usize, usize, usize)> = None; + for y in 0..h { + for x in 0..w { + if hole[y * w + x] { + b = Some(match b { + None => (x, y, x + 1, y + 1), + Some((x0, y0, x1, y1)) => (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1)), + }); + } + } + } + b +} + +/// Per-level PatchMatch/EM state. +struct Solver<'a> { + w: usize, + h: usize, + ch: usize, + r: i32, + img: &'a mut Vec, + hole: &'a [bool], + /// Valid source centres: the whole patch is inside the grid and contains no hole pixel. + valid: Vec, + valid_list: Vec<(i32, i32)>, + /// Target centres (patch overlaps the hole), row-major. + targets: Vec, + /// Cancellation: checked per row band; a cancelled solve's output is discarded. + ctl: &'a (dyn Fn() -> bool + Sync), +} + +impl Solver<'_> { + #[inline] + fn is_valid(&self, x: i32, y: i32) -> bool { + x >= 0 && y >= 0 && (x as usize) < self.w && (y as usize) < self.h && self.valid[y as usize * self.w + x as usize] + } + + /// SSD between the (clipped) target patch at `t` and the source patch at `s`, per sample. + fn dist(&self, t: (i32, i32), s: (i32, i32), cutoff: f32) -> f32 { + let (w, h, ch, r) = (self.w as i32, self.h as i32, self.ch, self.r); + let full = ((2 * r + 1) * (2 * r + 1)) as f32 * ch as f32; + let mut sum = 0.0f32; + let mut n = 0usize; + for dy in -r..=r { + let ty = t.1 + dy; + if ty < 0 || ty >= h { + continue; + } + let sy = s.1 + dy; + for dx in -r..=r { + let tx = t.0 + dx; + if tx < 0 || tx >= w { + continue; + } + let ti = (ty as usize * self.w + tx as usize) * ch; + let si = (sy as usize * self.w + (s.0 + dx) as usize) * ch; + for c in 0..ch { + let d = self.img[ti + c] - self.img[si + c]; + sum += d * d; + } + n += ch; + } + // Early out: the mean can only be ≥ sum / (full patch size). + if sum > cutoff * full { + return f32::INFINITY; + } + } + if n == 0 { f32::INFINITY } else { sum / n as f32 } + } + + /// One PatchMatch pass over all target rows, in parallel bands (each band propagates within itself). + fn patchmatch(&self, nnf: &mut [(i32, i32)], cost: &mut [f32], pass: usize, seed: u64, search_radius: i32) { + const BAND: usize = 16; + let w = self.w; + let reverse = pass % 2 == 1; + let band_fn = |bi: usize, nn: &mut [(i32, i32)], co: &mut [f32]| { + if (self.ctl)() { + return; + } + let mut rng = Rng::new(seed ^ (bi as u64).wrapping_mul(0x2545_F491_4F6C_DD1D) ^ ((pass as u64) << 40)); + let y0 = bi * BAND; + let rows = nn.len() / w; + let step: i32 = if reverse { -1 } else { 1 }; + for ry in 0..rows { + let ly = if reverse { rows - 1 - ry } else { ry }; + let y = y0 + ly; + for rx in 0..w { + let x = if reverse { w - 1 - rx } else { rx }; + let li = ly * w + x; + if !self.targets[y * w + x] { + continue; + } + let t = (x as i32, y as i32); + let (mut best, mut bc) = (nn[li], co[li]); + // Propagation from the already-visited neighbours (within this band). + let nx = x as i32 - step; + if nx >= 0 && (nx as usize) < w { + let c = nn[ly * w + nx as usize]; + let cand = (c.0 + step, c.1); + if self.is_valid(cand.0, cand.1) && cand != best { + let d = self.dist(t, cand, bc); + if d < bc { + (best, bc) = (cand, d); + } + } + } + let nly = ly as i32 - step; + if nly >= 0 && (nly as usize) < rows { + let c = nn[nly as usize * w + x]; + let cand = (c.0, c.1 + step); + if self.is_valid(cand.0, cand.1) && cand != best { + let d = self.dist(t, cand, bc); + if d < bc { + (best, bc) = (cand, d); + } + } + } + // Random search in exponentially shrinking windows. + let mut rad = search_radius; + while rad >= 1 { + let cand = (best.0 + rng.range(-rad, rad), best.1 + rng.range(-rad, rad)); + if self.is_valid(cand.0, cand.1) && cand != best { + let d = self.dist(t, cand, bc); + if d < bc { + (best, bc) = (cand, d); + } + } + rad /= 2; + } + nn[li] = best; + co[li] = bc; + } + } + }; + #[cfg(not(target_arch = "wasm32"))] + if w * self.h >= PAR_MIN { + use rayon::prelude::*; + nnf.par_chunks_mut(BAND * w).zip(cost.par_chunks_mut(BAND * w)).enumerate().for_each(|(bi, (nn, co))| band_fn(bi, nn, co)); + return; + } + for (bi, (nn, co)) in nnf.chunks_mut(BAND * w).zip(cost.chunks_mut(BAND * w)).enumerate() { + band_fn(bi, nn, co); + } + } + + /// Re-estimate hole pixels by a similarity-weighted vote of all overlapping patch matches. + fn vote(&mut self, nnf: &[(i32, i32)], cost: &[f32]) { + let (w, h, ch, r) = (self.w, self.h, self.ch, self.r); + // σ² from the 75th percentile of match costs (Wexler et al. use a robust scale like this). + let mut cs: Vec = cost.iter().zip(&self.targets).filter(|(c, t)| **t && c.is_finite()).map(|(c, _)| *c).collect(); + let sigma2 = if cs.is_empty() { + 1.0 + } else { + let k = (cs.len() * 3 / 4).min(cs.len() - 1); + let (_, v, _) = cs.select_nth_unstable_by(k, |a, b| a.total_cmp(b)); + v.max(1e-6) + }; + let img = &*self.img; + let row = |y: usize| -> Vec<(usize, Vec)> { + let mut out = Vec::new(); + if (self.ctl)() { + return out; + } + for x in 0..w { + if !self.hole[y * w + x] { + continue; + } + let mut acc = vec![0.0f32; ch]; + let mut wsum = 0.0f32; + for dy in -r..=r { + let ty = y as i32 + dy; + if ty < 0 || ty as usize >= h { + continue; + } + for dx in -r..=r { + let tx = x as i32 + dx; + if tx < 0 || tx as usize >= w { + continue; + } + let ti = ty as usize * w + tx as usize; + if !self.targets[ti] || !cost[ti].is_finite() { + continue; + } + let s = nnf[ti]; + let (sx, sy) = (s.0 - dx, s.1 - dy); + let wt = (-cost[ti] / (2.0 * sigma2)).exp().max(1e-8); + let si = (sy as usize * w + sx as usize) * ch; + for c in 0..ch { + acc[c] += img[si + c] * wt; + } + wsum += wt; + } + } + if wsum > 0.0 { + acc.iter_mut().for_each(|v| *v /= wsum); + out.push((x, acc)); + } + } + out + }; + #[cfg(not(target_arch = "wasm32"))] + let rows: Vec)>> = if w * h >= PAR_MIN { + use rayon::prelude::*; + (0..h).into_par_iter().map(row).collect() + } else { + (0..h).map(row).collect() + }; + #[cfg(target_arch = "wasm32")] + let rows: Vec)>> = (0..h).map(row).collect(); + for (y, r) in rows.into_iter().enumerate() { + for (x, v) in r { + self.img[(y * w + x) * ch..(y * w + x + 1) * ch].copy_from_slice(&v); + } + } + } +} + +/// Content-aware completion of `hole` (see module docs). Returns `None` when there is no fully-known +/// patch to copy from (the caller can fall back to [`membrane_fill`]). +pub fn complete(w: usize, h: usize, ch: usize, img: &[f32], hole: &[bool], p: &CompleteParams) -> Option> { + // Never cancelled, so never `Err`. + complete_with(w, h, ch, img, hole, p, &|| false).unwrap_or(None) +} + +/// [`complete`] that can be cancelled (checked per EM iteration, PatchMatch pass and row band) +/// (`cancelled` is polled per EM iteration, PatchMatch pass and row band): `Err(Cancelled)` once it +/// returned `true`. `Ok(None)` also for mismatched buffer sizes or `ch == 0`. +pub fn complete_with( + w: usize, + h: usize, + ch: usize, + img: &[f32], + hole: &[bool], + p: &CompleteParams, + cancelled: &(dyn Fn() -> bool + Sync), +) -> Result>, Cancelled> { + let check = || if cancelled() { Err(Cancelled) } else { Ok(()) }; + if ch == 0 || Some(hole.len()) != w.checked_mul(h) || Some(img.len()) != hole.len().checked_mul(ch) { + return Ok(None); + } + let Some((bx0, by0, bx1, by1)) = hole_bbox(w, h, hole) else { return Ok(Some(img.to_vec())) }; + let r = p.patch_radius.max(1); + let psz = 2 * r + 1; + // Pyramid: shrink until the hole is a few patches across. + let mut levels = vec![Level { w, h, img: img.to_vec(), hole: hole.to_vec() }]; + let mut ext = (bx1 - bx0).max(by1 - by0); + let mut exts = vec![ext]; + while let Some(l) = levels.last() { + check()?; + if ext <= 2 * psz || l.w / 2 < 3 * psz || l.h / 2 < 3 * psz { + break; + } + let d = downsample(l, ch); + levels.push(d); + ext = ext.div_ceil(2); + exts.push(ext); + } + let nlev = levels.len(); + let mut nnf: Vec<(i32, i32)> = Vec::new(); + let mut prev: Option<(usize, usize, Vec)> = None; + for li in (0..nlev).rev() { + check()?; + let Level { w: lw, h: lh, img: mut limg, hole: lhole } = levels[li].clone(); + // Initialise the hole: membrane at the coarsest level, upsampled result elsewhere. + match &prev { + None => limg = membrane_fill(lw, lh, ch, &limg, &lhole), + Some((pw, ph, pimg)) => { + for y in 0..lh { + for x in 0..lw { + if lhole[y * lw + x] { + let (sx, sy) = ((x / 2).min(pw - 1), (y / 2).min(ph - 1)); + let (d, s) = ((y * lw + x) * ch, (sy * pw + sx) * ch); + limg[d..d + ch].copy_from_slice(&pimg[s..s + ch]); + } + } + } + } + } + let ri = r as i32; + let hs = integral(lw, lh, &lhole); + let mut valid = vec![false; lw * lh]; + let mut valid_list = Vec::new(); + let mut targets = vec![false; lw * lh]; + for y in 0..lh as i32 { + if y % 32 == 0 { + check()?; + } + for x in 0..lw as i32 { + let cnt = window_count(&hs, lw, lh, x - ri, y - ri, x + ri + 1, y + ri + 1); + let inside = x >= ri && y >= ri && x + ri < lw as i32 && y + ri < lh as i32; + let i = y as usize * lw + x as usize; + if inside && cnt == 0 { + valid[i] = true; + valid_list.push((x, y)); + } + targets[i] = cnt > 0; + } + } + if valid_list.is_empty() { + if li == 0 { + return Ok(None); + } + prev = Some((lw, lh, limg)); + nnf.clear(); + continue; + } + let mut solver = Solver { w: lw, h: lh, ch, r: ri, img: &mut limg, hole: &lhole, valid, valid_list, targets, ctl: cancelled }; + // NNF init: upsampled from the coarser level where possible, random otherwise. + let mut rng = Rng::new(p.seed ^ (li as u64) << 20); + let pw_prev = prev.as_ref().map_or(0, |p| p.0); + let old = std::mem::take(&mut nnf); + let mut new_nnf = vec![(0i32, 0i32); lw * lh]; + for y in 0..lh { + if y % 32 == 0 { + check()?; + } + for x in 0..lw { + if !solver.targets[y * lw + x] { + continue; + } + let mut cand = None; + if !old.is_empty() && pw_prev > 0 { + let pi = (y / 2) * pw_prev + (x / 2); + if let Some(&(sx, sy)) = old.get(pi) { + let c = (sx * 2 + (x % 2) as i32, sy * 2 + (y % 2) as i32); + if solver.is_valid(c.0, c.1) { + cand = Some(c); + } + } + } + new_nnf[y * lw + x] = cand.unwrap_or_else(|| solver.valid_list[(rng.next() % solver.valid_list.len() as u64) as usize]); + } + } + nnf = new_nnf; + let refine_only = li + 1 < nlev && exts[li] > p.refine_extent; + let em = if li + 1 == nlev { + p.em_iters + } else if refine_only { + 1 + } else { + (p.em_iters >> (nlev - 1 - li)).max(p.min_em_iters) + }; + let pm_iters = if refine_only { 1 } else { p.pm_iters }; + let full_search = (lw.max(lh)) as i32; + let mut cost = vec![f32::INFINITY; lw * lh]; + for it in 0..em { + // Costs change as hole pixels are re-estimated: refresh before each PatchMatch. + for (i, c) in cost.iter_mut().enumerate() { + if i % (lw * 32).max(1) == 0 { + check()?; + } + if solver.targets[i] { + *c = solver.dist(((i % lw) as i32, (i / lw) as i32), nnf[i], f32::INFINITY); + } + } + // Wide random search on coarse levels / first iteration, local refinement afterwards. + let radius = if refine_only { + 2 + } else if li + 1 == nlev || it == 0 { + full_search + } else { + (4 * psz as i32).min(full_search) + }; + for pass in 0..pm_iters { + check()?; + solver.patchmatch(&mut nnf, &mut cost, pass + it * pm_iters, p.seed.wrapping_add((li * 1000 + it) as u64), radius); + } + check()?; + solver.vote(&nnf, &cost); + check()?; + } + prev = Some((lw, lh, limg)); + } + Ok(prev.map(|(_, _, img)| img)) +} + +/// Proximity match: the displacement `(dx, dy)` (within `max_radius`) whose surroundings best match the +/// ring of `ring` pixels around the hole, with the displaced hole fully inside the grid and outside the +#[cfg(test)] +mod tests { + use super::*; + + /// Smooth gradient plus deterministic fine noise. + fn texture(w: usize, h: usize) -> Vec { + let mut rng = Rng::new(7); + (0..w * h) + .flat_map(|i| { + let (x, y) = ((i % w) as f32, (i / w) as f32); + let n = (rng.next() % 1000) as f32 / 1000.0 * 0.04 - 0.02; + [0.3 + 0.004 * x + n, 0.5 + 0.002 * y + n, 0.4 + n] + }) + .collect() + } + + fn disc(w: usize, h: usize, cx: f32, cy: f32, r: f32) -> Vec { + (0..w * h).map(|i| ((i % w) as f32 - cx).hypot((i / w) as f32 - cy) < r).collect() + } + + fn hole_rmse(a: &[f32], b: &[f32], hole: &[bool], ch: usize) -> f32 { + let (mut e, mut n) = (0.0f32, 0); + for (i, hh) in hole.iter().enumerate() { + if *hh { + for c in 0..ch { + e += (a[i * ch + c] - b[i * ch + c]).powi(2); + n += 1; + } + } + } + (e / n as f32).sqrt() + } + + #[test] + fn completion_fills_blemish_from_texture() { + let (w, h, ch) = (96, 96, 3); + let clean = texture(w, h); + let hole = disc(w, h, 48.0, 48.0, 9.0); + let mut dirty = clean.clone(); + for (i, hh) in hole.iter().enumerate() { + if *hh { + dirty[i * ch..i * ch + ch].copy_from_slice(&[0.02, 0.02, 0.02]); + } + } + let out = complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap(); + let before = hole_rmse(&dirty, &clean, &hole, ch); + let after = hole_rmse(&out, &clean, &hole, ch); + assert!(after < 0.06 && after < before * 0.15, "rmse before {before} after {after}"); + // Outside the hole untouched. + assert!(out.iter().zip(&dirty).zip(hole.iter().flat_map(|h| [*h; 3])).all(|((a, b), h)| h || a == b)); + // Deterministic. + assert_eq!(out, complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap()); + } + + #[test] + fn completion_continues_structure() { + // Vertical stripes: the filled hole should keep the stripes (not a blur). + let (w, h, ch) = (80, 80, 1); + let clean: Vec = (0..w * h).map(|i| if (i % w / 4) % 2 == 0 { 0.2 } else { 0.8 }).collect(); + let hole = disc(w, h, 40.0, 40.0, 7.0); + let dirty: Vec = clean.iter().zip(&hole).map(|(c, h)| if *h { 0.5 } else { *c }).collect(); + let out = complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap(); + let err = hole_rmse(&out, &clean, &hole, ch); + assert!(err < 0.15, "{err}"); + } +} diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index c8219f025..197020e45 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -14,6 +14,7 @@ lightcraft-color = { workspace = true } lightcraft-raster = { workspace = true } lightcraft-develop = { workspace = true } lightcraft-raw = { workspace = true } +lightcraft-inpaint = { workspace = true } rayon = { workspace = true } serde = { workspace = true } diff --git a/crates/pipeline/src/spots.rs b/crates/pipeline/src/spots.rs index c50bcf799..483f83f3c 100644 --- a/crates/pipeline/src/spots.rs +++ b/crates/pipeline/src/spots.rs @@ -5,6 +5,10 @@ //! (frequency separation — a fast, stable approximation of gradient-domain healing). //! - **Remove** is Heal with an automatically chosen source: candidate offsets on rings around the //! spot are scored by how well the source's surrounding annulus matches the target's. +//! - **Fill** (content-aware) rebuilds the spot from patches of its surroundings instead of one +//! source: PatchMatch-based completion ([`lightcraft_inpaint::complete`]), coarse to fine, so the +//! structure is chosen at a scale set by the spot's size (the same at preview and export size). +//! Deterministic; the source offset is ignored. use lightcraft_develop::{Spot, SpotMode}; use lightcraft_geom::Point; @@ -235,6 +239,19 @@ pub fn window_for_reads_checked( ) } +/// Content-aware fill of the box: `src` becomes `tgt` with every pixel under the stroke (`alpha > 0`) +/// rebuilt from the rest of the box. Falls back to a smooth membrane fill when the box holds no whole +/// patch to copy from. +fn fill_box(src: &mut [[f32; 3]], tgt: &[[f32; 3]], alpha: &[f32], bw: usize, bh: usize) { + let hole: Vec = alpha.iter().map(|a| *a > 0.0).collect(); + let flat: Vec = tgt.iter().flatten().copied().collect(); + let out = lightcraft_inpaint::complete(bw, bh, 3, &flat, &hole, &lightcraft_inpaint::CompleteParams::default()) + .unwrap_or_else(|| lightcraft_inpaint::membrane_fill(bw, bh, 3, &flat, &hole)); + for (d, v) in src.iter_mut().zip(out.chunks_exact(3)) { + *d = [v[0], v[1], v[2]]; + } +} + /// Apply all spots to `img` (output resolution). `ppl` = output pixels per long-edge unit. pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) { if spots.is_empty() { @@ -253,7 +270,9 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) { }) .collect(); let Some(&first) = pts.first() else { continue }; + let fill = spot.mode == SpotMode::Fill; let offset = match spot.source_offset { + _ if fill => (0.0, 0.0), Some(o) => { let a = to_out.apply(Point::new(0.0, 0.0)); let b = to_out.apply(o); @@ -269,7 +288,8 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) { x1 = x1.max(p.0 + r); y1 = y1.max(p.1 + r); } - let pad = r; + // content-aware fill reads the spot's surroundings (the window holds 2 radii around it) + let pad = if fill { 2.0 * r } else { r }; let (bx0, by0) = (((x0 - pad).floor().max(0.0)) as usize, ((y0 - pad).floor().max(0.0)) as usize); let (bx1, by1) = (((x1 + pad).ceil() as usize).min(w), ((y1 + pad).ceil() as usize).min(h)); if bx1 <= bx0 || by1 <= by0 { @@ -292,7 +312,10 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) { tgt[y * bw + x] = img.get(bx0 + x, by0 + y); } } - let heal = spot.mode != SpotMode::Clone; + if fill { + fill_box(&mut src, &tgt, &alpha, bw, bh); + } + let heal = matches!(spot.mode, SpotMode::Remove | SpotMode::Heal); let low = if heal { // Tone correction from the spot's *surroundings* only: normalized convolution of // (target − source) over pixels outside the stroke (a membrane-like interpolation). @@ -361,6 +384,79 @@ mod tests { assert!((c[0] - 0.26).abs() < 0.03, "{c:?}"); } + /// Vertical stripes (period `period` px) with a black blot over the middle. + fn striped_with_blot(w: usize, h: usize, period: usize) -> Rgb32f { + let mut img = Rgb32f::from_fn(w, h, |x, _| if (x / period).is_multiple_of(2) { [0.2; 3] } else { [0.7; 3] }); + let (cx, cy, r) = (w as f32 / 2.0, h as f32 / 2.0, w as f32 * 0.04); + for y in 0..h { + for x in 0..w { + if (x as f32 + 0.5 - cx).hypot(y as f32 + 0.5 - cy) < r { + img.set(x, y, [0.0; 3]); + } + } + } + img + } + + fn fill_spot() -> Spot { + // a source offset is ignored by Fill + Spot { + mode: SpotMode::Fill, + points: vec![Point::new(0.5, 0.5)], + size: 0.06, + feather: 0.0, + source_offset: Some(Point::new(0.3, 0.0)), + ..Default::default() + } + } + + #[test] + fn fill_rebuilds_structure_deterministically() { + let (w, h) = (200, 160); + let mut img = striped_with_blot(w, h, 8); + let frame = Frame::new(w, h, &DevelopSettings::default(), true); + apply(&mut img, &[fill_spot()], &frame, frame.px_per_long(w)); + // the blot is gone and the stripes continue through it + let mut err = 0.0f32; + for x in 90..110 { + let want = if (x / 8) % 2 == 0 { 0.2 } else { 0.7 }; + err += (img.get(x, 80)[0] - want).abs(); + } + assert!(err / 20.0 < 0.12, "mean error {}", err / 20.0); + let mut again = striped_with_blot(w, h, 8); + apply(&mut again, &[fill_spot()], &frame, frame.px_per_long(w)); + assert_eq!(img.data, again.data); + } + + #[test] + fn fill_matches_between_preview_and_export_size() { + // the same photo at two sizes: the half-size fill matches the full-size one, downsampled + let (mut big, mut small) = (striped_with_blot(320, 240, 16), striped_with_blot(160, 120, 8)); + let (fb, fs) = (Frame::new(320, 240, &DevelopSettings::default(), true), Frame::new(160, 120, &DevelopSettings::default(), true)); + apply(&mut big, &[fill_spot()], &fb, fb.px_per_long(320)); + apply(&mut small, &[fill_spot()], &fs, fs.px_per_long(160)); + let (mut err, mut n) = (0.0f32, 0); + for y in 50..70 { + for x in 70..90 { + let b: f32 = [(0, 0), (1, 0), (0, 1), (1, 1)].iter().map(|(dx, dy)| big.get(2 * x + dx, 2 * y + dy)[0]).sum::() / 4.0; + err += (b - small.get(x, y)[0]).abs(); + n += 1; + } + } + assert!(err / n as f32 <= 0.08, "mean difference {}", err / n as f32); + } + + #[test] + fn fill_survives_hostile_spots() { + let frame = Frame::new(40, 30, &DevelopSettings::default(), true); + for (p, size) in [((0.0, 0.0), 2.0), ((-5.0, 9.0), 0.1), ((0.5, 0.5), 0.0), ((f64::NAN, 0.5), 0.1), ((0.5, 0.5), f64::INFINITY)] { + let mut img = Rgb32f::from_fn(40, 30, |x, _| [x as f32 / 40.0; 3]); + let spot = Spot { mode: SpotMode::Fill, points: vec![Point::new(p.0, p.1)], size, ..Default::default() }; + apply(&mut img, &[spot], &frame, frame.px_per_long(40)); + assert!(img.data.iter().flatten().all(|v| v.is_finite())); + } + } + #[test] fn clone_copies_source() { let mut img = Rgb32f::from_fn(100, 100, |x, _| if x < 50 { [0.1; 3] } else { [0.9; 3] }); diff --git a/crates/ui-egui/locales/de.json b/crates/ui-egui/locales/de.json index 6520fb3e3..a4169bad1 100644 --- a/crates/ui-egui/locales/de.json +++ b/crates/ui-egui/locales/de.json @@ -2039,5 +2039,6 @@ "Downloading model": "Modell wird heruntergeladen", "Finding faces": "Gesichter werden gesucht", "Synchronizing folder": "Ordner wird synchronisiert", - "Looking for changes": "Änderungen werden gesucht" + "Looking for changes": "Änderungen werden gesucht", + "Content-Aware": "Inhaltsbasiert" } diff --git a/crates/ui-egui/locales/es.json b/crates/ui-egui/locales/es.json index fc53b6392..9920b9e61 100644 --- a/crates/ui-egui/locales/es.json +++ b/crates/ui-egui/locales/es.json @@ -1884,5 +1884,6 @@ "Downloading model": "Descargando modelo", "Finding faces": "Buscando caras", "Synchronizing folder": "Sincronizando carpeta", - "Looking for changes": "Buscando cambios" + "Looking for changes": "Buscando cambios", + "Content-Aware": "Según el contenido" } diff --git a/crates/ui-egui/locales/fr.json b/crates/ui-egui/locales/fr.json index 53509a503..bbea4b894 100644 --- a/crates/ui-egui/locales/fr.json +++ b/crates/ui-egui/locales/fr.json @@ -1884,5 +1884,6 @@ "Panels": "Panels", "Reset Panel Layout": "Reset Panel Layout", "Resize panel window": "Resize panel window", - "Resize panels": "Resize panels" + "Resize panels": "Resize panels", + "Content-Aware": "Contenu pris en compte" } diff --git a/crates/ui-egui/locales/ja.json b/crates/ui-egui/locales/ja.json index 2babd30b6..9556aa280 100644 --- a/crates/ui-egui/locales/ja.json +++ b/crates/ui-egui/locales/ja.json @@ -1886,5 +1886,6 @@ "Downloading model": "モデルをダウンロード中", "Finding faces": "顔を検索中", "Synchronizing folder": "フォルダを同期中", - "Looking for changes": "変更を確認中" + "Looking for changes": "変更を確認中", + "Content-Aware": "コンテンツに応じる" } diff --git a/crates/ui-egui/locales/pt-br.json b/crates/ui-egui/locales/pt-br.json index dfe472818..b4ac44cc5 100644 --- a/crates/ui-egui/locales/pt-br.json +++ b/crates/ui-egui/locales/pt-br.json @@ -1884,5 +1884,6 @@ "Downloading model": "Baixando modelo", "Finding faces": "Procurando rostos", "Synchronizing folder": "Sincronizando pasta", - "Looking for changes": "Procurando alterações" + "Looking for changes": "Procurando alterações", + "Content-Aware": "Sensível ao conteúdo" } diff --git a/crates/ui-egui/locales/ru.json b/crates/ui-egui/locales/ru.json index fafb65a14..d1331062f 100644 --- a/crates/ui-egui/locales/ru.json +++ b/crates/ui-egui/locales/ru.json @@ -1598,5 +1598,6 @@ "Finding faces": "Поиск лиц", "Quit Anyway": "Всё равно выйти", "Synchronizing folder": "Синхронизация папки", - "Looking for changes": "Поиск изменений" + "Looking for changes": "Поиск изменений", + "Content-Aware": "С учётом содержимого" } diff --git a/crates/ui-egui/locales/uk.json b/crates/ui-egui/locales/uk.json index 7ebad5b10..f09bee86a 100644 --- a/crates/ui-egui/locales/uk.json +++ b/crates/ui-egui/locales/uk.json @@ -2078,5 +2078,6 @@ "Panels": "Panels", "Reset Panel Layout": "Reset Panel Layout", "Resize panel window": "Resize panel window", - "Resize panels": "Resize panels" + "Resize panels": "Resize panels", + "Content-Aware": "З урахуванням вмісту" } diff --git a/crates/ui-egui/locales/zh-hans.json b/crates/ui-egui/locales/zh-hans.json index 8a0e58b8e..ef587fe3e 100644 --- a/crates/ui-egui/locales/zh-hans.json +++ b/crates/ui-egui/locales/zh-hans.json @@ -1884,5 +1884,6 @@ "Downloading model": "正在下载模型", "Finding faces": "正在查找人脸", "Synchronizing folder": "正在同步文件夹", - "Looking for changes": "正在查找更改" + "Looking for changes": "正在查找更改", + "Content-Aware": "内容识别" } diff --git a/crates/ui-egui/locales/zh-hant.json b/crates/ui-egui/locales/zh-hant.json index f1cbec3a0..4c9bb8689 100644 --- a/crates/ui-egui/locales/zh-hant.json +++ b/crates/ui-egui/locales/zh-hant.json @@ -1884,5 +1884,6 @@ "Downloading model": "正在下載模型", "Finding faces": "正在尋找臉孔", "Synchronizing folder": "正在同步資料夾", - "Looking for changes": "正在尋找變更" + "Looking for changes": "正在尋找變更", + "Content-Aware": "內容感知" } diff --git a/crates/ui-egui/src/panels/detail.rs b/crates/ui-egui/src/panels/detail.rs index c3678ae20..eb2bcd2f9 100644 --- a/crates/ui-egui/src/panels/detail.rs +++ b/crates/ui-egui/src/panels/detail.rs @@ -1875,6 +1875,7 @@ fn remove_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respo let mode = match app.ui.tool.as_str() { "heal" => "heal", "clone" => "clone", + "fill" => "fill", _ => "remove", }; let pts: Vec<[f64; 2]> = points.iter().map(|q| [q.x, q.y]).collect(); diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs index 1c5a04ad9..f7c8c145e 100644 --- a/crates/ui-egui/src/panels/right.rs +++ b/crates/ui-egui/src/panels/right.rs @@ -343,7 +343,7 @@ fn remove(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { header(ui, "Remove"); padded(ui, |ui| { ui.horizontal_wrapped(|ui| { - for (label, tool) in [("Remove", "remove"), ("Heal", "heal"), ("Clone", "clone")] { + for (label, tool) in [("Remove", "remove"), ("Content-Aware", "fill"), ("Heal", "heal"), ("Clone", "clone")] { if text_button(ui, &format!("removeMode-{tool}"), label, app.ui.tool == tool).clicked() { app.ui.tool = tool.into(); } @@ -383,6 +383,7 @@ fn remove(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { lightcraft_develop::SpotMode::Heal => "Heal", lightcraft_develop::SpotMode::Clone => "Clone", lightcraft_develop::SpotMode::Remove => "Remove", + lightcraft_develop::SpotMode::Fill => "Content-Aware", }; super::edit::sub_title(ui, &crate::i18n::tr_format!("{mode} spot {} of {}", i + 1, d.spots.len(), mode = mode)); } diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 4443cb230..2ad481d8f 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -46,6 +46,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("codecs", Class::Layer(1)), ("meta", Class::Layer(1)), ("develop", Class::Layer(1)), + ("inpaint", Class::Layer(1)), ("faces", Class::Layer(1)), ("scenes", Class::Layer(1)), ("pipeline", Class::Layer(2)), From e14207599852d0923ce3bf0baec528e0587bb956 Mon Sep 17 00:00:00 2001 From: Andrew Frahn Date: Sun, 11 Oct 2026 20:02:17 +1100 Subject: [PATCH 2/2] NOTICE: credit PhotoCraft for the ported inpaint code; Cargo.lock version for lightcraft-inpaint Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 2 +- NOTICE | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 40a9988fc..8d5c23df1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3293,7 +3293,7 @@ dependencies = [ [[package]] name = "lightcraft-inpaint" -version = "0.5.0" +version = "0.6.0" dependencies = [ "rayon", ] diff --git a/NOTICE b/NOTICE index 5fd60227f..45b08fd4e 100644 --- a/NOTICE +++ b/NOTICE @@ -22,6 +22,12 @@ Foundation project licensed under the Apache License 2.0, and from the CIE 1931 functions it ships (CIE, CC-BY-4.0). The matrices were fitted by the LightCraft contributors (2026) with tools/spectral_camera_table.py. See assets/ATTRIBUTION.md. +crates/inpaint (lightcraft-inpaint) ports the inpainting (PatchMatch completion) and Poisson membrane +fill code of PhotoCraft (https://github.com/storytold/photocraft, crates/algo/src/inpaint.rs and +poisson.rs), Copyright the PhotoCraft contributors, licensed under the MIT License or the Apache +License 2.0, at your option. The port was modified by the LightCraft contributors (2026); each +ported file says so. + crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers (https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker), Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.