diff --git a/crates/engine/src/availability.rs b/crates/engine/src/availability.rs new file mode 100644 index 000000000..5976a3b2e --- /dev/null +++ b/crates/engine/src/availability.rs @@ -0,0 +1,215 @@ +//! Is a file there? — answered without blocking the caller. +//! +//! On a sleeping NAS, a dropped share or a USB drive spinning up, one `stat` can take as long as +//! the OS or SMB timeout. The UI asks about files every frame (grid cells, the Info panel, the +//! Missing Photos view), so in the app the answers come from a cache that a worker thread fills +//! ([`Availability::run_in_background`]): an unknown file is reported as `None` ("assume it is +//! there"; the render worker finds out if it isn't), known answers are rechecked after +//! [`RECHECK`]. Everywhere else (CLI, MCP, tests) the default is synchronous: every question is a +//! fresh check, as before. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +/// Checks whether a path exists (injectable: tests use a slow one). +pub type ExistsProbe = Arc bool + Send + Sync>; +/// Called (from the worker) when an answer changed, e.g. to repaint the UI. +pub type Waker = Arc; + +/// How long a cached answer is used before it is checked again (it is still returned meanwhile). +pub const RECHECK: Duration = Duration::from_secs(15); + +fn fs_exists(path: &str) -> bool { + // the browser build has no file system: its originals are always "there" + cfg!(target_arch = "wasm32") || std::path::Path::new(path).exists() +} + +#[derive(Default)] +struct State { + /// path → (exists, when checked) + known: HashMap, + queue: VecDeque, + queued: HashSet, + worker: bool, + /// Bumped whenever an answer changes (callers recompute what depends on it). + generation: u64, +} + +fn lock(m: &Mutex) -> MutexGuard<'_, State> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// File availability, synchronous or cached + background-checked. Cloning shares the cache. +#[derive(Clone)] +pub struct Availability { + probe: ExistsProbe, + background: Option, + state: Arc>, +} + +impl Default for Availability { + fn default() -> Self { + Self { probe: Arc::new(fs_exists), background: None, state: Arc::default() } + } +} + +impl std::fmt::Debug for Availability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Availability").field("background", &self.background.is_some()).finish() + } +} + +impl Availability { + /// Replace the existence check (tests: a slow or counting probe). Forgets cached answers. + pub fn set_probe(&mut self, probe: ExistsProbe) { + self.probe = probe; + let mut st = lock(&self.state); + st.known.clear(); + st.generation += 1; + } + + /// Answer from the cache and check on a worker thread from now on; `wake` runs when an + /// answer changed. No effect in the browser build (no threads, no file system). + pub fn run_in_background(&mut self, wake: Waker) { + if cfg!(target_arch = "wasm32") { + return; + } + self.background = Some(wake); + } + + /// The existence check itself (blocking: for worker threads, e.g. the Rename preview's). + pub fn probe(&self) -> ExistsProbe { + self.probe.clone() + } + + pub fn is_background(&self) -> bool { + self.background.is_some() + } + + /// Does `path` exist? Synchronous mode: checked now. Background mode: the cached answer + /// (`None` while unknown); unknown or stale answers are queued for the worker. + pub fn exists(&self, path: &str) -> Option { + let Some(wake) = &self.background else { + return Some((self.probe)(path)); + }; + let mut st = lock(&self.state); + let now = std::time::Instant::now(); + let cached = st.known.get(path).copied(); + if cached.is_none_or(|(_, at)| now.saturating_duration_since(at) >= RECHECK) && !st.queued.contains(path) { + st.queued.insert(path.to_string()); + st.queue.push_back(path.to_string()); + if !st.worker { + st.worker = true; + drop(st); + self.spawn_worker(wake.clone()); + return cached.map(|c| c.0); + } + } + cached.map(|c| c.0) + } + + /// Known to be missing (unknown counts as present). + pub fn is_offline(&self, path: &str) -> bool { + self.exists(path) == Some(false) + } + + /// Changes whenever a cached answer changes. + pub fn generation(&self) -> u64 { + lock(&self.state).generation + } + + /// Paths waiting for the worker. + pub fn pending(&self) -> usize { + lock(&self.state).queue.len() + } + + fn spawn_worker(&self, wake: Waker) { + let (state, probe) = (self.state.clone(), self.probe.clone()); + let started = std::thread::Builder::new().name("lc-availability".into()).spawn(move || { + loop { + let path = { + let mut st = lock(&state); + let Some(p) = st.queue.pop_front() else { + st.worker = false; + return; + }; + p + }; + let ok = probe(&path); + let changed = { + let mut st = lock(&state); + st.queued.remove(&path); + let prev = st.known.insert(path, (ok, std::time::Instant::now())); + let changed = prev.map(|p| p.0) != Some(ok); + if changed { + st.generation += 1; + } + changed + }; + if changed { + wake(); + } + } + }); + if started.is_err() { + // try again on the next question + lock(&self.state).worker = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn synchronous_by_default() { + let mut a = Availability::default(); + let n = Arc::new(AtomicUsize::new(0)); + let c = n.clone(); + a.set_probe(Arc::new(move |p| { + c.fetch_add(1, Ordering::SeqCst); + p == "here" + })); + assert_eq!(a.exists("here"), Some(true)); + assert_eq!(a.exists("gone"), Some(false)); + assert_eq!(a.exists("gone"), Some(false)); + assert_eq!(n.load(Ordering::SeqCst), 3, "every question is a fresh check"); + } + + #[test] + fn background_answers_from_the_cache_without_blocking() { + let mut a = Availability::default(); + let caller = std::thread::current().id(); + let on_caller = Arc::new(AtomicUsize::new(0)); + let c = on_caller.clone(); + a.set_probe(Arc::new(move |p| { + if std::thread::current().id() == caller { + c.fetch_add(1, Ordering::SeqCst); + } + std::thread::sleep(Duration::from_millis(200)); // a sleeping NAS + p == "here" + })); + let woke = Arc::new(AtomicUsize::new(0)); + let w = woke.clone(); + a.run_in_background(Arc::new(move || { + w.fetch_add(1, Ordering::SeqCst); + })); + let t0 = std::time::Instant::now(); + assert_eq!(a.exists("here"), None, "unknown at first"); + assert_eq!(a.exists("gone"), None); + assert!(t0.elapsed() < Duration::from_millis(150), "the caller never waits for the probe"); + let deadline = std::time::Instant::now() + Duration::from_secs(20); + while (a.exists("here").is_none() || a.exists("gone").is_none()) && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(a.exists("here"), Some(true)); + assert_eq!(a.exists("gone"), Some(false)); + assert!(a.is_offline("gone") && !a.is_offline("here") && !a.is_offline("unknown")); + assert_eq!(on_caller.load(Ordering::SeqCst), 0, "never probed on the calling thread"); + assert!(woke.load(Ordering::SeqCst) >= 2); + assert!(a.generation() >= 2); + } +} diff --git a/crates/engine/src/cmd/convert.rs b/crates/engine/src/cmd/convert.rs index 900c5257d..c81267bfb 100644 --- a/crates/engine/src/cmd/convert.rs +++ b/crates/engine/src/cmd/convert.rs @@ -26,7 +26,12 @@ fn dng_names(path: &str) -> impl Iterator { /// written to a temp file, synced, read back identical and only then given its name — never /// replacing an existing file. On any failure no DNG is left behind and the raw stays in use. pub(crate) fn write_dng_for(s: &Session, path: &str, packet: String) -> std::result::Result { - let bytes = match &s.media.file_bytes { + write_dng_with(s.media.file_bytes.as_ref(), path, packet) +} + +/// [`write_dng_for`] without the session (imports convert on a worker thread). +pub(crate) fn write_dng_with(file_bytes: Option<&crate::merge::ByteReader>, path: &str, packet: String) -> std::result::Result { + let bytes = match file_bytes { Some(r) => r(path)?, None => std::fs::read(path).map_err(|e| format!("{path}: {e}"))?, }; diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 7c81427c3..74e5d5b0f 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -7,6 +7,105 @@ use serde_json::{Value, json}; use super::{CommandSpec, always, bad, bool_or, cmd, has_active, has_selection, ok, str_param}; use crate::{LibrarySource, Result, Selection, Session}; +/// The auto-import folder's visible files with their sizes (the file-system half of +/// `library.autoImportScan`; the app lists on a worker thread and passes `listing`). +pub fn list_auto_import_folder(folder: &str) -> std::result::Result, String> { + let rd = std::fs::read_dir(folder).map_err(|e| format!("{folder}: {e}"))?; + Ok(rd + .flatten() + .filter(|e| !e.file_name().to_string_lossy().starts_with('.')) + .filter_map(|e| { + let m = std::fs::metadata(e.path()).ok()?; + m.is_file().then(|| (e.path().to_string_lossy().to_string(), m.len())) + }) + .collect()) +} + +/// `library.import`'s params, checked (no file-system calls). +pub struct ImportRequest { + pub paths: Vec, + pub opts: crate::import::ImportOptions, + pub album: Option, + pub album_name: Option, +} + +/// Parse and check `library.import`'s params (the app's import task runs the import itself, on a +/// worker thread, with the same options). +pub fn import_params(s: &Session, p: &Value) -> Result { + let paths = strs(p, "paths"); + let mode = match str_param(p, "mode").unwrap_or("add") { + "add" => crate::import::ImportMode::Add, + "copy" => crate::import::ImportMode::Copy, + "move" => crate::import::ImportMode::Move, + other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy|move)"))), + }; + let preset = match str_param(p, "preset").filter(|x| !x.is_empty()) { + Some(id) => Some(s.presets.iter().find(|x| x.id == id).cloned().ok_or_else(|| bad("library.import", format!("unknown preset `{id}`")))?), + None => None, + }; + let album = p.get("album").and_then(Value::as_u64); + if let Some(a) = album + && s.catalog.album(AlbumId(a)).is_none_or(|al| al.folder || al.is_smart()) + { + return Err(bad("library.import", "album must be a regular album")); + } + let organize = match str_param(p, "organize") { + Some(o) => crate::import::Organize::parse(o).ok_or_else(|| { + bad("library.import", format!("unknown organize `{o}` (date|month|flat, or a folder template like {{date:%Y}}/{{date:%Y%m%d}})")) + })?, + None => Default::default(), + }; + if let crate::import::Organize::Template(t) = &organize + && let Some(e) = crate::rename::folder_template_error(t) + { + return Err(bad("library.import", e)); + } + let metadata_preset = str_param(p, "metadataPreset").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string); + if let Some(n) = &metadata_preset + && !s.metadata_presets.iter().any(|m| m.name.eq_ignore_ascii_case(n)) + { + return Err(bad("library.import", format!("unknown metadata preset `{n}`"))); + } + let opts = crate::import::ImportOptions { + mode, + preset, + keywords: strs(p, "keywords"), + destination: str_param(p, "destination").map(str::to_string), + organize, + rename: str_param(p, "rename").map(str::to_string), + rename_start: p.get("renameStart").and_then(Value::as_u64).unwrap_or(1) as usize, + metadata_preset, + convert_dng: bool_or(p, "dng", false), + local: bool_or(p, "local", false), + }; + let album_name = str_param(p, "albumName").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string); + Ok(ImportRequest { paths, opts, album, album_name }) +} + +/// After a batch of an import was committed: the album (a new one named `album_name` when there +/// is no `album` and something was imported) gets the photos. Returns the report as JSON (with +/// `album`). +pub fn import_batch_done(s: &mut Session, report: crate::import::ImportReport, mut album: Option, album_name: Option<&str>) -> Result { + let mut report = serde_json::to_value(report).unwrap_or_default(); + let imported: Vec = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(); + if album.is_none() + && let Some(name) = album_name + && !imported.is_empty() + { + let r = s.execute("album.create", &json!({"name": name}))?; + album = r["id"].as_u64(); + } + if let Some(a) = album + && !imported.is_empty() + { + // (album.addPhotos wants a selection: the new photos are about to be it) + s.selection = Selection::single(PhotoId(imported[0])); + s.execute("album.addPhotos", &json!({"id": a, "ids": imported}))?; + report["album"] = json!(a); + } + Ok(report) +} + fn album_param(p: &Value, key: &str, c: &str) -> Result { p.get(key).and_then(Value::as_u64).map(AlbumId).ok_or_else(|| bad(c, format!("missing album `{key}`"))) } @@ -657,9 +756,9 @@ pub fn specs() -> Vec { "Auto Import Now", [], None, - "{} — add the watched folder's new photos (files the library doesn't have yet; partial / still-copying files wait for the next scan) → {imported, folder}", + "{listing?: [[path, size]] (the folder as listed by the caller, e.g. on a worker thread), start?: bool (false: return the `library.import` params as `import` instead of importing)} — add the watched folder's new photos (files the library doesn't have yet; partial / still-copying files wait for the next scan) → {imported, folder, import?}", always, - |s, _| { + |s, p| { let Some(folder) = s.import_defaults.auto_folder.clone() else { return Ok(json!({"imported": [], "folder": null})) }; // only files that stopped growing: a file still being written is left for later let known: std::collections::HashSet = s @@ -670,18 +769,20 @@ pub fn specs() -> Vec { _ => None, }) .collect(); + // the folder's files and sizes: listed here, or already listed on a worker thread (the app) + let listing: Vec<(String, u64)> = match p.get("listing").and_then(Value::as_array) { + Some(l) => l.iter().filter_map(|e| Some((e.get(0)?.as_str()?.to_string(), e.get(1)?.as_u64()?))).collect(), + None => list_auto_import_folder(&folder).map_err(|e| bad("library.autoImportScan", e))?, + }; let mut fresh = Vec::new(); - for e in std::fs::read_dir(&folder).map_err(|e| bad("library.autoImportScan", format!("{folder}: {e}")))?.flatten() { - let path = e.path(); - let ps = path.to_string_lossy().to_string(); - if !path.is_file() || known.contains(&ps) || path.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.')) { + for (ps, size) in listing { + if known.contains(&ps) { continue; } // each file is tried once (a non-photo isn't retried every scan) if s.auto_import_seen.get(&ps) == Some(&u64::MAX) { continue; } - let size = e.metadata().map(|m| m.len()).unwrap_or(0); let seen = s.auto_import_seen.insert(ps.clone(), size); if size > 0 && seen == Some(size) { fresh.push(ps); @@ -701,6 +802,10 @@ pub fn specs() -> Vec { for f in &fresh { s.auto_import_seen.insert(f.clone(), u64::MAX); } + if p.get("start").and_then(Value::as_bool) == Some(false) { + // the caller imports them (the app: on a worker thread) + return Ok(json!({"imported": [], "folder": folder, "import": params})); + } let sel = s.selection.clone(); let r = s.execute("library.import", ¶ms)?; // arriving photos don't take over the selection @@ -751,79 +856,15 @@ pub fn specs() -> Vec { "{paths: [file or folder (recursive)], mode?: add|copy|move (add = reference the files in place; copy = into the library's Originals/YYYY/YYYY-MM-DD/; move = as copy, then each original and its XMP sidecars are removed from the source — only after the copy is verified (a hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate and unchecked files keep their sources; a taken name gets -1, -2…; undo removes the photos from the library but leaves the files at the destination), destination?: folder for copies / moves, organize?: date (YYYY/YYYY-MM-DD) | month (YYYY/YYYY-MM) | flat | a folder template, e.g. `{date:%Y}/{date:%Y%m%d}` → 2026/20260114 (the template's `/` make the folders, each level expanded with the rename tokens and made a safe folder name: never outside the destination; must be relative, no `..`; a level with missing metadata is `unknown`) — dated by capture time, else the import time, rename?: file-name template for copies, original extension added (tokens: {name} {num} {seq} {seq:N} {date} {date:%Y%m%d} {folder} {camera} {lens} {iso} {rating} {title} {creator} {ext}; photo.renameTokens explains each; blank = keep names), renameStart?: 1, metadataPreset?: name, dng?: bool (copy raws as DNG; copy only), local?: bool (browsing: the photos stay out of the library, like library.browse; not with move), album?: albumId, albumName?: new album, preset?: presetId, keywords?: [..]} → {imported, duplicates, failed, moved?: [{from, to, sidecars?}], kept?: [{path, reason}] (move: sources left in place and why), album?}", always, |s, p| { - let paths = strs(p, "paths"); - if paths.is_empty() { + let req = import_params(s, p)?; + if req.paths.is_empty() { return Err(bad("library.import", "no paths")); } - let mode = match str_param(p, "mode").unwrap_or("add") { - "add" => crate::import::ImportMode::Add, - "copy" => crate::import::ImportMode::Copy, - "move" => crate::import::ImportMode::Move, - other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy|move)"))), - }; - let preset = match str_param(p, "preset").filter(|x| !x.is_empty()) { - Some(id) => { - Some(s.presets.iter().find(|x| x.id == id).cloned().ok_or_else(|| bad("library.import", format!("unknown preset `{id}`")))?) - } - None => None, - }; - let mut album = p.get("album").and_then(Value::as_u64); - if let Some(a) = album - && s.catalog.album(AlbumId(a)).is_none_or(|al| al.folder || al.is_smart()) - { - return Err(bad("library.import", "album must be a regular album")); - } - let organize = match str_param(p, "organize") { - Some(o) => crate::import::Organize::parse(o).ok_or_else(|| { - bad( - "library.import", - format!("unknown organize `{o}` (date|month|flat, or a folder template like {{date:%Y}}/{{date:%Y%m%d}})"), - ) - })?, - None => Default::default(), - }; - if let crate::import::Organize::Template(t) = &organize - && let Some(e) = crate::rename::folder_template_error(t) - { - return Err(bad("library.import", e)); - } - let metadata_preset = str_param(p, "metadataPreset").map(str::trim).filter(|n| !n.is_empty()).map(str::to_string); - if let Some(n) = &metadata_preset - && !s.metadata_presets.iter().any(|m| m.name.eq_ignore_ascii_case(n)) - { - return Err(bad("library.import", format!("unknown metadata preset `{n}`"))); - } - let opts = crate::import::ImportOptions { - mode, - preset, - keywords: strs(p, "keywords"), - destination: str_param(p, "destination").map(str::to_string), - organize, - rename: str_param(p, "rename").map(str::to_string), - rename_start: p.get("renameStart").and_then(Value::as_u64).unwrap_or(1) as usize, - metadata_preset, - convert_dng: super::bool_or(p, "dng", false), - local: super::bool_or(p, "local", false), - }; let undo0 = s.undo.len(); - let mut report = serde_json::to_value(crate::import::import_with(s, &paths, &opts)?).unwrap_or_default(); - let imported: Vec = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(); - if album.is_none() - && let Some(name) = str_param(p, "albumName").map(str::trim).filter(|n| !n.is_empty()) - && !imported.is_empty() - { - let r = s.execute("album.create", &json!({"name": name}))?; - album = r["id"].as_u64(); - } - if let Some(a) = album - && !imported.is_empty() - { - // (album.addPhotos wants a selection: the new photos are about to be it) - s.selection = Selection::single(PhotoId(imported[0])); - s.execute("album.addPhotos", &json!({"id": a, "ids": imported}))?; - report["album"] = json!(a); - } + let report = crate::import::import_with(s, &req.paths, &req.opts)?; + let report = import_batch_done(s, report, req.album, req.album_name.as_deref())?; // one undo step for the whole import + let imported: Vec = report["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(); let n = s.undo.len().saturating_sub(undo0); s.merge_undo(n, &format!("Add {} Photo{}", imported.len(), if imported.len() == 1 { "" } else { "s" })); if let Some(f) = imported.first() { diff --git a/crates/engine/src/cmd/missing.rs b/crates/engine/src/cmd/missing.rs index bf35f22e0..3d5fb49a3 100644 --- a/crates/engine/src/cmd/missing.rs +++ b/crates/engine/src/cmd/missing.rs @@ -45,9 +45,10 @@ pub fn missing(s: &Session) -> Vec<(PhotoId, String)> { } /// Whether photo `id` is in scope and its file is gone (the Missing Photos view checks only -/// the photos its query already narrowed to). -pub fn is_missing(cat: &Catalog, id: PhotoId) -> bool { - !cfg!(target_arch = "wasm32") && cat.photo(id).and_then(|p| checked_path(p)).is_some_and(|f| !Path::new(f).exists()) +/// the photos its query already narrowed to). In the app `avail` answers from its cache (files +/// not checked yet count as present until the background check finds them gone). +pub fn is_missing(cat: &Catalog, avail: &crate::availability::Availability, id: PhotoId) -> bool { + !cfg!(target_arch = "wasm32") && cat.photo(id).and_then(|p| checked_path(p)).is_some_and(|f| avail.is_offline(f)) } fn relink_op(id: PhotoId, path: &str) -> Op { @@ -94,8 +95,112 @@ fn file_hash(path: &str) -> Option { std::fs::read(path).ok().map(|b| lightcraft_preview::hash_bytes(&b).to_string()) } -/// Search `folder` (recursively) for each missing photo's file and relink every match in one -/// undoable step. +/// An in-scope photo ([`checked_path`]) as Find Missing Photos sees it: its file, the size and +/// content hash the library knows (0 / `None` = unknown). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FindCandidate { + pub id: PhotoId, + pub path: String, + pub size: u64, + pub hash: Option, +} + +/// Every in-scope photo, for [`plan_find_missing`]: the missing ones are looked for, and no +/// photo's file is used for another. No file-system calls, so the app takes this snapshot on the +/// UI thread and plans on a worker without holding the session. +pub fn find_candidates(cat: &Catalog) -> Vec { + cat.photos() + .filter_map(|p| { + checked_path(p).map(|f| FindCandidate { id: p.id, path: f.to_string(), size: p.file_size, hash: stored_hash(p).map(str::to_string) }) + }) + .collect() +} + +/// How a missing photo's file was recognised. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MatchBy { + /// Same name and size (and content, when the library knows its hash). + Name, + /// Renamed: same size and content hash. + Content, +} + +impl MatchBy { + pub fn as_str(self) -> &'static str { + match self { + MatchBy::Name => "name", + MatchBy::Content => "content", + } + } +} + +/// A missing photo and the file found for it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FoundFile { + pub id: PhotoId, + pub from: String, + pub to: String, + pub by: MatchBy, +} + +/// A missing photo with several look-alike files (same name and size, no hash to tell them +/// apart): skipped, never guessed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AmbiguousFile { + pub id: PhotoId, + pub from: String, + pub candidates: Vec, +} + +/// What Find Missing Photos found ([`plan_find_missing`]), before relinking. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FindPlan { + pub found: Vec, + /// In-scope photos that stay missing (the ambiguous ones included). + pub missing: usize, + pub ambiguous: Vec, +} + +impl FindPlan { + /// `{found: [{id, from, to, by}], missing, ambiguous: [{id, from, candidates}]}` — the + /// command's result, and what `library.findMissing {found, …}` takes back. + pub fn to_json(&self) -> Value { + let found: Vec = self.found.iter().map(|f| json!({"id": f.id.0, "from": f.from, "to": f.to, "by": f.by.as_str()})).collect(); + let ambiguous: Vec = self.ambiguous.iter().map(|a| json!({"id": a.id.0, "from": a.from, "candidates": a.candidates})).collect(); + json!({"found": found, "missing": self.missing, "ambiguous": ambiguous}) + } + + /// The inverse of [`FindPlan::to_json`] (entries it can't read are dropped). + pub fn from_json(v: &Value) -> FindPlan { + let list = |k: &str| v.get(k).and_then(Value::as_array).cloned().unwrap_or_default(); + let found = list("found") + .iter() + .filter_map(|f| { + let by = if f.get("by").and_then(Value::as_str) == Some("content") { MatchBy::Content } else { MatchBy::Name }; + Some(FoundFile { + id: PhotoId(f.get("id")?.as_u64()?), + from: str_param(f, "from")?.to_string(), + to: str_param(f, "to")?.to_string(), + by, + }) + }) + .collect(); + let ambiguous = list("ambiguous") + .iter() + .filter_map(|a| { + let candidates = a.get("candidates")?.as_array()?.iter().filter_map(|c| c.as_str().map(str::to_string)).collect(); + Some(AmbiguousFile { id: PhotoId(a.get("id")?.as_u64()?), from: str_param(a, "from")?.to_string(), candidates }) + }) + .collect(); + let missing = v.get("missing").and_then(Value::as_u64).map_or(0, |n| usize::try_from(n).unwrap_or(usize::MAX)); + FindPlan { found, missing, ambiguous } + } +} + +/// The file-system half of Find Missing Photos: which `candidates` are gone, and which file in +/// `folder` (recursively) is each one's. Touches only the disk — never the session — so the app +/// runs it on a worker thread (walking a big folder on a network share and hashing look-alikes +/// takes a while) and relinks the result with `library.findMissing {found, missing, ambiguous}`. /// /// Matching, per photo: /// 1. a file with the same name (any letter case), the same size and the same content hash; @@ -107,20 +212,21 @@ fn file_hash(path: &str) -> Option { /// /// Files that are already some photo's original, or that another photo was just matched with, /// are not used. -fn find_missing(s: &mut Session, p: &Value) -> Result { - const C: &str = "library.findMissing"; - let folder = str_param(p, "folder").ok_or_else(|| bad(C, "missing `folder`"))?; +pub fn plan_find_missing(candidates: &[FindCandidate], folder: &str) -> std::result::Result { if !Path::new(folder).is_dir() { - return Err(bad(C, format!("{folder}: not a folder"))); + return Err(format!("{folder}: not a folder")); } - let lost = missing(s); + if cfg!(target_arch = "wasm32") { + return Ok(FindPlan::default()); + } + let lost: Vec<&FindCandidate> = candidates.iter().filter(|c| !Path::new(&c.path).exists()).collect(); if lost.is_empty() { - return Ok(json!({"found": [], "missing": 0, "ambiguous": []})); + return Ok(FindPlan::default()); } - let in_use: HashSet = s.catalog.photos().filter_map(|p| checked_path(p).map(str::to_string)).collect(); + let in_use: HashSet<&str> = candidates.iter().map(|c| c.path.as_str()).collect(); // file name (lower case) → candidate paths; size → paths (filled lazily) let mut by_name: HashMap> = HashMap::new(); - let files: Vec = crate::import::expand(&[folder.to_string()], None).into_iter().filter(|f| !in_use.contains(f)).collect(); + let files: Vec = crate::import::expand(&[folder.to_string()], None).into_iter().filter(|f| !in_use.contains(f.as_str())).collect(); for f in &files { if let Some(n) = Path::new(f).file_name() { by_name.entry(n.to_string_lossy().to_lowercase()).or_default().push(f.clone()); @@ -131,24 +237,21 @@ fn find_missing(s: &mut Session, p: &Value) -> Result { let mut hash_of = |f: &str| hashes.entry(f.to_string()).or_insert_with(|| file_hash(f)).clone(); let size_of = |f: &str| std::fs::metadata(f).map(|m| m.len()).ok(); let mut used: HashSet = HashSet::new(); - let mut ops = Vec::new(); - let mut found = Vec::new(); - let mut ambiguous = Vec::new(); - for (id, old) in &lost { - let Some(ph) = s.catalog.photo(*id) else { continue }; - let name = Path::new(old).file_name().map(|n| n.to_string_lossy().to_lowercase()).unwrap_or_default(); - let size = ph.file_size; - let hash = stored_hash(ph).map(str::to_string); + let mut plan = FindPlan::default(); + for c in &lost { + let name = Path::new(&c.path).file_name().map(|n| n.to_string_lossy().to_lowercase()).unwrap_or_default(); + let size = c.size; + let hash = c.hash.as_deref(); let named: Vec<&String> = by_name .get(&name) - .map(|c| c.iter().filter(|f| !used.contains(*f) && (size == 0 || size_of(f) == Some(size))).collect()) + .map(|v| v.iter().filter(|f| !used.contains(*f) && (size == 0 || size_of(f) == Some(size))).collect()) .unwrap_or_default(); // 1. same name and size, confirmed by the content hash - let mut hit: Option<(String, &str)> = - hash.as_deref().and_then(|h| named.iter().find(|f| hash_of(f).as_deref() == Some(h))).map(|f| (f.to_string(), "name")); + let mut hit: Option<(String, MatchBy)> = + hash.and_then(|h| named.iter().find(|f| hash_of(f).as_deref() == Some(h))).map(|f| (f.to_string(), MatchBy::Name)); // 2. renamed: same size and content if hit.is_none() - && let Some(h) = &hash + && let Some(h) = hash && size > 0 { let sizes = by_size.get_or_insert_with(|| { @@ -162,36 +265,75 @@ fn find_missing(s: &mut Session, p: &Value) -> Result { }); hit = sizes .get(&size) - .and_then(|c| c.iter().find(|f| !used.contains(*f) && hash_of(f).as_deref() == Some(h.as_str()))) - .map(|f| (f.clone(), "content")); + .and_then(|v| v.iter().find(|f| !used.contains(*f) && hash_of(f).as_deref() == Some(h))) + .map(|f| (f.clone(), MatchBy::Content)); } // 3. same name and size only (no hash, or the file was edited in place): one candidate // is taken, several are ambiguous and skipped rather than guessed if hit.is_none() { - match named.len() { - 1 => hit = Some((named[0].clone(), "name")), - 0 => {} + match named.as_slice() { + [one] => hit = Some(((*one).clone(), MatchBy::Name)), + [] => {} _ => { - ambiguous.push(json!({"id": id.0, "from": old, "candidates": named})); + plan.ambiguous.push(AmbiguousFile { id: c.id, from: c.path.clone(), candidates: named.iter().map(|f| (*f).clone()).collect() }); continue; } } } let Some((path, by)) = hit else { continue }; - ops.push(relink_op(*id, &path)); - found.push(json!({"id": id.0, "from": old, "to": path, "by": by})); - used.insert(path); + used.insert(path.clone()); + plan.found.push(FoundFile { id: c.id, from: c.path.clone(), to: path, by }); } - let still = lost.len() - found.len(); - if !ops.is_empty() { - s.commit("Find Missing Photos", Op::Batch { ops })?; + plan.missing = lost.len() - plan.found.len(); + Ok(plan) +} + +/// Relink what [`plan_find_missing`] found, in one undoable step, re-checked against the +/// catalog as it is now (the search may have run on a worker while the library changed): a +/// photo that no longer points at the path it was missing from (relinked, removed) is left +/// alone, and a file that is now some photo's original — or that the plan names twice — is not +/// used (that photo stays missing). `missing` is the plan's count, corrected for what changed +/// among the photos the plan names. +fn commit_find(s: &mut Session, plan: FindPlan) -> Result { + let still_at = |cat: &Catalog, id: PhotoId, from: &str| cat.photo(id).and_then(|p| checked_path(p)) == Some(from); + let mut in_use: HashSet = candidates(&s.catalog).into_iter().collect(); + let mut found = Vec::new(); + let mut missing = plan.missing; + for f in plan.found { + if !still_at(&s.catalog, f.id, &f.from) { + continue; + } + if in_use.contains(&f.to) { + missing += 1; + continue; + } + in_use.insert(f.to.clone()); + found.push(f); + } + let planned = plan.ambiguous.len(); + let ambiguous: Vec = plan.ambiguous.into_iter().filter(|a| still_at(&s.catalog, a.id, &a.from)).collect(); + // ambiguous photos relinked meanwhile aren't missing any more + missing = missing.saturating_sub(planned - ambiguous.len()); + if !found.is_empty() { + s.commit("Find Missing Photos", Op::Batch { ops: found.iter().map(|f| relink_op(f.id, &f.to)).collect() })?; for f in &found { - if let Some(id) = f["id"].as_u64() { - s.media.forget(PhotoId(id)); - } + s.media.forget(f.id); } } - Ok(json!({"found": found, "missing": still, "ambiguous": ambiguous})) + Ok(FindPlan { found, missing, ambiguous }.to_json()) +} + +/// Search `folder` (recursively) for each missing photo's file ([`plan_find_missing`]) and +/// relink every match in one undoable step. With `found` instead (what a search on a worker +/// thread returned, as `{found, missing, ambiguous}`), only the relinking is done. +fn find_missing(s: &mut Session, p: &Value) -> Result { + const C: &str = "library.findMissing"; + if p.get("found").is_some_and(Value::is_array) { + return commit_find(s, FindPlan::from_json(p)); + } + let folder = str_param(p, "folder").ok_or_else(|| bad(C, "missing `folder`"))?; + let plan = plan_find_missing(&find_candidates(&s.catalog), folder).map_err(|e| bad(C, e))?; + commit_find(s, plan) } pub fn specs() -> Vec { @@ -213,7 +355,7 @@ pub fn specs() -> Vec { "Find Missing Photos", [], None, - "{folder} — relink every missing photo whose file is somewhere in the folder: same name and size (and content, when the library knows its hash), or — renamed — same size and content → {found: [{id, from, to, by: name|content}], missing, ambiguous: [{id, from, candidates}] (several same-name candidates and no hash to tell them apart: skipped)}", + "{folder} — relink every missing photo whose file is somewhere in the folder: same name and size (and content, when the library knows its hash), or — renamed — same size and content; or {found, missing, ambiguous} — relink what a search already found (the app searches on a worker thread; photos relinked meanwhile and files now in use are skipped) → {found: [{id, from, to, by: name|content}], missing, ambiguous: [{id, from, candidates}] (several same-name candidates and no hash to tell them apart: skipped)}", always, find_missing ), diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index ad7b03487..9fc831161 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -15,7 +15,7 @@ mod edit; mod export; pub mod filters; pub mod keywords; -mod library; +pub mod library; pub mod lut_profiles; pub mod manage; mod masks; diff --git a/crates/engine/src/cmd/previews.rs b/crates/engine/src/cmd/previews.rs index 44b2cebd9..8a2bf7e53 100644 --- a/crates/engine/src/cmd/previews.rs +++ b/crates/engine/src/cmd/previews.rs @@ -23,6 +23,12 @@ pub struct PreviewBuild { pub failed: AtomicUsize, pub cancel: AtomicBool, pub finished: AtomicBool, + /// Damaged smart previews rebuilt (Build Smart Previews only). + pub repaired: AtomicUsize, + /// What is built ("" = previews; "smart previews" — Build Smart Previews runs the same way). + pub what: &'static str, + /// Why the whole build failed (e.g. the smart previews folder isn't there). + pub error: std::sync::Mutex>, } impl PreviewBuild { @@ -33,8 +39,15 @@ impl PreviewBuild { "failed": self.failed.load(Ordering::Relaxed), "running": !self.finished.load(Ordering::Relaxed), "cancelled": self.cancel.load(Ordering::Relaxed), + "what": if self.what.is_empty() { "previews" } else { self.what }, + "repaired": self.repaired.load(Ordering::Relaxed), + "error": self.error(), }) } + + pub fn error(&self) -> Option { + self.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + } } /// The jobs that build `id`'s previews: the largest grid thumbnail, then the view render at @@ -94,56 +107,134 @@ fn build(s: &mut Session, p: &Value) -> Result { Ok(state.json()) } +/// One photo's smart preview to build or discard: (photo, its file in the smart previews folder, +/// how to load the original at preview size). +#[cfg(not(target_arch = "wasm32"))] +type SmartJob = (lightcraft_catalog::PhotoId, std::path::PathBuf, Option); + +/// What Build / Discard Smart Previews did. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Default)] +struct SmartCounts { + /// Proxies there now (complete ones kept, missing or damaged ones written). + built: usize, + /// Of those, damaged ones (cut short by a crash or a full drive) that were rebuilt. + repaired: usize, + removed: usize, + /// `[[id, why]]` + failed: Vec, +} + +/// The file-system half of Build / Discard Smart Previews (runs on a worker thread in the app). +/// `custom`: the folder was chosen (on another drive, perhaps): it must be there, it is never +/// recreated. With `state`, progress and cancel. +#[cfg(not(target_arch = "wasm32"))] +fn smart_run( + dir: &std::path::Path, + custom: bool, + discard: bool, + jobs: Vec, + state: Option<&PreviewBuild>, +) -> std::result::Result { + let mut n = SmartCounts::default(); + if !discard { + // a chosen folder on another drive must be there: never recreate it on this one + if custom { + crate::smart::check_writable(dir)?; + } else { + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + } + } + for (id, path, source) in jobs { + if state.is_some_and(|st| st.cancel.load(Ordering::Relaxed)) { + break; + } + if discard { + if std::fs::remove_file(&path).is_ok() { + n.removed += 1; + } + } else { + // a complete proxy is kept; a missing or damaged one (cut short by a crash or a full + // drive) is (re)built + let damaged = path.exists(); + if damaged && crate::smart::is_valid(&path) { + n.built += 1; + } else { + // atomic: a failed write leaves no partial proxy that would pass for a built one + let r = source + .ok_or_else(|| "nothing to build from".to_string()) + .and_then(|s| s.load()) + .and_then(|src| crate::smart::encode(&src)) + .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display()))); + match r { + Ok(()) => { + n.built += 1; + if damaged { + n.repaired += 1; + if let Some(st) = state { + st.repaired.fetch_add(1, Ordering::Relaxed); + } + } + } + Err(e) => { + n.failed.push(json!([id.0, e])); + if let Some(st) = state { + st.failed.fetch_add(1, Ordering::Relaxed); + } + } + } + } + } + if let Some(st) = state { + st.done.fetch_add(1, Ordering::Relaxed); + } + } + Ok(n) +} + /// Smart previews: build (from the original, at preview size) or discard the proxies that keep -/// photos editable while their originals are offline. +/// photos editable while their originals are offline. With `background` (the app) the files are +/// read and written on a worker thread; progress like Build Previews (`library.previewProgress`). #[cfg(not(target_arch = "wasm32"))] fn smart(s: &mut Session, p: &Value) -> Result { const C: &str = "library.smartPreviews"; let dir = s.media.smart_dir.clone().ok_or_else(|| bad(C, "smart previews need a library on disk"))?; let discard = p.get("discard").and_then(Value::as_bool).unwrap_or(false); - let ids = if p.get("ids").is_some() || !s.selection.ids.is_empty() { s.targets(p) } else { s.visible_cloned() }; - let (mut built, mut repaired, mut removed, mut failed) = (0usize, 0usize, 0usize, Vec::new()); - if !discard { - // a chosen folder on another drive must be there: never recreate it on this one - if s.smart_previews_dir.is_some() { - crate::smart::check_writable(&dir).map_err(|e| bad(C, e))?; - } else { - std::fs::create_dir_all(&dir).map_err(|e| bad(C, format!("{}: {e}", dir.display())))?; - } + let background = p.get("background").and_then(Value::as_bool).unwrap_or(false); + if background && s.preview_build.as_ref().is_some_and(|b| !b.finished.load(Ordering::Relaxed)) { + return Err(bad(C, "a preview build is already running")); } + let ids = if p.get("ids").is_some() || !s.selection.ids.is_empty() { s.targets(p) } else { s.visible_cloned() }; + let mut jobs: Vec = Vec::new(); for id in ids { let Some(ph) = s.catalog.photo(id).cloned() else { continue }; if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) { continue; } let path = dir.join(crate::smart::file_name(&ph)); - if discard { - if std::fs::remove_file(&path).is_ok() { - removed += 1; - } - continue; - } - // a complete proxy is kept; a missing or damaged one (cut short by a crash or a full - // drive) is (re)built - let damaged = path.exists(); - if damaged && crate::smart::is_valid(&path) { - built += 1; - continue; - } - // atomic: a failed write leaves no partial proxy that would pass for a built one - let r = s - .source_now(id, crate::media::SourceLevel::Preview) - .and_then(|src| crate::smart::encode(&src)) - .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display()))); - match r { - Ok(()) => { - built += 1; - repaired += usize::from(damaged); - } - Err(e) => failed.push(json!([id.0, e])), - } + let source = (!discard).then(|| s.media.source_ref(&ph, crate::media::SourceLevel::Preview)); + jobs.push((id, path, source)); } - Ok(json!({"built": built, "repaired": repaired, "removed": removed, "failed": failed})) + let custom = s.smart_previews_dir.is_some(); + if background { + let what = if discard { "discarding smart previews" } else { "smart previews" }; + let state = Arc::new(PreviewBuild { total: jobs.len(), what, ..Default::default() }); + s.preview_build = Some(state.clone()); + let st = state.clone(); + std::thread::Builder::new() + .name("lc-smart-previews".into()) + .spawn(move || { + if let Err(e) = smart_run(&dir, custom, discard, jobs, Some(&st)) { + log::warn!("{C}: {e}"); + *st.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(e); + } + st.finished.store(true, Ordering::Relaxed); + }) + .map_err(|e| bad(C, format!("could not start: {e}")))?; + return Ok(state.json()); + } + let r = smart_run(&dir, custom, discard, jobs, None).map_err(|e| bad(C, e))?; + Ok(json!({"built": r.built, "repaired": r.repaired, "removed": r.removed, "failed": r.failed})) } /// The smart previews folder: where it is, whether it is custom, what it holds, whether it can be @@ -225,7 +316,7 @@ pub fn specs() -> Vec { "Build Smart Previews", [], None, - "{ids?, discard?: bool} — build (or discard) the smart previews of the selected photos (else all in view): compact proxies in the library that keep photos editable and exportable (at proxy size) while their originals are offline; a damaged proxy (cut short) is rebuilt → {built, repaired, removed, failed}", + "{ids?, discard?: bool, background?: bool} — build (or discard) the smart previews of the selected photos (else all in view): compact proxies in the library that keep photos editable and exportable (at proxy size) while their originals are offline; a damaged proxy (cut short) is rebuilt → {built, repaired, removed, failed}; with background (the app's menu) the files are read and written on a worker thread → {total, done, failed, repaired, running, what} like library.buildPreviews (progress: library.previewProgress, stop: library.cancelPreviews)", always, smart ), @@ -256,7 +347,7 @@ pub fn specs() -> Vec { always, build ), - cmd!(query "library.previewProgress", "Preview Build Progress", [], None, "{} → {total, done, failed, running, cancelled} | null", always, |s, _| { + cmd!(query "library.previewProgress", "Preview Build Progress", [], None, "{} → {total, done, failed, repaired, running, cancelled, what, error} | null", always, |s, _| { Ok(s.preview_build.as_ref().map_or(Value::Null, |b| b.json())) }), cmd!("library.cancelPreviews", "Cancel Preview Build", [], None, "{}", always, |s, _| { diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 8eb4c0108..a85d62873 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -517,205 +517,392 @@ pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Res } /// Import files/folders with options (preset, keywords). See the module docs. +/// +/// The file-system half ([`ImportJob::prepare`]: expanding folders, probing, reading sidecars, +/// copying / placing files) and the catalog half ([`commit_prepared`]) are separate so the app can +/// run the first on a worker thread; here both run in turn. pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> crate::Result { - let mode = opts.mode; - // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of every - // added file in its own storage already, so "copy" there means "add". - let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move); - let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode }; - if mode == ImportMode::Move && opts.local { - return Err(crate::EngineError::Other("browsing a folder never moves its files".into())); - } - let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - // where copies go: the chosen folder, else the library's Originals/ - let copy_root = opts - .destination - .as_deref() - .filter(|d| !d.trim().is_empty()) - .map(std::path::PathBuf::from) - .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals"))); - if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { - return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); - } - let mut seq = opts.rename_start.max(1); - let files = expand(paths, lib_dir.as_deref()); - let mut report = ImportReport { scanned: files.len(), ..Default::default() }; + let mut job = ImportJob::new(s, opts.clone())?; + let prepared = job.prepare(paths, &std::sync::atomic::AtomicBool::new(false)); + // probes a scan left for files not imported now stay for a later import + s.import_probes.extend(std::mem::take(&mut job.cache)); + commit_prepared(s, &job.opts, &job.now, prepared) +} - // existing paths and content hashes - let mut by_path: HashMap<&str, PhotoId> = HashMap::new(); - let mut by_hash: HashMap = HashMap::new(); - for p in s.catalog.photos() { - if let Source::File { path } = &p.source { - by_path.insert(path.as_str(), p.id); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), p.id); - } - } - let mut todo = Vec::new(); - // a file that was only browsed (Local) joins the library when it is imported for real - let mut promote = Vec::new(); - for f in files { - match by_path.get(f.as_str()) { - Some(id) if !opts.local && s.catalog.photo(*id).is_some_and(|p| p.local) => { - promote.push(Op::SetLocal { id: *id, local: false }); - report.imported.push(id.0); +/// Everything the file-system half of an import needs, detached from the session so it can run +/// on another thread (a network share or a card can take minutes). Built by [`ImportJob::new`]; +/// [`ImportJob::prepare`] may be called for several batches in turn (renamed copies keep counting, +/// duplicates are found across batches); each batch is then [`commit_prepared`] on the session. +pub struct ImportJob { + pub opts: ImportOptions, + /// The mode in effect (Copy / Move become Add in a library without its own folder). + mode: ImportMode, + lib_dir: Option, + copy_root: Option, + /// path → (photo, a Local browse record?) + by_path: HashMap, + /// content hash → the photo that has it (`None`: a file earlier in this import) + by_hash: HashMap>, + probe: Option, + file_bytes: Option, + /// Probes from a preceding scan (not read again). + cache: HashMap, + naming: crate::sidecar::SidecarNaming, + now: String, + seq: usize, + /// Files found so far (folders expanded) and handled so far, for progress. + pub total: std::sync::Arc, + pub done: std::sync::Arc, +} + +/// The outcome of [`ImportJob::prepare`] for one batch, in file order. +#[derive(Debug, Default)] +pub struct Prepared { + pub scanned: usize, + items: Vec, +} + +#[derive(Debug)] +enum PreparedItem { + /// A Local (browsed) record imported for real. + Promote(PhotoId), + Duplicate { + path: String, + existing: Option, + reason: &'static str, + hash: Option, + }, + Failed(String, String), + Kept(Kept), + Ready(Box), +} + +#[derive(Debug)] +struct ReadyFile { + path: String, + stored: String, + info: ProbeInfo, + sidecar: Option, + placed: Option, +} + +impl Prepared { + /// Undo what the batch did on disk that only a commit would make permanent: files placed by a + /// Move (their sources were never touched). Copies stay, like copies of a failed commit. + pub fn rollback(&self) { + for it in &self.items { + if let PreparedItem::Ready(r) = it + && let Some(pl) = &r.placed + { + crate::import_move::rollback(pl); } - Some(id) => report.duplicates.push(Duplicate { path: f, existing: Some(id.0), reason: "path" }), - None => todo.push(f), } } - // files a preceding scan already probed are not read again (a network share is slow to read) - let cached: Vec> = todo.iter().map(|f| s.import_probes.remove(f)).collect(); - let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); - let mut fresh = probe_all(s.media.file_probe.as_ref(), &missing, &ScanProgress::default()).into_iter(); - let probed: Vec> = - cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); - crate::memory::release(); - let now = (s.clock)(); - let mut ops = promote; - // Move: placed at the destination, sources untouched until the records are committed - let mut placed: Vec = Vec::new(); - for (path, info) in todo.into_iter().zip(probed) { - let info = match info { - Ok(i) => i, - Err(e) => { - log::warn!("import {path}: {e}"); - report.failed.push((path, e)); + /// Entries in the batch (after expanding folders). + pub fn len(&self) -> usize { + self.items.len() + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } +} + +impl ImportJob { + /// Snapshot what an import with `opts` needs from the session (no file-system calls). Takes + /// the probes a preceding scan left ([`Session::import_probes`]). + pub fn new(s: &mut Session, opts: ImportOptions) -> crate::Result { + let mode = opts.mode; + // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of + // every added file in its own storage already, so "copy" there means "add". + let transfer = matches!(mode, ImportMode::Copy | ImportMode::Move); + let mode = if transfer && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode }; + if mode == ImportMode::Move && opts.local { + return Err(crate::EngineError::Other("browsing a folder never moves its files".into())); + } + let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); + // where copies go: the chosen folder, else the library's Originals/ + let copy_root = opts + .destination + .as_deref() + .filter(|d| !d.trim().is_empty()) + .map(std::path::PathBuf::from) + .or_else(|| lib_dir.as_ref().map(|l| l.join("Originals"))); + if matches!(mode, ImportMode::Copy | ImportMode::Move) && copy_root.is_none() { + return Err(crate::EngineError::Other("copying or moving needs a destination folder or an open library".into())); + } + let mut by_path = HashMap::new(); + let mut by_hash = HashMap::new(); + for p in s.catalog.photos() { + if let Source::File { path } = &p.source { + by_path.insert(path.clone(), (p.id, p.local)); + } + if let Some(h) = &p.content_hash { + by_hash.insert(h.clone(), Some(p.id)); + } + } + Ok(ImportJob { + seq: opts.rename_start.max(1), + opts, + mode, + lib_dir, + copy_root, + by_path, + by_hash, + probe: s.media.file_probe.clone(), + file_bytes: s.media.file_bytes.clone(), + cache: std::mem::take(&mut s.import_probes), + naming: s.xmp.naming, + now: (s.clock)(), + total: Default::default(), + done: Default::default(), + }) + } + + /// The file-system half of importing `paths`: expand folders, skip known paths, probe, skip + /// duplicates by content, read sidecars, copy or place (Move) the files. Nothing in the + /// catalog changes. Stops before the next file once `cancel` is set. + pub fn prepare(&mut self, paths: &[String], cancel: &std::sync::atomic::AtomicBool) -> Prepared { + let files = self.expand(paths); + self.prepare_files(files, cancel) + } + + /// Expand `paths` (folders recursively, the library's own folder skipped) into the files an + /// import would handle; counted in [`ImportJob::total`]. + pub fn expand(&self, paths: &[String]) -> Vec { + let files = expand(paths, self.lib_dir.as_deref()); + self.total.fetch_add(files.len(), std::sync::atomic::Ordering::Relaxed); + files + } + + /// When the import started ([`Session::clock`]), for [`commit_prepared`]. + pub fn now(&self) -> &str { + &self.now + } + + /// [`ImportJob::prepare`] for files [`ImportJob::expand`] returned. + pub fn prepare_files(&mut self, files: Vec, cancel: &std::sync::atomic::AtomicBool) -> Prepared { + use std::sync::atomic::Ordering::Relaxed; + let opts = self.opts.clone(); + let mode = self.mode; + let mut out = Prepared { scanned: files.len(), items: Vec::new() }; + let mut todo = Vec::new(); + for f in files { + match self.by_path.get(f.as_str()).copied() { + Some((id, true)) if !opts.local => { + // a file that was only browsed (Local) joins the library when it is imported for real + self.by_path.insert(f, (id, false)); + out.items.push(PreparedItem::Promote(id)); + self.done.fetch_add(1, Relaxed); + } + Some((id, _)) => { + out.items.push(PreparedItem::Duplicate { path: f, existing: Some(id), reason: "path", hash: None }); + self.done.fetch_add(1, Relaxed); + } + None => todo.push(f), + } + } + + // files a preceding scan already probed are not read again (a network share is slow to read) + let cached: Vec> = todo.iter().map(|f| self.cache.remove(f)).collect(); + let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); + let progress = ScanProgress::default(); + let mut fresh = probe_all(self.probe.as_ref(), &missing, &progress).into_iter(); + let probed: Vec> = + cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); + crate::memory::release(); + for (path, info) in todo.into_iter().zip(probed) { + if cancel.load(Relaxed) { + break; + } + self.done.fetch_add(1, Relaxed); + let mut info = match info { + Ok(i) => i, + Err(e) => { + log::warn!("import {path}: {e}"); + out.items.push(PreparedItem::Failed(path, e)); + continue; + } + }; + if let Some(h) = &info.content_hash + && !opts.local + && let Some(existing) = self.by_hash.get(h) + { + out.items.push(PreparedItem::Duplicate { path, existing: *existing, reason: "content", hash: Some(h.clone()) }); continue; } - }; - if let Some(h) = &info.content_hash - && !opts.local - && let Some(id) = by_hash.get(h) - { - report.duplicates.push(Duplicate { path, existing: Some(id.0), reason: "content" }); - continue; - } - // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files - // and names the copy when the file itself has none - let raw = info.kind == MediaKind::Raw; - let packet = crate::sidecar::find_sidecar(&path, s.xmp.naming) - .and_then(|f| std::fs::read_to_string(f).ok()) - .or_else(|| info.xmp.clone().filter(|_| raw)); - let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, raw).map(|sc| sc.resolve_label(&s.catalog)) { - Ok(sc) => (sc != crate::sidecar::SidecarData::default()).then_some(sc), - Err(e) => { - log::warn!("import {path}: XMP: {e}"); - None + // the XMP sidecar (or a raw's embedded XMP), read before copying: its capture time files + // and names the copy when the file itself has none + let raw = info.kind == MediaKind::Raw; + let packet = crate::sidecar::find_sidecar(&path, self.naming) + .and_then(|f| std::fs::read_to_string(f).ok()) + .or_else(|| info.xmp.clone().filter(|_| raw)); + let sidecar = packet.and_then(|x| match crate::sidecar::parse_sidecar(&x, raw) { + Ok(sc) => Some(sc), + Err(e) => { + log::warn!("import {path}: XMP: {e}"); + None + } + }); + if info.captured.is_none() { + info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone()); } - }); - let mut info = info; - if info.captured.is_none() { - info.captured = sidecar.as_ref().and_then(|sc| sc.captured.clone()); - } - let stored = match (mode, ©_root) { - (ImportMode::Copy | ImportMode::Move, Some(root)) - if !crate::import_move::inside(Path::new(&path), root) - && !lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) => + let mut placed = None; + let stored = match (mode, &self.copy_root) { + (ImportMode::Copy | ImportMode::Move, Some(root)) + if !crate::import_move::inside(Path::new(&path), root) + && !self.lib_dir.as_ref().is_some_and(|l| crate::import_move::inside(Path::new(&path), l)) => + { + // the templates see the photo as it will be catalogued (undated: the import time) + let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &self.now); + q.captured = info.captured.clone(); + q.meta = info.meta.clone(); + let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| { + let n = crate::rename::expand(t, &q, self.seq); + self.seq += 1; + n + }); + let folders = opts.organize.folders(&q); + if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) { + let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f)); + let name = name.unwrap_or(own); + match crate::import_move::place(Path::new(&path), &dir, &name) { + Ok(pl) => { + let dst = pl.dst.to_string_lossy().to_string(); + placed = Some(pl); + dst + } + Err(e) => { + out.items.push(PreparedItem::Failed(path, e)); + continue; + } + } + } else { + if mode == ImportMode::Move { + let reason = "a symbolic link: the file it points to was copied, the link and its target stay"; + out.items.push(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() })); + } + match copy_into(root, &path, &folders, name.as_deref()) { + Ok(p) => p, + Err(e) => { + out.items.push(PreparedItem::Failed(path, e)); + continue; + } + } + } + } + (ImportMode::Move, _) => { + let reason = "already inside the destination or the library: added where it is"; + out.items.push(PreparedItem::Kept(Kept { path: path.clone(), reason: reason.into() })); + path.clone() + } + _ => path.clone(), + }; + if let Some(h) = &info.content_hash { + self.by_hash.insert(h.clone(), None); + } + // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace) + let mut stored = stored; + if opts.convert_dng + && mode == ImportMode::Copy + && stored != path + && info.kind == MediaKind::Raw + && !info.format.eq_ignore_ascii_case("DNG") { - // the templates see the photo as it will be catalogued (undated: the import time) - let own = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let mut q = Photo::new(PhotoId(0), Source::File { path: path.clone() }, &own, &info.format, 0, 0, &now); - q.captured = info.captured.clone(); - q.meta = info.meta.clone(); - let name = opts.rename.as_deref().filter(|t| !t.trim().is_empty()).map(|t| { - let n = crate::rename::expand(t, &q, seq); - seq += 1; - n + match crate::cmd::convert::write_dng_with(self.file_bytes.as_ref(), &stored, String::new()) { + Ok(dng) => { + let _ = std::fs::remove_file(&stored); + stored = dng; + info.format = "DNG".into(); + } + Err(e) => log::warn!("import {path}: copy as DNG: {e}"), + } + } + out.items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed }))); + } + out + } +} + +/// The catalog half of an import: add the photos [`ImportJob::prepare`] readied (one undoable op; +/// a Move's sources are removed once it is saved). On a failed commit a Move's placed files are +/// taken back. +pub fn commit_prepared(s: &mut Session, opts: &ImportOptions, now: &str, prepared: Prepared) -> crate::Result { + let now = now.to_string(); + let mut report = ImportReport { scanned: prepared.scanned, ..Default::default() }; + let mut ops = Vec::new(); + // Move: placed at the destination, sources untouched until the records are committed + let mut placed: Vec = Vec::new(); + // photos added by this batch, by content (a duplicate of a file earlier in the import names it) + let mut new_hash: HashMap = HashMap::new(); + for it in prepared.items { + match it { + PreparedItem::Promote(id) => { + if s.catalog.photo(id).is_some_and(|p| p.local) { + ops.push(Op::SetLocal { id, local: false }); + } + report.imported.push(id.0); + } + PreparedItem::Duplicate { path, existing, reason, hash } => { + let existing = existing.or_else(|| { + let h = hash?; + new_hash.get(&h).copied().or_else(|| s.catalog.photos().find(|p| p.content_hash.as_deref() == Some(h.as_str())).map(|p| p.id)) }); - let folders = opts.organize.folders(&q); - if mode == ImportMode::Move && !crate::import_move::is_symlink(Path::new(&path)) { - let dir = folders.iter().fold(root.to_path_buf(), |d, f| d.join(f)); - let name = name.unwrap_or(own); - match crate::import_move::place(Path::new(&path), &dir, &name) { - Ok(pl) => { - let dst = pl.dst.to_string_lossy().to_string(); - placed.push(pl); - dst - } - Err(e) => { - report.failed.push((path, e)); - continue; - } - } - } else { - if mode == ImportMode::Move { - let reason = "a symbolic link: the file it points to was copied, the link and its target stay"; - report.kept.push(Kept { path: path.clone(), reason: reason.into() }); - } - match copy_into(root, &path, &folders, name.as_deref()) { - Ok(p) => p, - Err(e) => { - report.failed.push((path, e)); - continue; - } + report.duplicates.push(Duplicate { path, existing: existing.map(|i| i.0), reason }); + } + PreparedItem::Failed(path, e) => report.failed.push((path, e)), + PreparedItem::Kept(k) => report.kept.push(k), + PreparedItem::Ready(r) => { + let ReadyFile { path, stored, info, sidecar, placed: pl } = *r; + placed.extend(pl); + let id = s.catalog.alloc_photo_id(); + if let Some(h) = &info.content_hash { + new_hash.insert(h.clone(), id); + } + let sidecar = sidecar.map(|sc| sc.resolve_label(&s.catalog)).filter(|sc| *sc != crate::sidecar::SidecarData::default()); + // a copy is catalogued under its new name + let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); + let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now); + p.kind = info.kind; + p.file_size = info.file_size; + p.captured = info.captured; + p.meta = info.meta; + p.as_shot_wb = info.as_shot_wb; + p.content_hash = info.content_hash; + p.embedded_lens = info.embedded_lens; + p.preview_only = info.preview_only.clone(); + apply_import_defaults(s, &mut p); + if let Some(sc) = &sidecar { + crate::sidecar::merge_into(&mut p, sc, &now); + report.sidecars += 1; + } + if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { + crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); + } + for k in &opts.keywords { + let k = lightcraft_catalog::keywords::clean(k); + if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) { + p.meta.keywords.push(k); } } - } - (ImportMode::Move, _) => { - let reason = "already inside the destination or the library: added where it is"; - report.kept.push(Kept { path: path.clone(), reason: reason.into() }); - path.clone() - } - _ => path.clone(), - }; - let id = s.catalog.alloc_photo_id(); - if let Some(h) = &info.content_hash { - by_hash.insert(h.clone(), id); - } - // Copy as DNG: the copied raw becomes a DNG (the copy is ours to replace) - let mut stored = stored; - if opts.convert_dng && mode == ImportMode::Copy && stored != path && info.kind == MediaKind::Raw && !info.format.eq_ignore_ascii_case("DNG") { - match crate::cmd::convert::write_dng_for(s, &stored, String::new()) { - Ok(dng) => { - let _ = std::fs::remove_file(&stored); - stored = dng; - info.format = "DNG".into(); + if let Some(preset) = &opts.preset { + let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0)); + let label = format!("Preset: {}", preset.name); + p.develop = d.clone(); + p.edited = Some(now.clone()); + p.history.push(lightcraft_catalog::HistoryStep { label, settings: d }); } - Err(e) => log::warn!("import {path}: copy as DNG: {e}"), + report.imported.push(id.0); + p.local = opts.local; + if opts.local { + // the state as browsed: a later change keeps the record from being forgotten + p.set_local_baseline(); + } + ops.push(Op::AddPhoto { photo: Box::new(p) }); } } - // a copy is catalogued under its new name - let name = Path::new(&stored).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); - let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now); - p.kind = info.kind; - p.file_size = info.file_size; - p.captured = info.captured; - p.meta = info.meta; - p.as_shot_wb = info.as_shot_wb; - p.content_hash = info.content_hash; - p.embedded_lens = info.embedded_lens; - p.preview_only = info.preview_only.clone(); - apply_import_defaults(s, &mut p); - if let Some(sc) = &sidecar { - crate::sidecar::merge_into(&mut p, sc, &now); - report.sidecars += 1; - } - if let Some(mp) = opts.metadata_preset.as_ref().and_then(|n| s.metadata_presets.iter().find(|m| m.name.eq_ignore_ascii_case(n))) { - crate::cmd::metadata::apply_to(&mut p.meta, &mp.fields); - } - for k in &opts.keywords { - let k = lightcraft_catalog::keywords::clean(k); - if !k.is_empty() && !p.meta.keywords.iter().any(|x| x.eq_ignore_ascii_case(&k)) { - p.meta.keywords.push(k); - } - } - if let Some(preset) = &opts.preset { - let d = std::sync::Arc::new(preset.apply(&p.develop, 1.0)); - let label = format!("Preset: {}", preset.name); - p.develop = d.clone(); - p.edited = Some(now.clone()); - p.history.push(lightcraft_catalog::HistoryStep { label, settings: d }); - } - report.imported.push(id.0); - p.local = opts.local; - if opts.local { - // the state as browsed: a later change keeps the record from being forgotten - p.set_local_baseline(); - } - ops.push(Op::AddPhoto { photo: Box::new(p) }); } let log0 = s.pending_log.len(); if !ops.is_empty() diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index bb4760ada..ecdb078d5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -10,6 +10,7 @@ //! run them off the UI thread. #![forbid(unsafe_code)] +pub mod availability; pub mod cmd; pub mod crs; pub mod crs_masks; @@ -272,11 +273,22 @@ impl Session { (spec.enabled)(self).map_err(|why| EngineError::Disabled(id.to_string(), why))?; let empty = Value::Object(Default::default()); let params = if params.is_null() { &empty } else { params }; + self.run_command(id, spec.journal.then_some(params), |s| (spec.run)(s, params)) + } + + /// Run `f` as command `id` with what [`Session::execute`] does around every command: the + /// panic guard, auto versions, XMP sidecar auto-write and the durable save (a failed save is + /// `NotSaved`). Not journaled. The app's import task commits its batches this way. + pub fn execute_fn(&mut self, id: &str, f: impl FnOnce(&mut Session) -> Result) -> Result { + self.run_command(id, None, f) + } + + fn run_command(&mut self, id: &str, journal: Option<&Value>, f: impl FnOnce(&mut Session) -> Result) -> Result { let log_start = self.pending_log.len(); let was_active = self.active(); self.depth += 1; // last-resort guard: a panic in a command is that command's error, not a crash - let r = guard::catch(&format!("`{id}`"), || (spec.run)(self, params)).unwrap_or_else(|e| Err(EngineError::Other(e))); + let r = guard::catch(&format!("`{id}`"), || f(self)).unwrap_or_else(|e| Err(EngineError::Other(e))); self.depth -= 1; if self.depth == 0 && was_active.is_some() && self.active() != was_active { self.previous_active = was_active; @@ -284,7 +296,10 @@ impl Session { self.auto_version(left); } } - if r.is_ok() && spec.journal && self.depth == 0 { + if r.is_ok() + && let Some(params) = journal + && self.depth == 0 + { self.journal.push((id.to_string(), params.clone())); if self.journal.len() > 10_000 { self.journal.drain(..1000); @@ -560,7 +575,11 @@ impl Session { /// Photos shown in the grid/filmstrip for the current source, filter and sort. pub fn visible(&mut self) -> &[PhotoId] { - let key = (self.catalog.revision, format!("{:?}|{:?}|{:?}|{:?}", self.source, self.filter, self.sort, self.browse)); + let mut key = (self.catalog.revision, format!("{:?}|{:?}|{:?}|{:?}", self.source, self.filter, self.sort, self.browse)); + if self.source == LibrarySource::Missing && self.media.availability.is_background() { + // the view fills in as the background checks find files gone + key.1.push_str(&format!("|{}", self.media.availability.generation())); + } if self.visible_key.as_ref() != Some(&key) { // "in the last N days" rules count back from the session's clock lightcraft_catalog::rules::set_now(Some((self.clock)())); @@ -595,8 +614,8 @@ impl Session { } if self.source == LibrarySource::Missing { // only the photos the query kept (library photos, not Local browse records) are checked - let cat = &self.catalog; - visible.retain(|id| cmd::missing::is_missing(cat, *id)); + let (cat, avail) = (&self.catalog, &self.media.availability); + visible.retain(|id| cmd::missing::is_missing(cat, avail, *id)); } if self.source != LibrarySource::RecentlyDeleted { visible = self.catalog.arrange_stacks(&visible); diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 18e67fafd..561b7b24d 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -82,6 +82,9 @@ pub enum SourceRef { path: String, max_edge: usize, loader: Option, + /// The photo's smart preview, used when the original can't be read (offline drive). The + /// UI thread never checks whether the original is there; the render worker finds out. + fallback: Option, }, /// A smart preview standing in for a missing original. Smart { @@ -94,10 +97,17 @@ impl SourceRef { match self { SourceRef::Loaded(a) => Ok(a.clone()), SourceRef::Demo { scene, max_edge } => Ok(Arc::new(scene.render_fit(*max_edge))), - SourceRef::File { path, max_edge, loader } => match loader { - Some(l) => l(path, *max_edge).map(|(img, _)| Arc::new(img)), - None => Err(format!("no decoder available for {path}")), - }, + SourceRef::File { path, max_edge, loader, fallback } => { + let r = match loader { + Some(l) => l(path, *max_edge).map(|(img, _)| Arc::new(img)), + None => Err(format!("no decoder available for {path}")), + }; + match (r, fallback) { + #[cfg(not(target_arch = "wasm32"))] + (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")), + (r, _) => r, + } + } #[cfg(not(target_arch = "wasm32"))] SourceRef::Smart { path } => crate::smart::load(path), #[cfg(target_arch = "wasm32")] @@ -125,6 +135,9 @@ pub struct MediaCache { pub file_bytes: Option, /// The library's smart previews folder (originals offline: render from the proxy). pub smart_dir: Option, + /// Whether originals (and smart previews) are on disk: cached and checked off the UI thread + /// in the app (see [`crate::availability`]). + pub availability: crate::availability::Availability, scenes: Vec, /// Rendered thumbnails (memory, plus disk once a library is attached). pub rendered: Arc, @@ -144,6 +157,7 @@ impl Default for MediaCache { preview_loader: None, file_bytes: None, smart_dir: None, + availability: Default::default(), scenes: Vec::new(), rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))), } @@ -284,15 +298,22 @@ impl MediaCache { (Source::Demo { .. }, SourceLevel::Full) => p.width.max(p.height).max(1) as usize, _ => usize::MAX, }); - // the original is offline: its smart preview, when there is one + // the original is offline: its smart preview, when there is one. Only cached answers are + // used here (no file-system call on the UI thread in the app); an original not known to + // be offline is tried first and its smart preview is the render worker's fallback. #[cfg(not(target_arch = "wasm32"))] - if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) - && !std::path::Path::new(path).exists() - { + if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) { let sp = dir.join(crate::smart::file_name(p)); - if crate::smart::is_valid(&sp) { + // (a proxy cut short fails to load, like the offline original; the worker's fallback + // in `SourceRef::load` only uses a proxy that passes `smart::is_valid`) + if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) { return SourceRef::Smart { path: sp }; } + let mut r = self.origin_ref(&p.source, max_edge); + if let SourceRef::File { fallback, .. } = &mut r { + *fallback = Some(sp); + } + return r; } self.origin_ref(&p.source, max_edge) } @@ -307,10 +328,10 @@ impl MediaCache { } match self.scenes.iter().find(|s| s.id == *scene) { Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge }, - None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None }, + None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None }, } } - Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone() }, + Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None }, } } } diff --git a/crates/engine/src/rename.rs b/crates/engine/src/rename.rs index 511aa8b49..27fc5fb5f 100644 --- a/crates/engine/src/rename.rs +++ b/crates/engine/src/rename.rs @@ -538,6 +538,92 @@ fn set_file_ops_to(op: &Op, to: &HashSet<&str>, out: &mut Vec) { } } +/// The photos among `ids` a rename handles, in order: photos sharing one file (virtual copies) +/// rename once. No file-system calls. +pub fn rename_photos(cat: &lightcraft_catalog::Catalog, ids: &[PhotoId]) -> Vec> { + let mut seen_paths: HashSet = HashSet::new(); + ids.iter() + .filter_map(|id| cat.photo(*id)) + .filter(|p| match &p.source { + Source::File { path } => seen_paths.insert(path.clone()), + _ => true, + }) + .cloned() + .collect() +} + +/// [`Session::plan_rename`] for [`rename_photos`], with the "is this name taken on disk?" check +/// supplied: the Rename dialog plans its preview on a worker thread (a check can block on a slow +/// drive). Whether a case-only change names this very file is decided on the disk. +pub fn plan_rename_photos(photos: &[std::sync::Arc], template: &str, start: usize, exists: &dyn Fn(&str) -> bool) -> Vec { + plan_rename_core(photos, template, start, &|p| exists(&p.to_string_lossy()), &|a, b| RealFs.same_file(a, b)) +} + +/// Plan renaming `photos`: `exists` says whether something answers to a path, `same_file` whether +/// two spellings name one file (see [`MoveFs`]). +fn plan_rename_core( + photos: &[std::sync::Arc], + template: &str, + start: usize, + exists: &dyn Fn(&Path) -> bool, + same_file: &dyn Fn(&Path, &Path) -> bool, +) -> Vec { + let mut taken: HashSet = HashSet::new(); // lower-case target paths/names claimed in this batch + let mut out = Vec::new(); + for (seq, p) in (start..).zip(photos) { + let want = expand(template, p, seq); + let (stem, ext) = split_ext(&want); + let (stem, ext) = (stem.to_string(), ext.to_string()); + let candidate = |k: usize| { + if k == 0 { + want.clone() + } else if ext.is_empty() { + format!("{stem}-{k}") + } else { + format!("{stem}-{k}.{ext}") + } + }; + let (to, to_path) = match &p.source { + Source::File { path } => { + let dir = Path::new(path).parent().map(Path::to_path_buf).unwrap_or_default(); + let mut k = 0; + loop { + let name = candidate(k); + let tp = dir.join(&name).to_string_lossy().to_string(); + let key = tp.to_lowercase(); + // this very file, maybe spelled differently (case-insensitive volume)? By + // identity: on a case-sensitive volume `img_1.jpg` may be another photo + let (from, to) = (Path::new(path), Path::new(&tp)); + let same = tp == *path || (from != to && key == path.to_lowercase() && exists(to) && same_file(from, to)); + // free: not claimed in this batch and not on disk (unless it is this very file). + // A file this batch moves away still counts as taken: simple and safe. + if !taken.contains(&key) && (same || !exists(to)) { + taken.insert(key); + break (name, Some(tp)); + } + k += 1; + } + } + Source::Demo { .. } => { + let mut k = 0; + loop { + let name = candidate(k); + if taken.insert(format!("demo:{}", name.to_lowercase())) { + break (name, None); + } + k += 1; + } + } + }; + let from_path = match &p.source { + Source::File { path } => Some(path.clone()), + _ => None, + }; + out.push(RenamePlan { id: p.id.0, from: p.file_name.clone(), to, from_path, to_path }); + } + out +} + impl Session { /// Plan renaming `ids` with `template` (sequence numbers from `start`), resolving collisions. pub fn plan_rename(&self, ids: &[PhotoId], template: &str, start: usize) -> Vec { @@ -546,70 +632,7 @@ impl Session { /// [`Session::plan_rename`] on the file system `fs`. pub(crate) fn plan_rename_with(&self, fs: &dyn MoveFs, ids: &[PhotoId], template: &str, start: usize) -> Vec { - // photos sharing one file (virtual copies) rename once - let mut seen_paths: HashSet = HashSet::new(); - let mut taken: HashSet = HashSet::new(); // lower-case target paths/names claimed in this batch - let mut out = Vec::new(); - let mut seq = start; - for id in ids { - let Some(p) = self.catalog.photo(*id) else { continue }; - if let Source::File { path } = &p.source - && !seen_paths.insert(path.clone()) - { - continue; - } - let want = expand(template, p, seq); - seq += 1; - let (stem, ext) = split_ext(&want); - let (stem, ext) = (stem.to_string(), ext.to_string()); - let candidate = |k: usize| { - if k == 0 { - want.clone() - } else if ext.is_empty() { - format!("{stem}-{k}") - } else { - format!("{stem}-{k}.{ext}") - } - }; - let (to, to_path) = match &p.source { - Source::File { path } => { - let dir = Path::new(path).parent().map(Path::to_path_buf).unwrap_or_default(); - let mut k = 0; - loop { - let name = candidate(k); - let tp = dir.join(&name).to_string_lossy().to_string(); - let key = tp.to_lowercase(); - // this very file, maybe spelled differently (case-insensitive volume)? By - // identity: on a case-sensitive volume `img_1.jpg` may be another photo - let same = tp == *path || is_respelling(fs, Path::new(path), Path::new(&tp)); - // free: not claimed in this batch and not on disk (unless it is this very file). - // A file this batch moves away still counts as taken: simple and safe. - let on_disk = fs.exists(Path::new(&tp)) && !same; - if !taken.contains(&key) && !on_disk { - taken.insert(key); - break (name, Some(tp)); - } - k += 1; - } - } - Source::Demo { .. } => { - let mut k = 0; - loop { - let name = candidate(k); - if taken.insert(format!("demo:{}", name.to_lowercase())) { - break (name, None); - } - k += 1; - } - } - }; - let from_path = match &p.source { - Source::File { path } => Some(path.clone()), - _ => None, - }; - out.push(RenamePlan { id: id.0, from: p.file_name.clone(), to, from_path, to_path }); - } - out + plan_rename_core(&rename_photos(&self.catalog, ids), template, start, &|p| fs.exists(p), &|a, b| fs.same_file(a, b)) } /// Carry out `plans`: move the files (rolled back on failure), then commit one undoable op diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs index e83a4d344..dd3f7ca1f 100644 --- a/crates/engine/src/tests_import.rs +++ b/crates/engine/src/tests_import.rs @@ -144,11 +144,10 @@ fn browsing_a_folder_lists_its_photos_without_adding_them() { let _ = std::fs::remove_dir_all(&dir); } -/// Issue #105: Find Missing finds renamed files by content, prefers the content match over a -/// same-name same-size impostor, and skips (reports) photos it can't tell apart. -#[test] -fn find_missing_matches_renamed_files_by_content() { - let dir = temp_dir("missing-content"); +/// Photos a.png, b.png, c.png imported from `old`, then: a renamed into `moved`; b renamed, with +/// an impostor (same name and size, other bytes) under its old name; c gone, with two impostors. +fn renamed_and_ambiguous_scene(tag: &str) -> (std::path::PathBuf, Session) { + let dir = temp_dir(tag); std::fs::create_dir_all(dir.join("old")).unwrap(); std::fs::create_dir_all(dir.join("moved/real")).unwrap(); for (n, seed) in [("a.png", 1), ("b.png", 2), ("c.png", 3)] { @@ -156,20 +155,26 @@ fn find_missing_matches_renamed_files_by_content() { } let mut s = Session::new().with_fs(); s.execute("library.import", &json!({"paths": [dir.join("old").to_string_lossy()]})).unwrap(); - // an impostor: same name and size, other bytes let impostor = |src: &Path, dst: &Path| { let mut b = std::fs::read(src).unwrap(); let k = b.len() - 20; b[k] ^= 0xff; std::fs::write(dst, b).unwrap(); }; - // a: renamed; b: renamed, with an impostor under its old name; c: gone, two impostors std::fs::rename(dir.join("old/a.png"), dir.join("moved/Trip-001.png")).unwrap(); impostor(&dir.join("old/b.png"), &dir.join("moved/b.png")); std::fs::rename(dir.join("old/b.png"), dir.join("moved/real/Trip-002.png")).unwrap(); impostor(&dir.join("old/c.png"), &dir.join("moved/c.png")); impostor(&dir.join("old/c.png"), &dir.join("moved/real/c.png")); std::fs::remove_file(dir.join("old/c.png")).unwrap(); + (dir, s) +} + +/// Issue #105: Find Missing finds renamed files by content, prefers the content match over a +/// same-name same-size impostor, and skips (reports) photos it can't tell apart. +#[test] +fn find_missing_matches_renamed_files_by_content() { + let (dir, mut s) = renamed_and_ambiguous_scene("missing-content"); let r = s.execute("library.findMissing", &json!({"folder": dir.join("moved").to_string_lossy()})).unwrap(); let found = r["found"].as_array().unwrap(); assert_eq!(found.len(), 2, "{r}"); @@ -183,6 +188,55 @@ fn find_missing_matches_renamed_files_by_content() { let _ = std::fs::remove_dir_all(&dir); } +/// Issue #104: the app plans Find Missing on a worker thread (no session) and relinks the plan +/// afterwards; that gives exactly what the synchronous command gives — renamed files by content, +/// impostors and look-alikes skipped (`ambiguous`). +#[test] +fn find_missing_planned_on_a_worker_matches_the_command() { + use crate::cmd::missing::{find_candidates, plan_find_missing}; + let (dir, mut s) = renamed_and_ambiguous_scene("missing-worker"); + let folder = dir.join("moved").to_string_lossy().to_string(); + let candidates = find_candidates(&s.catalog); + let f = folder.clone(); + let plan = std::thread::spawn(move || plan_find_missing(&candidates, &f)).join().unwrap().unwrap(); + assert_eq!((plan.found.len(), plan.missing, plan.ambiguous.len()), (2, 1, 1), "{plan:?}"); + let applied = s.execute("library.findMissing", &plan.to_json()).unwrap(); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(s.execute("library.missing", &json!({})).unwrap().as_array().map(Vec::len), Some(3), "undo restores all three"); + let direct = s.execute("library.findMissing", &json!({"folder": folder})).unwrap(); + assert_eq!(applied, direct); + assert_eq!(direct["found"].as_array().map(Vec::len), Some(2), "{direct}"); + assert_eq!(direct["ambiguous"].as_array().map(Vec::len), Some(1), "{direct}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Issue #104: a plan made on a worker is re-checked when it is applied: a photo relinked +/// meanwhile is left alone, and a file that is now another photo's original isn't used. +#[test] +fn find_missing_plan_is_rechecked_when_applied() { + use crate::cmd::missing::{find_candidates, plan_find_missing}; + let (dir, mut s) = renamed_and_ambiguous_scene("missing-recheck"); + let plan = plan_find_missing(&find_candidates(&s.catalog), &dir.join("moved").to_string_lossy()).unwrap(); + let to = |name: &str| plan.found.iter().find(|f| f.from.ends_with(name)).unwrap().clone(); + let (a, b) = (to("a.png"), to("b.png")); + // meanwhile: a is relinked by hand (elsewhere), and the ambiguous c is pointed at b's file + std::fs::copy(&a.to, dir.join("a-elsewhere.png")).unwrap(); + s.execute("photo.relink", &json!({"id": a.id.0, "path": dir.join("a-elsewhere.png").to_string_lossy()})).unwrap(); + let c = plan.ambiguous[0].id; + s.execute("photo.relink", &json!({"id": c.0, "path": b.to})).unwrap(); + let r = s.execute("library.findMissing", &plan.to_json()).unwrap(); + assert_eq!(r["found"].as_array().map(Vec::len), Some(0), "{r}"); + assert_eq!(r["ambiguous"].as_array().map(Vec::len), Some(0), "c isn't missing any more: {r}"); + assert_eq!(r["missing"], 1, "b stays missing: its file is c's now: {r}"); + let path_of = |id: lightcraft_catalog::PhotoId| match &s.catalog.photo(id).unwrap().source { + lightcraft_catalog::Source::File { path } => path.clone(), + _ => String::new(), + }; + assert_eq!(path_of(a.id), dir.join("a-elsewhere.png").to_string_lossy()); + assert_eq!(path_of(b.id), b.from); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn missing_files_are_found_and_relinked() { let dir = std::env::temp_dir().join(format!("lc-missing-{}", std::process::id())); @@ -425,6 +479,20 @@ fn damaged_smart_previews_are_rebuilt_and_failed_writes_leave_none() { let r = s.execute("library.smartPreviews", &json!({})).unwrap(); assert_eq!((r["built"].as_u64(), r["repaired"].as_u64()), (Some(1), Some(1)), "{r}"); assert_eq!(std::fs::read(&file).unwrap(), full, "rebuilt whole"); + // the app's background build (issue #104) checks and repairs the same way, on a worker thread + std::fs::write(&file, &full[..full.len() / 2]).unwrap(); + let r = s.execute("library.smartPreviews", &json!({"background": true})).unwrap(); + assert_eq!(r["what"], "smart previews", "{r}"); + let t0 = std::time::Instant::now(); + let r = loop { + let r = s.execute("library.previewProgress", &json!({})).unwrap(); + if r["running"] == false || t0.elapsed() > std::time::Duration::from_secs(60) { + break r; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + }; + assert_eq!((r["done"].as_u64(), r["failed"].as_u64(), r["repaired"].as_u64()), (Some(1), Some(0), Some(1)), "{r}"); + assert_eq!(std::fs::read(&file).unwrap(), full, "rebuilt whole in the background"); let _ = std::fs::remove_dir_all(&src); let _ = std::fs::remove_dir_all(&lib); } diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index feaa847c4..9e1f09252 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -88,6 +88,8 @@ pub fn inspect(app: &LightcraftApp, ctx: &egui::Context) -> Value { "libraryProblem": app.library_problem.as_ref().map(crate::panels::library_problem::LibraryProblem::to_json), "scan": app.scan.as_ref().map(crate::import::ScanTask::status), "export": {"running": app.export.as_ref().map(crate::export_task::ExportTask::status), "last": app.last_export_result}, + "import": app.import.as_ref().map(crate::import::ImportTask::status), + "tasks": app.tasks.labels(), "memory": memory(app), }) } diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index 893c3a408..a8c6e062d 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -176,6 +176,7 @@ impl Headless { || self.app.scan.is_some() || self.app.import.is_some() || self.app.export.is_some() + || !self.app.tasks.is_empty() || !self.app.synthetic.is_empty() || !self.events.is_empty() } diff --git a/crates/ui-egui/src/import.rs b/crates/ui-egui/src/import.rs index a098d26f7..2058eb97f 100644 --- a/crates/ui-egui/src/import.rs +++ b/crates/ui-egui/src/import.rs @@ -3,8 +3,9 @@ //! a grid of thumbnails with checkboxes (duplicates marked and unchecked), the destination (add in //! place / copy or move into the library's `Originals/` or a chosen folder, filed by day, by month, //! into one folder or by a custom folder template, optionally renamed), an album -//! (existing or new), a preset and keywords to apply. Importing runs in small batches, one per -//! frame, with a progress window; the whole import is one undo step. +//! (existing or new), a preset and keywords to apply. Importing runs on a worker thread (files are +//! probed, copied or moved there) and its batches join the catalog between frames, with a progress +//! window and Cancel; the whole import is one undo step. use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with}; @@ -17,7 +18,7 @@ use crate::render::Slot; use crate::theme::Tokens; use crate::widgets::register; -/// Files per batch (one batch per frame, so the progress window updates). +/// Files per batch (each batch joins the catalog as it is ready, so the progress window updates). pub(crate) const BATCH: usize = 8; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -91,9 +92,13 @@ impl ImportDialog { } } -/// A running import (see the module docs). +/// A running import (see the module docs). The file-system work — expanding folders, probing, +/// reading sidecars, copying or placing files — runs on a worker thread +/// ([`lightcraft_engine::import::ImportJob`]); each batch it readies is added to the catalog on the +/// UI thread between frames, so a slow drive never stalls the window. #[derive(Debug, Default)] pub struct ImportTask { + /// Files and folders to import (handed to the worker when the task starts). queue: Vec, pub total: usize, pub done: usize, @@ -109,6 +114,156 @@ pub struct ImportTask { /// Reading a folder for the Local view: the photos stay out of the library, and nothing is /// selected or announced as added. browse: bool, + /// Cancel was pressed: no further files are started; batches already readied are added. + pub cancelled: bool, + /// Auto Import (the watched folder): the selection stays as it is, a short toast when done. + auto: bool, + /// Auto Import: the selection to keep. + keep_selection: Option, + run: Option, +} + +impl ImportTask { + /// An import of `paths` (files or folders) with `library.import` params (without `paths`). + pub fn new(paths: Vec, params: Value, undo0: usize, browse: bool) -> Self { + ImportTask { total: paths.len(), queue: paths, params, undo0, browse, ..Default::default() } + } + + /// An Auto Import (see [`ImportTask::auto`]). + pub fn auto(mut self) -> Self { + self.auto = true; + self + } + + /// `{done, total, imported, cancelled}` for `ui.inspect`. + pub fn status(&self) -> Value { + json!({"done": self.done, "total": self.total, "imported": self.imported, "cancelled": self.cancelled}) + } +} + +/// Where an import's batches are readied: a worker thread, or (browser build: no threads, and no +/// slow drives) inline, one batch per frame. +enum Runner { + #[cfg(not(target_arch = "wasm32"))] + Thread(std::sync::mpsc::Receiver), + #[cfg(target_arch = "wasm32")] + Inline { job: Box, queue: Vec, files: Option> }, +} + +struct ImportRun { + runner: Runner, + cancel: std::sync::Arc, + total: std::sync::Arc, + done: std::sync::Arc, + opts: lightcraft_engine::import::ImportOptions, + now: String, + album: Option, + album_name: Option, +} + +impl std::fmt::Debug for ImportRun { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ImportRun").field("cancel", &self.cancel.load(Ordering::Relaxed)).finish() + } +} + +impl Drop for ImportRun { + /// A dropped import (replaced, or the app closing) stops its worker at the next file. + fn drop(&mut self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +impl ImportRun { + fn start( + job: lightcraft_engine::import::ImportJob, + queue: Vec, + album: Option, + album_name: Option, + ctx: &egui::Context, + ) -> Result { + let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (total, done) = (job.total.clone(), job.done.clone()); + let (opts, now) = (job.opts.clone(), job.now().to_string()); + let runner = Self::spawn(job, queue, cancel.clone(), ctx)?; + Ok(ImportRun { runner, cancel, total, done, opts, now, album, album_name }) + } + + #[cfg(not(target_arch = "wasm32"))] + fn spawn( + mut job: lightcraft_engine::import::ImportJob, + queue: Vec, + cancel: std::sync::Arc, + ctx: &egui::Context, + ) -> Result { + let (tx, rx) = std::sync::mpsc::channel(); + let ctx = ctx.clone(); + let work = move || { + let files = job.expand(&queue); + for chunk in files.chunks(BATCH) { + if cancel.load(Ordering::Relaxed) { + break; + } + let prepared = match lightcraft_engine::guard::catch("import", || job.prepare_files(chunk.to_vec(), &cancel)) { + Ok(p) => p, + Err(_) => break, // logged; the files readied so far are added + }; + if let Err(unsent) = tx.send(prepared) { + // the import was dropped: nothing will add these, so take back what a Move placed + unsent.0.rollback(); + break; + } + ctx.request_repaint(); + } + ctx.request_repaint(); + }; + std::thread::Builder::new().name("lc-import".into()).spawn(work).map(|_| Runner::Thread(rx)).map_err(|e| format!("could not start: {e}")) + } + + #[cfg(target_arch = "wasm32")] + fn spawn( + job: lightcraft_engine::import::ImportJob, + queue: Vec, + _cancel: std::sync::Arc, + _ctx: &egui::Context, + ) -> Result { + Ok(Runner::Inline { job: Box::new(job), queue, files: None }) + } + + /// Readied batches (at most a few per frame). `None` once the worker is done. + fn next_batches(&mut self) -> Option> { + match &mut self.runner { + #[cfg(not(target_arch = "wasm32"))] + Runner::Thread(rx) => { + let mut out = Vec::new(); + while out.len() < 4 { + match rx.try_recv() { + Ok(p) => out.push(p), + Err(std::sync::mpsc::TryRecvError::Empty) => return Some(out), + Err(std::sync::mpsc::TryRecvError::Disconnected) => return if out.is_empty() { None } else { Some(out) }, + } + } + Some(out) + } + #[cfg(target_arch = "wasm32")] + Runner::Inline { job, queue, files } => { + if self.cancel.load(Ordering::Relaxed) { + return None; + } + if files.is_none() { + *files = Some(job.expand(queue).into()); + return Some(Vec::new()); + } + let Some(files) = files.as_mut() else { return None }; + if files.is_empty() { + return None; + } + let n = files.len().min(BATCH); + let batch: Vec = files.drain(..n).collect(); + Some(vec![job.prepare_files(batch, &self.cancel)]) + } + } + } } /// A folder scan running on a worker thread (a network share can take minutes to read; the @@ -232,7 +387,8 @@ pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) { if !queue.is_empty() { let undo0 = app.session.undo.len(); let total = queue.len(); - app.import = Some(ImportTask { queue, total, params: json!({"mode": "add", "local": true}), undo0, browse: true, ..Default::default() }); + let _ = total; + app.import = Some(ImportTask::new(queue, json!({"mode": "add", "local": true}), undo0, true)); } return; } @@ -326,28 +482,82 @@ pub fn start(app: &mut LightcraftApp, d: &ImportDialog) -> Result } } let total = queue.len(); - app.import = Some(ImportTask { queue, total, params, undo0, ..Default::default() }); + app.import = Some(ImportTask::new(queue, params, undo0, false)); app.renderer.forget_imports(); Ok(json!({"importing": total})) } -/// Run one batch of the import in progress (called every frame). -pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { - let Some(task) = app.import.as_mut() else { return }; - // (browsing reads a sidecar per file, slow on a network share: keep each frame short) - let n = task.queue.len().min(if task.browse { 2 } else { BATCH }); - let batch: Vec = task.queue.drain(..n).collect(); - let mut p = task.params.clone(); - p["paths"] = json!(batch); - // renamed copies keep counting across batches - if p.get("rename").is_some() { - p["renameStart"] = json!(1 + task.imported); +/// Start importing `paths` (files or folders) in the background, e.g. dropped on the window. +pub fn start_paths(app: &mut LightcraftApp, paths: Vec) -> Result { + if app.import.is_some() || app.scan.as_ref().is_some_and(|t| !t.browse) { + return Err("an import is running".into()); } - let r = app.session.execute("library.import", &p); - let Some(task) = app.import.as_mut() else { return }; - task.done += n; + let undo0 = app.session.undo.len(); + let total = paths.len(); + app.import = Some(ImportTask::new(paths, json!({"mode": "add"}), undo0, false)); + Ok(json!({"importing": total})) +} + +/// Advance the import in progress (called every frame): start its worker, then add the batches it +/// has readied to the catalog. +pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(mut task) = app.import.take() else { return }; + if task.run.is_none() { + let mut p = task.params.clone(); + p["paths"] = json!(task.queue); + let started = lightcraft_engine::cmd::library::import_params(&app.session, &p) + .and_then(|req| Ok((lightcraft_engine::import::ImportJob::new(&mut app.session, req.opts)?, req.album, req.album_name))) + .map_err(|e| e.to_string()) + .and_then(|(job, album, name)| ImportRun::start(job, std::mem::take(&mut task.queue), album, name, ctx)); + if task.auto { + task.keep_selection = Some(app.session.selection.clone()); + } + match started { + Ok(run) => task.run = Some(run), + Err(e) => { + log::warn!("import: {e}"); + app.toast(ctx, format!("Import failed: {e}")); + return; + } + } + } + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + let Some(run) = task.run.as_mut() else { return }; + let batches = run.next_batches(); + let finished = batches.is_none(); + for prepared in batches.into_iter().flatten() { + commit_batch(app, &mut task, prepared); + // arriving photos don't take over the selection + if let Some(sel) = &task.keep_selection { + app.session.selection = sel.clone(); + } + } + if let Some(run) = &task.run { + task.total = run.total.load(Ordering::Relaxed).max(task.total); + task.done = run.done.load(Ordering::Relaxed); + } + if !finished { + app.import = Some(task); + return; + } + finish(app, ctx, task); +} + +/// Add one readied batch to the catalog (an undo step merged into the import's at the end). +fn commit_batch(app: &mut LightcraftApp, task: &mut ImportTask, prepared: lightcraft_engine::import::Prepared) { + let Some(run) = task.run.as_ref() else { return }; + let n = prepared.len(); + let (opts, now, album, album_name) = (&run.opts, run.now.as_str(), run.album, run.album_name.clone()); + let r = app.session.execute_fn("library.import", |s| { + let report = lightcraft_engine::import::commit_prepared(s, opts, now, prepared)?; + lightcraft_engine::cmd::library::import_batch_done(s, report, album, album_name.as_deref()) + }); match r { Ok(v) => { + // a new album (named in the params) is created with the first photos; later batches join it + if let (Some(a), Some(run)) = (v["album"].as_u64(), task.run.as_mut()) { + run.album = Some(a); + } let len = |k: &str| v[k].as_array().map_or(0, Vec::len); task.imported += len("imported"); task.duplicates += len("duplicates"); @@ -366,14 +576,19 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { task.failed += n; } } - ctx.request_repaint(); - if !task.queue.is_empty() { - return; - } - let Some(task) = app.import.take() else { return }; +} + +/// The import is done (or cancelled): one undo step, select the first photo, say what happened. +fn finish(app: &mut LightcraftApp, ctx: &egui::Context, task: ImportTask) { let steps = app.session.undo.len().saturating_sub(task.undo0); let label = format!("Add {} Photo{}", task.imported, if task.imported == 1 { "" } else { "s" }); app.session.merge_undo(steps, &label); + if task.auto { + if task.imported > 0 { + app.toast(ctx, format!("Auto Import: added {} photo{}", task.imported, if task.imported == 1 { "" } else { "s" })); + } + return; + } if task.browse { if task.failed > 0 { app.toast(ctx, format!("{} photo{} not readable", task.failed, if task.failed == 1 { "" } else { "s" })); @@ -384,7 +599,11 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { let _ = app.run("library.select", json!({"ids": [f]})); } let plural = |n: usize| if n == 1 { "" } else { "s" }; - let mut msg = format!("Imported {} photo{}", task.imported, plural(task.imported)); + let mut msg = if task.cancelled { + format!("Import cancelled · {} photo{} added", task.imported, plural(task.imported)) + } else { + format!("Imported {} photo{}", task.imported, plural(task.imported)) + }; if task.params["mode"] == "move" { msg.push_str(&format!(" · {} moved", task.moved)); if task.kept > 0 { @@ -400,19 +619,34 @@ pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { app.toast(ctx, msg); } -/// The progress window while an import runs. +/// The progress window while an import runs, with Cancel (files already copied or added stay; +/// nothing new is started). pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) { let Some(task) = &app.import else { return }; let t = Tokens::get(ctx); let frac = task.done as f32 / task.total.max(1) as f32; - let text = format!("{} photos… {} of {}", if task.browse { "Reading" } else { "Adding" }, task.done, task.total); - egui::Window::new("Importing").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 60.0]).show( + let text = if task.cancelled { + "Stopping…".to_string() + } else { + format!("{} photos… {} of {}", if task.browse { "Reading" } else { "Adding" }, task.done, task.total) + }; + let mut cancel = false; + egui::Window::new("Importing").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show( ctx, |ui| { ui.label(egui::RichText::new(text).color(t.text)); ui.add(egui::ProgressBar::new(frac).desired_width(320.0)); + let r = ui.add_enabled(!task.cancelled, egui::Button::new("Cancel")); + register(ui.ctx(), "button:importCancel", r.rect); + cancel = r.clicked(); }, ); + if cancel && let Some(task) = app.import.as_mut() { + task.cancelled = true; + if let Some(run) = &task.run { + run.cancel.store(true, Ordering::Relaxed); + } + } } /// The dialog body: options, then the candidate grid. @@ -458,7 +692,13 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, d: &mut ImportDialog) { // options if !d.sources.is_empty() { field(ui, "Source", |ui| { - let r = ui.label(egui::RichText::new(source_summary(&d.sources)).color(t.text_label)).on_hover_text(d.sources.join("\n")); + // (whether each source is a folder is checked off the UI thread, once) + let joined = d.sources.join("\n"); + let summary = crate::panels::left::fs_cached(ui, "import-source", &joined, f64::INFINITY, |s| { + source_summary(&s.split('\n').map(str::to_string).collect::>()) + }) + .unwrap_or_else(|| format!("{} item{}", d.sources.len(), if d.sources.len() == 1 { "" } else { "s" })); + let r = ui.label(egui::RichText::new(summary).color(t.text_label)).on_hover_text(joined); register(ui.ctx(), "label:importSource", r.rect); }); ui.label( diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index b6ea924c4..2e478d459 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -21,6 +21,7 @@ pub mod render; pub mod shortcuts; pub mod softpaint; pub mod state; +pub mod tasks; pub mod theme; pub mod widgets; @@ -33,6 +34,8 @@ mod tests_library_problem; #[cfg(test)] mod tests_masking; #[cfg(test)] +mod tests_offline; +#[cfg(test)] mod tests_panels; #[cfg(test)] mod tests_quit_unsaved; @@ -169,6 +172,8 @@ pub struct LightcraftApp { pub scan: Option, /// A background export in progress. pub export: Option, + /// Background file-system work of other commands (Find Missing Photos, auto import…). + pub tasks: tasks::Tasks, /// The files of the last finished background export (`ui.inspect` → `export.last`). pub last_export_result: Option, /// The look the loupe shows while the pointer rests on a preset or profile (set by the @@ -224,6 +229,7 @@ impl LightcraftApp { import: None, scan: None, export: None, + tasks: Default::default(), last_export_result: None, hover_preview: None, window_is_fullscreen: false, @@ -286,7 +292,12 @@ impl LightcraftApp { if self.ui.preview_build_seen != Some((key, true)) { self.ui.preview_build_seen = Some((key, true)); let (done, failed) = (b.done.load(Ordering::Relaxed), b.failed.load(Ordering::Relaxed)); - let mut msg = format!("Previews ready for {done} photo{}", if done == 1 { "" } else { "s" }); + let plural = if done == 1 { "" } else { "s" }; + let mut msg = match (b.error(), b.what) { + (Some(e), what) => format!("{}: {e}", if what.is_empty() { "previews" } else { what }), + (None, "") => format!("Previews ready for {done} photo{plural}"), + (None, what) => format!("Done ({what}): {done} photo{plural}"), + }; if failed > 0 { msg.push_str(&format!(" · {failed} couldn't be rendered")); } @@ -295,7 +306,11 @@ impl LightcraftApp { } else { if self.ui.preview_build_seen != Some((key, false)) { self.ui.preview_build_seen = Some((key, false)); - self.toast(ctx, format!("Building previews for {} photos…", b.total)); + let msg = match b.what { + "" => format!("Building previews for {} photos…", b.total), + what => format!("Working on {what} for {} photos…", b.total), + }; + self.toast(ctx, msg); } ctx.request_repaint_after(std::time::Duration::from_millis(250)); } @@ -469,6 +484,10 @@ impl LightcraftApp { // File → Add from Device lists cards scanned in the background: show hot-plugs let repaint = ctx.clone(); lightcraft_engine::devices::on_change(move || repaint.request_repaint()); + // "is the original there?" (grid, Info panel, Missing Photos) answers from a cache a + // worker fills: a sleeping NAS or a dropped share never stalls a frame + let repaint = ctx.clone(); + self.session.media.availability.run_in_background(std::sync::Arc::new(move || repaint.request_repaint())); self.styled = true; } else { self.fonts_ready = true; @@ -493,6 +512,7 @@ impl LightcraftApp { merge::poll(self, ctx); import::poll_scan(self, ctx); import::tick(self, ctx); + tasks::poll(self, ctx); self.preview_build_status(ctx); self.save_status(ctx); self.slideshow_tick(ctx); @@ -507,17 +527,33 @@ impl LightcraftApp { } } self.ui.was_focused = focused; - // auto import: scan the watched folder every few seconds + // auto import: list the watched folder every few seconds (on a worker thread: it may be on + // a network share), then import what's new like any import (also on a worker thread) #[cfg(not(target_arch = "wasm32"))] - if self.session.import_defaults.auto_folder.is_some() { + if let Some(folder) = self.session.import_defaults.auto_folder.clone() { + const LABEL: &str = "Auto Import"; let now = ctx.input(|i| i.time); - if now - self.ui.auto_import_at >= 3.0 { + if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) { self.ui.auto_import_at = now; - if let Ok(r) = self.session.execute("library.autoImportScan", &serde_json::json!({})) { - let n = r["imported"].as_array().map_or(0, Vec::len); - if n > 0 { - self.toast(ctx, format!("Auto Import: added {n} photo{}", if n == 1 { "" } else { "s" })); + let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder); + let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result, String>| { + let listing = match listing { + Ok(l) => l, + Err(e) => return log::debug!("auto import: {e}"), + }; + let p = serde_json::json!({"listing": listing, "start": false}); + let Ok(r) = app.session.execute("library.autoImportScan", &p) else { return }; + let Some(mut params) = r.get("import").cloned().filter(|_| app.import.is_none()) else { return }; + let paths: Vec = + params["paths"].as_array().into_iter().flatten().filter_map(|v| v.as_str().map(str::to_string)).collect(); + if let Some(o) = params.as_object_mut() { + o.remove("paths"); } + let undo0 = app.session.undo.len(); + app.import = Some(import::ImportTask::new(paths, params, undo0, false).auto()); + }; + if let Err(e) = tasks::spawn(self, LABEL, work, done) { + log::warn!("{e}"); } } ctx.request_repaint_after(std::time::Duration::from_secs(3)); @@ -537,8 +573,11 @@ impl LightcraftApp { if !presets.is_empty() { let _ = self.run("file.importPresets", serde_json::json!({"paths": presets})); } - if !photos.is_empty() { - let _ = self.run("library.import", serde_json::json!({"paths": photos})); + // read and added on a worker thread (dropped folders can be large, or on a slow drive) + if !photos.is_empty() + && let Err(e) = import::start_paths(self, photos) + { + self.toast(ctx, e); } } } diff --git a/crates/ui-egui/src/menubar.rs b/crates/ui-egui/src/menubar.rs index 71571d326..ab92acebb 100644 --- a/crates/ui-egui/src/menubar.rs +++ b/crates/ui-egui/src/menubar.rs @@ -325,20 +325,21 @@ fn expanded(app: &LightcraftApp, name: &str) -> Option> { item("library.buildPreviews", json!({"size": "full"}), format!("Build 1:1 Previews ({scope})"), None, !running, None), item("library.cancelPreviews", Value::Null, "Stop Building Previews", None, running, None), MenuNode::Separator, + // (read and written on a worker thread: the originals may be on a slow drive) item( "library.smartPreviews", - Value::Null, + json!({"background": true}), format!("Build Smart Previews ({scope})"), None, - app.session.media.smart_dir.is_some(), + app.session.media.smart_dir.is_some() && !running, None, ), item( "library.smartPreviews", - json!({"discard": true}), + json!({"discard": true, "background": true}), format!("Discard Smart Previews ({scope})"), None, - app.session.media.smart_dir.is_some(), + app.session.media.smart_dir.is_some() && !running, None, ), MenuNode::Separator, @@ -936,7 +937,7 @@ mod tests { let mut app = LightcraftApp::new(lightcraft_engine::Session::new().with_fs(), Default::default()); app.session.execute("library.import", &json!({"paths": [dir.join("a").to_string_lossy()]})).unwrap(); std::fs::rename(dir.join("a/one.png"), dir.join("b/one.png")).unwrap(); - let r = run_item(&mut app, "file.findMissing", json!({"folder": dir.join("b").to_string_lossy()})).unwrap(); + let r = run_item(&mut app, "file.findMissing", json!({"folder": dir.join("b").to_string_lossy(), "wait": true})).unwrap(); assert_eq!(r["found"].as_array().map(Vec::len), Some(1), "{r}"); // Locate: an explicit file std::fs::rename(dir.join("b/one.png"), dir.join("one-renamed.png")).unwrap(); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index b4ef0ab3c..32b1dfb3c 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -928,18 +928,40 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option app.services.pick_folder.as_mut().and_then(|f| f()), }; let Some(folder) = folder else { return Some(Ok(Value::Null)) }; - let r = app.run("library.findMissing", json!({"folder": folder})); - if let Ok(v) = &r { - let n = v["found"].as_array().map_or(0, Vec::len); - let left = v["missing"].as_u64().unwrap_or(0); - let unsure = v["ambiguous"].as_array().map_or(0, Vec::len); - let unsure = if unsure > 0 { format!(" ({unsure} with several look-alike files: use Locate)") } else { String::new() }; - app.toast( - &egui::Context::default(), - format!("Found {n} missing photo{}; {left} still missing{unsure}", if n == 1 { "" } else { "s" }), - ); + // the search (checking every photo's file, walking the folder) runs on a worker thread; + // the relinking happens back here, as one undo step + const LABEL: &str = "Find Missing Photos"; + if app.tasks.is_running(LABEL) { + return Some(Err("Find Missing Photos is already searching".into())); } - r + let candidates = lightcraft_engine::cmd::missing::find_candidates(&app.session.catalog); + let work = move || lightcraft_engine::cmd::missing::plan_find_missing(&candidates, &folder); + let done = |app: &mut LightcraftApp, ctx: &egui::Context, plan: Result| { + // relinked under the session as it is now: photos relinked meanwhile and files + // now in use are skipped + let r = plan.and_then(|plan| app.run("library.findMissing", plan.to_json())); + match r { + Ok(v) => { + let n = v["found"].as_array().map_or(0, Vec::len); + let left = v["missing"].as_u64().unwrap_or(0); + let unsure = v["ambiguous"].as_array().map_or(0, Vec::len); + let unsure = if unsure > 0 { format!(" ({unsure} with several look-alike files: use Locate)") } else { String::new() }; + app.toast(ctx, format!("Found {n} missing photo{}; {left} still missing{unsure}", if n == 1 { "" } else { "s" })); + app.ui.last_find_missing = Some(v); + } + Err(e) => app.toast(ctx, e), + } + }; + app.ui.last_find_missing = None; + if let Err(e) = crate::tasks::spawn(app, LABEL, work, done) { + return Some(Err(e)); + } + if p.get("wait").and_then(Value::as_bool).unwrap_or(false) { + let ctx = app.tasks.repaint.clone().unwrap_or_default(); + crate::tasks::wait(app, &ctx, std::time::Duration::from_secs(600)); + return Some(Ok(app.ui.last_find_missing.clone().unwrap_or(Value::Null))); + } + Ok(json!({"background": true})) } "photo.tagFromTracklog" => { // a GPX file → GPS for the selected photos by capture time (one undo step) diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index da1b386dd..31ffb7865 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -7,6 +7,53 @@ use crate::LightcraftApp; use crate::state::Dialog; use crate::theme::Tokens; +/// Rows the Rename dialog previews. +const RENAME_PREVIEW_ROWS: usize = 6; + +/// The Rename dialog's preview: its first rows (`None` while being planned) and how many files +/// are renamed. Which names are taken is checked on disk, which can block on a slow drive, so the +/// rows are planned on a worker thread, and only when the template, start number, photos or +/// catalog change — never once per frame. +pub(crate) fn rename_preview( + app: &mut LightcraftApp, + ctx: &egui::Context, + template: &str, + start: usize, +) -> (Option>, usize) { + use std::hash::{Hash, Hasher}; + type Rows = std::sync::Arc>>>; + let ids = app.session.targets(&json!({})); + let photos = lightcraft_engine::rename::rename_photos(&app.session.catalog, &ids); + let total = photos.len(); + let mut h = std::collections::hash_map::DefaultHasher::new(); + (template, start, app.session.catalog.revision, &ids).hash(&mut h); + let key = h.finish(); + let id = egui::Id::new("rename-preview-rows"); + let read = |rows: &Rows| rows.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); + if let Some((k, rows)) = ctx.data(|d| d.get_temp::<(u64, Rows)>(id)) + && k == key + { + return (read(&rows), total); + } + let rows: Rows = Default::default(); + let first: Vec<_> = photos.into_iter().take(RENAME_PREVIEW_ROWS).collect(); + let (out, template, repaint) = (rows.clone(), template.to_string(), ctx.clone()); + let exists = app.session.media.availability.probe(); + let work = move || { + let plans = lightcraft_engine::rename::plan_rename_photos(&first, &template, start, &|f| exists(f)); + *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(plans); + repaint.request_repaint(); + }; + #[cfg(not(target_arch = "wasm32"))] + if let Err(e) = std::thread::Builder::new().name("lc-rename-preview".into()).spawn(work) { + log::warn!("rename preview: {e}"); + } + #[cfg(target_arch = "wasm32")] + work(); + ctx.data_mut(|d| d.insert_temp(id, (key, rows.clone()))); + (read(&rows), total) +} + /// Help ▸ What's New (docs/whats-new.md). pub const WHATS_NEW: &str = include_str!("../../../../docs/whats-new.md"); @@ -316,16 +363,23 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { egui::RichText::new("Tags: see Tags beside the template. Files are renamed on disk (with their XMP sidecars); existing names get -1, -2…") .color(t.text_dim), ); - let preview = app.session.execute("photo.renamePreview", &json!({"template": template, "start": start})).unwrap_or_default(); - egui::Grid::new("rename-preview").num_columns(3).spacing([8.0, 2.0]).show(ui, |ui| { - for pl in preview.as_array().into_iter().flatten().take(6) { - ui.label(egui::RichText::new(pl["from"].as_str().unwrap_or("")).color(t.text_dim)); - ui.label(egui::RichText::new("→").color(t.text_dim)); - ui.label(egui::RichText::new(pl["to"].as_str().unwrap_or("")).color(t.text)); - ui.end_row(); + let (preview, total) = rename_preview(app, ui.ctx(), template, *start as usize); + match &preview { + Some(rows) => { + egui::Grid::new("rename-preview").num_columns(3).spacing([8.0, 2.0]).show(ui, |ui| { + for pl in rows { + ui.label(egui::RichText::new(&pl.from).color(t.text_dim)); + ui.label(egui::RichText::new("→").color(t.text_dim)); + ui.label(egui::RichText::new(&pl.to).color(t.text)); + ui.end_row(); + } + }); } - }); - let more = preview.as_array().map_or(0, Vec::len).saturating_sub(6); + None => { + ui.label(egui::RichText::new("Checking names…").color(t.text_dim)); + } + } + let more = total.saturating_sub(RENAME_PREVIEW_ROWS); if more > 0 { ui.label(egui::RichText::new(format!("… and {more} more")).color(t.text_dim)); } diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs index eda9e6db6..dcebe199f 100644 --- a/crates/ui-egui/src/panels/grid.rs +++ b/crates/ui-egui/src/panels/grid.rs @@ -867,7 +867,8 @@ pub fn context_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { } ui.separator(); // the original moved or its drive is gone: point the photo at the file again - let missing = matches!(&app.session.catalog.photo(id).map(|p| p.source.clone()), Some(lightcraft_catalog::Source::File { path }) if !std::path::Path::new(path).exists()); + // (a cached answer, checked off the UI thread) + let missing = matches!(&app.session.catalog.photo(id).map(|p| p.source.clone()), Some(lightcraft_catalog::Source::File { path }) if app.session.media.availability.is_offline(path)); if missing && ui.button("Locate Missing File…").clicked() { let _ = app.run("photo.locate", json!({})); } diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 941e5070f..5b96941c8 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -200,7 +200,8 @@ fn local_section(app: &mut LightcraftApp, ui: &mut egui::Ui) { for (name, sub) in [("Pictures", "Pictures"), ("Desktop", "Desktop"), ("Downloads", "Downloads"), ("Home", "")] { // joined with the platform's separator, like the paths browsing gives back let p = if sub.is_empty() { home.clone() } else { std::path::Path::new(&home).join(sub).to_string_lossy().to_string() }; - if std::path::Path::new(&p).is_dir() { + // (checked off the UI thread: a home folder can be on a network share) + if fs_cached(ui, "is-dir", &p, 5.0, |p| std::path::Path::new(p).is_dir()) == Some(true) { builtin.push((name.to_string(), p)); } } @@ -305,15 +306,13 @@ pub(crate) fn local_places( out } -/// The subfolders of `path` (not hidden ones), sorted; listed at most every 2 s per folder. +/// The subfolders of `path` (not hidden ones), sorted; listed on a worker thread at most every +/// 2 s per folder (empty until the first listing). fn subfolders(ui: &egui::Ui, path: &str) -> Vec<(String, String)> { - let id = egui::Id::new(("subfolders", path.to_string())); - let now = ui.input(|i| i.time); - if let Some((t, v)) = ui.data(|d| d.get_temp::<(f64, Vec<(String, String)>)>(id)) - && now - t < 2.0 - { - return v; - } + fs_cached(ui, "subfolders", path, 2.0, list_subfolders).unwrap_or_default() +} + +fn list_subfolders(path: &str) -> Vec<(String, String)> { let mut v: Vec<(String, String)> = std::fs::read_dir(path) .map(|rd| { rd.flatten() @@ -326,10 +325,59 @@ fn subfolders(ui: &egui::Ui, path: &str) -> Vec<(String, String)> { }) .unwrap_or_default(); v.sort_by_key(|(n, _)| n.to_lowercase()); - ui.data_mut(|d| d.insert_temp(id, (now, v.clone()))); v } +/// A file-system answer for `path` (`f(path)`), kept per `kind` and path and refreshed on a worker +/// thread at most every `every` seconds: a folder on a sleeping NAS, a dropped share or a +/// spinning-up drive never blocks a frame. `None` until the first answer arrives. +pub(crate) fn fs_cached(ui: &egui::Ui, kind: &'static str, path: &str, every: f64, f: fn(&str) -> T) -> Option { + struct Entry { + value: Option, + at: Option, + running: bool, + } + type Cell = std::sync::Arc>>; + let id = egui::Id::new(("fs-cached", kind, path.to_string())); + let now = ui.input(|i| i.time); + let cell: Cell = match ui.data(|d| d.get_temp::>(id)) { + Some(c) => c, + None => { + let c: Cell = std::sync::Arc::new(std::sync::Mutex::new(Entry { value: None, at: None, running: false })); + ui.data_mut(|d| d.insert_temp(id, c.clone())); + c + } + }; + let lock = |c: &Cell| c.lock().unwrap_or_else(std::sync::PoisonError::into_inner).value.clone(); + let start = { + let mut e = cell.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let due = !e.running && e.at.is_none_or(|at| now - at >= every || now < at); + if due { + e.running = true; + e.at = Some(now); + } + due + }; + if start { + let (out, path, repaint) = (cell.clone(), path.to_string(), ui.ctx().clone()); + let work = move || { + let v = f(&path); + let mut e = out.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + e.value = Some(v); + e.running = false; + drop(e); + repaint.request_repaint(); + }; + #[cfg(not(target_arch = "wasm32"))] + if std::thread::Builder::new().name("lc-fs-list".into()).spawn(work).is_err() { + cell.lock().unwrap_or_else(std::sync::PoisonError::into_inner).running = false; + } + #[cfg(target_arch = "wasm32")] + work(); + } + lock(&cell) +} + /// A folder on disk with a disclosure triangle: click browses it, the triangle lists its /// subfolders. In the tree that holds the folder being browsed (`reveal`), the folders on the /// way down to it open whenever the browsed folder changes, so it shows highlighted in place. diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs index b2cef17fc..a4b27e328 100644 --- a/crates/ui-egui/src/panels/right.rs +++ b/crates/ui-egui/src/panels/right.rs @@ -535,9 +535,16 @@ fn info(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { ui.label(egui::RichText::new(line).color(t.text_dim)); } // offline originals / smart previews + // (cached answers, checked off the UI thread: unknown counts as online / no smart preview) if let lightcraft_catalog::Source::File { path } = &p.source { - let online = std::path::Path::new(path).exists(); - let smart = lightcraft_engine::cmd::previews::has_smart_preview(&app.session, p.id); + let avail = &app.session.media.availability; + let online = !avail.is_offline(path); + let smart = app + .session + .media + .smart_dir + .as_ref() + .is_some_and(|d| avail.exists(&d.join(lightcraft_engine::smart::file_name(&p)).to_string_lossy()) == Some(true)); if !online || smart { let text = match (online, smart) { (false, true) => "Original offline · editing the smart preview", diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index c585c6a74..e295dee37 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -359,6 +359,9 @@ pub struct UiState { /// Transient toast text and its expiry (seconds of app time). #[serde(skip)] pub toast: Option<(String, f64)>, + /// The result of the last Find Missing Photos (it searches in the background). + #[serde(skip)] + pub last_find_missing: Option, #[serde(skip)] pub status: String, #[serde(skip)] @@ -589,6 +592,7 @@ impl Default for UiState { compare: None, reference: None, toast: None, + last_find_missing: None, status: String::new(), dialog: None, } diff --git a/crates/ui-egui/src/tasks.rs b/crates/ui-egui/src/tasks.rs new file mode 100644 index 000000000..2cf526acf --- /dev/null +++ b/crates/ui-egui/src/tasks.rs @@ -0,0 +1,112 @@ +//! Background tasks for commands whose file-system work can take long on a slow or offline drive +//! (Find Missing Photos walking a folder, listing the auto-import folder…): the work runs on a +//! worker thread and its result is applied on the UI thread between frames, so the window keeps +//! answering. Imports, scans, exports and preview builds have their own tasks with progress. + +use std::sync::mpsc::{Receiver, TryRecvError, channel}; + +use crate::LightcraftApp; + +/// Applies a task's result to the app (on the UI thread). +type Finish = Box; + +struct Task { + label: String, + rx: Receiver, +} + +/// The background tasks in flight. +#[derive(Default)] +pub struct Tasks { + running: Vec, + /// Repaints the window when a task finishes (set every frame). + pub(crate) repaint: Option, +} + +impl Tasks { + pub fn is_empty(&self) -> bool { + self.running.is_empty() + } + + /// What is running (`ui.inspect` → `tasks`). + pub fn labels(&self) -> Vec { + self.running.iter().map(|t| t.label.clone()).collect() + } + + pub fn is_running(&self, label: &str) -> bool { + self.running.iter().any(|t| t.label == label) + } +} + +/// Run `work` on a worker thread, then `done(app, ctx, result)` on the UI thread (in the browser +/// build, which has no threads, `work` runs at once and `done` on the next frame). +pub fn spawn( + app: &mut LightcraftApp, + label: &str, + work: impl FnOnce() -> T + Send + 'static, + done: impl FnOnce(&mut LightcraftApp, &egui::Context, T) + Send + 'static, +) -> Result<(), String> { + let (tx, rx) = channel::(); + let repaint = app.tasks.repaint.clone(); + let job = move || { + let t = work(); + let _ = tx.send(Box::new(move |app: &mut LightcraftApp, ctx: &egui::Context| done(app, ctx, t))); + if let Some(c) = repaint { + c.request_repaint(); + } + }; + #[cfg(not(target_arch = "wasm32"))] + std::thread::Builder::new().name(format!("lc-task-{label}")).spawn(job).map_err(|e| format!("{label}: could not start: {e}"))?; + #[cfg(target_arch = "wasm32")] + job(); + app.tasks.running.push(Task { label: label.to_string(), rx }); + Ok(()) +} + +/// Apply finished tasks (called every frame). +pub fn poll(app: &mut LightcraftApp, ctx: &egui::Context) { + app.tasks.repaint = Some(ctx.clone()); + let mut i = 0; + while i < app.tasks.running.len() { + match app.tasks.running[i].rx.try_recv() { + Ok(finish) => { + app.tasks.running.remove(i); + finish(app, ctx); + } + Err(TryRecvError::Empty) => i += 1, + Err(TryRecvError::Disconnected) => { + // the worker died (a panic, already logged) + let t = app.tasks.running.remove(i); + app.toast(ctx, format!("{} failed", t.label)); + } + } + } + if !app.tasks.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +/// Wait for every task and apply it (tests, and commands asked to `wait`). `false` on timeout. +pub fn wait(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> bool { + #[cfg(not(target_arch = "wasm32"))] + { + let t0 = std::time::Instant::now(); + loop { + poll(app, ctx); + if app.tasks.is_empty() { + return true; + } + if t0.elapsed() > timeout { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } + #[cfg(target_arch = "wasm32")] + { + // tasks ran inline: their results are already waiting + let _ = timeout; + poll(app, ctx); + app.tasks.is_empty() + } +} diff --git a/crates/ui-egui/src/tests_offline.rs b/crates/ui-egui/src/tests_offline.rs new file mode 100644 index 000000000..f682fba57 --- /dev/null +++ b/crates/ui-egui/src/tests_offline.rs @@ -0,0 +1,177 @@ +//! Slow or offline drives never stall a frame (issue #104): with an injected file-system probe +//! that takes far longer than a frame (a sleeping NAS), the grid, the Info panel, the Missing +//! Photos view, the Rename dialog and an import keep drawing frames quickly, the probe never runs +//! on the UI thread, and the background answers still arrive. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::ThreadId; +use std::time::{Duration, Instant}; + +use lightcraft_catalog::{Op, Photo, PhotoId, Source}; +use lightcraft_engine::{LibrarySource, Session}; +use serde_json::json; + +use crate::headless::Headless; +use crate::state::{RightPanel, ViewMode}; +use crate::{LightcraftApp, Services}; + +/// What one file-system call takes on the "NAS". +const SLOW: Duration = Duration::from_millis(400); +/// A frame that waited for one such call would take at least this long. +const FRAME_LIMIT: Duration = Duration::from_millis(300); +const N: u64 = 12; + +/// Threads the probe ran on, and how often. +#[derive(Clone, Default)] +struct Calls { + threads: Arc>>, + n: Arc, +} + +impl Calls { + fn record(&self) { + self.threads.lock().unwrap().push(std::thread::current().id()); + self.n.fetch_add(1, Ordering::SeqCst); + } + fn on(&self, t: ThreadId) -> usize { + self.threads.lock().unwrap().iter().filter(|x| **x == t).count() + } +} + +/// `N` photos whose originals are on an unreachable share, a smart previews folder, and a probe +/// that sleeps like a NAS waking up before saying "not there". +fn offline_library(calls: &Calls) -> Session { + let mut s = Session::new(); + for i in 0..N { + let path = format!("/lightcraft-offline-nas/IMG_{i:03}.jpg"); + let p = Photo::new(PhotoId(i + 1), Source::File { path }, &format!("IMG_{i:03}.jpg"), "JPEG", 600, 400, "2026-01-01T00:00:00"); + s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap(); + } + s.media.smart_dir = Some(std::env::temp_dir().join("lightcraft-offline-smart-previews-none")); + let c = calls.clone(); + s.media.availability.set_probe(Arc::new(move |_| { + c.record(); + std::thread::sleep(SLOW); + false + })); + s +} + +fn max_frame(h: &mut Headless, frames: usize) -> Duration { + let mut worst = Duration::ZERO; + for _ in 0..frames { + let t0 = Instant::now(); + h.step(); + worst = worst.max(t0.elapsed()); + } + worst +} + +#[test] +fn offline_originals_never_block_a_frame() { + let calls = Calls::default(); + let app = LightcraftApp::new(offline_library(&calls), Services { png: None, ..Default::default() }); + let mut h = Headless::new(app, [1200.0, 800.0], 1.0); + let ui_thread = std::thread::current().id(); + h.app.ui.view = ViewMode::PhotoGrid; + h.step(); + assert!(h.app.session.media.availability.is_background(), "the app checks files off the UI thread"); + + // the grid: every cell asks for its thumbnail (which asks whether the original is there) + h.app.session.execute("library.select", &json!({"ids": [1]})).unwrap(); + let worst = max_frame(&mut h, 20); + assert!(worst < FRAME_LIMIT, "grid frame took {worst:?}"); + + // the Info panel of the active photo, in the loupe + h.app.ui.view = ViewMode::Detail; + h.app.ui.right = RightPanel::Info; + let worst = max_frame(&mut h, 20); + assert!(worst < FRAME_LIMIT, "Info panel frame took {worst:?}"); + + // the Missing Photos view fills in as the background checks find the files gone + h.app.ui.view = ViewMode::PhotoGrid; + h.app.ui.right = RightPanel::None; + h.app.session.execute("library.source", &json!({"kind": "missing"})).unwrap(); + assert_eq!(h.app.session.source, LibrarySource::Missing); + let t0 = Instant::now(); + let mut worst = Duration::ZERO; + while h.app.session.visible().len() < N as usize && t0.elapsed() < Duration::from_secs(60) { + let f = Instant::now(); + h.step(); + worst = worst.max(f.elapsed()); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(h.app.session.visible().len(), N as usize, "every photo found missing in the end"); + assert!(worst < FRAME_LIMIT, "Missing Photos frame took {worst:?}"); + + // the Rename dialog's preview (it checks which names are taken on disk) + h.app.session.execute("library.select", &json!({"ids": [1, 2, 3]})).unwrap(); + h.app.ui.dialog = Some(crate::state::Dialog::Rename { template: "Trip-{seq:3}".into(), start: 1 }); + let worst = max_frame(&mut h, 20); + assert!(worst < FRAME_LIMIT, "Rename dialog frame took {worst:?}"); + + assert!(calls.n.load(Ordering::SeqCst) > 0, "the probe was asked"); + assert_eq!(calls.on(ui_thread), 0, "the slow probe never ran on the UI thread"); +} + +/// An import whose files take long to read (probe: decoding headers and hashing on a slow drive) +/// runs on a worker thread: frames stay quick, the probe never runs on the UI thread, and Cancel +/// stops it with what was read so far added. +#[test] +fn import_reads_files_off_the_ui_thread_and_can_be_cancelled() { + let dir = std::env::temp_dir().join(format!("lc-ui-slow-import-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let n = 40; + for i in 0..n { + std::fs::write(dir.join(format!("IMG_{i:03}.jpg")), format!("not really a jpeg {i}")).unwrap(); + } + let calls = Calls::default(); + let mut s = Session::new(); + let c = calls.clone(); + s.media.file_probe = Some(Arc::new(move |p: &str| { + c.record(); + std::thread::sleep(Duration::from_millis(150)); + Ok(lightcraft_engine::media::ProbeInfo { + width: 60, + height: 40, + format: "JPEG".into(), + content_hash: Some(p.to_string()), + ..Default::default() + }) + })); + let app = LightcraftApp::new(s, Services { png: None, ..Default::default() }); + let mut h = Headless::new(app, [1200.0, 800.0], 1.0); + let ui_thread = std::thread::current().id(); + h.step(); + let undo0 = h.app.session.undo.len(); + crate::import::start_paths(&mut h.app, vec![dir.to_string_lossy().to_string()]).unwrap(); + // some batches arrive; meanwhile every frame is quick + let t0 = Instant::now(); + let mut worst = Duration::ZERO; + while h.app.import.as_ref().is_some_and(|t| t.imported < 8) && t0.elapsed() < Duration::from_secs(60) { + let f = Instant::now(); + h.step(); + worst = worst.max(f.elapsed()); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(worst < FRAME_LIMIT, "import frame took {worst:?}"); + assert!(h.app.import.is_some(), "still importing"); + let status = h.app.import.as_ref().map(crate::import::ImportTask::status).unwrap_or_default(); + assert!(status["total"].as_u64().unwrap_or(0) >= 8, "{status}"); + // Cancel: the files being read finish, nothing new starts + let r = h.request("ui.clickWidget", json!({"id": "button:importCancel"}), Duration::from_secs(10)); + assert_eq!(r["ok"], true, "{r}"); + let t0 = Instant::now(); + while h.app.import.is_some() && t0.elapsed() < Duration::from_secs(60) { + h.step(); + std::thread::sleep(Duration::from_millis(5)); + } + assert!(h.app.import.is_none(), "stopped"); + let added = h.app.session.catalog.len(); + assert!((8..n).contains(&added), "cancelled part-way: {added} of {n} added"); + assert_eq!(h.app.session.undo.len(), undo0 + 1, "one undo step"); + assert_eq!(calls.on(ui_thread), 0, "files were never read on the UI thread"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 4723f7310..24fd2bdb4 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -30,7 +30,7 @@ mode ([mcp.md](mcp.md)) is a thin layer over this channel. Implementation: | `engine.commands` | — | Engine + UI commands: id, label, menu, shortcut, params doc, enabled | | `ui.menu.list` | — | Menu entries (flat: id, label, menu path, shortcut, enabled) | | `ui.menu.tree` | — | The menu bar as shown (File … Help): items `{id, params?, label, shortcut?, enabled, checked?}`, separators, submenus — the model behind the native macOS menu bar and the in-window menus | -| `ui.inspect` | — | UI state, window, canvas/image rects, `scroll: {grid, filmstrip}` (scroll offsets in points, `null` until drawn), active photo, selection, perf (`frameMs` = layout, `logicMs` = per-frame logic before it, `updateMs` = both, `maxUpdateMs`, `fps`, render queue …, `gpu` = adapter in use, `gpuReason` = why renders don't use the GPU, `gpuFallback` = latest render redone on the CPU and why — see `docs/gpu-pipeline.md`), status, memory (bytes per cache, see `library.memory`; plus stage caches and textures), `export: {running: {total, done, current} \| null, last}`, `notices` (warnings waiting to be shown, e.g. a damaged settings file; OK = `button:noticeOk`), `quitPrompt` (why quitting was stopped: unsaved changes; `button:quitRetry` / `button:quitAnyway` / `button:quitCancel`) | +| `ui.inspect` | — | UI state, window, canvas/image rects, `scroll: {grid, filmstrip}` (scroll offsets in points, `null` until drawn), active photo, selection, perf (`frameMs` = layout, `logicMs` = per-frame logic before it, `updateMs` = both, `maxUpdateMs`, `fps`, render queue …, `gpu` = adapter in use, `gpuReason` = why renders don't use the GPU, `gpuFallback` = latest render redone on the CPU and why — see `docs/gpu-pipeline.md`), status, memory (bytes per cache, see `library.memory`; plus stage caches and textures), `export: {running: {total, done, current} \| null, last}`, `notices` (warnings waiting to be shown, e.g. a damaged settings file; OK = `button:noticeOk`), `quitPrompt` (why quitting was stopped: unsaved changes; `button:quitRetry` / `button:quitAnyway` / `button:quitCancel`), `import: {done, total, imported, cancelled} \| null` (an import runs on a worker thread), `tasks` (other background work: `Find Missing Photos`, `Auto Import`) | | `ui.widgets` | `{filter?}` | On-screen widgets `{id, rect: [x, y, w, h]}` (screen points) | | `ui.clickWidget` / `ui.dragWidget` / `ui.hoverWidget` | `{id, count?, fx?, fy?}` / `{id, toX?, toY?, dx?, dy?, steps?}` / `{id, fx?, fy?}` | Real egui input on a widget (hover: the pointer rests on it, e.g. for preset/profile previews) | | `ui.move` / `ui.click` / `ui.drag` | `{x, y, count?, button?}` / `{x, y, toX, toY, steps?}` | Raw pointer input, screen points | diff --git a/docs/parity.md b/docs/parity.md index ac62ea34c..332c05e3b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -113,7 +113,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | Id | Feature | Tier | Status | Evidence | Missing / notes | |---|---|---|---|---|---| -| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog (Import Photos): the scanned source (a scanned folder is not added to Local), candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), transfer mode (add in place / copy / move, each explained), album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); batched import with a progress window; one undo step. Drag-and-drop still imports directly | +| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog (Import Photos): the scanned source (a scanned folder is not added to Local), candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), transfer mode (add in place / copy / move, each explained), album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); the import itself (probing, sidecars, copying / moving files) runs on a worker thread and its batches join the library between frames, with a progress window and Cancel; one undo step. Drag-and-drop goes through the same background import | | LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported | | LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | | | LR-IMP-DEVICE | Import from camera/card | P1 | ✅ | `cmd:library.devices`, `cmd:file.addFromDevice`, `crates/engine/src/devices.rs` | mounted volumes with a DCIM folder (macOS /Volumes, Linux /media and /run/media, Windows drive letters); File → Import from Device → the import review, copying into the library by default; menus serve the last scan and rescan in the background (never blocking a frame; hot-plugs repaint); copies are verified like Move's (a new file, synced, compared byte for byte; a bad copy is removed and reported as failed, never counted as imported; taken names get -1, -2…, never replaced — issue #96, `crates/engine/src/import_move.rs`); no PTP/MTP (cameras that don't mount as a disk) | @@ -480,7 +480,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | Id | Feature | Tier | Status | Evidence | Missing / notes | |---|---|---|---|---|---| | LR-CLOUD-SYNC | Cloud sync | OOS | 🚫 | | | -| LR-CLOUD-SMARTPREVIEW | Editable proxies | P2 | ✅ | `cmd:library.smartPreviews`, `cmd:library.smartPreviewsLocation`, `cmd:photo.smartPreview`, `crates/engine/src/smart.rs`, `crates/ui-egui/src/panels/settings.rs` | File ▸ Previews ▸ Build / Discard Smart Previews: ~1 MB proxies in the library; with the original offline the photo renders, edits and exports (at proxy size) from its proxy; Info shows the status; the proxy folder is chosen per library (Settings → Performance → Smart previews, saved in the library): the effective path, count and size are shown, changing it needs a choice for the proxies already built (move / leave / delete), and an unavailable or unwritable folder is an error, never a fallback to the library drive; proxies are written atomically and a damaged one (cut short by a crash or a full drive) doesn't count as built, is rebuilt (`repaired`) and is never moved over a good one (issue #106). The thumbnail cache stays in the library | +| LR-CLOUD-SMARTPREVIEW | Editable proxies | P2 | ✅ | `cmd:library.smartPreviews`, `cmd:library.smartPreviewsLocation`, `cmd:photo.smartPreview`, `crates/engine/src/smart.rs`, `crates/ui-egui/src/panels/settings.rs` | File ▸ Previews ▸ Build / Discard Smart Previews (on a worker thread, progress and Stop like Build Previews): ~1 MB proxies in the library; with the original offline the photo renders, edits and exports (at proxy size) from its proxy; Info shows the status; the proxy folder is chosen per library (Settings → Performance → Smart previews, saved in the library): the effective path, count and size are shown, changing it needs a choice for the proxies already built (move / leave / delete), and an unavailable or unwritable folder is an error, never a fallback to the library drive; proxies are written atomically and a damaged one (cut short by a crash or a full drive) doesn't count as built, is rebuilt (`repaired`) and is never moved over a good one (issue #106). The thumbnail cache stays in the library | | LR-AI-UPDATE-INDICATOR | AI-settings update indicator | P2 | ⬜ | | | | LR-AI-CREDITS | Generative credits | OOS | 🚫 | | | @@ -728,7 +728,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06- | LRC-LIB-PEOPLE | People view | P2 | ⬜ | | | | LRC-LIB-COMMENTS | Comments panel | P2 | ⬜ | | | | LRC-LIB-VISUALSEARCH | Find similar photos | P2 | ✅ | `cmd:library.findSimilar`, `crates/pipeline/src/cull.rs` (`signature`) | photo menu / Photo ▸ Find Similar Photos: look-alike photos (composition and tones), most similar first, as a filter (`only`) | -| LRC-LIB-MISSING | Missing files & relink | P1 | ✅ | `cmd:library.missing`, `cmd:photo.relink`, `cmd:library.findMissing`, `cmd:file.findMissing`, `cmd:photo.locate`, `crates/engine/src/cmd/missing.rs` (`checked_path`) | File → Find Missing Photos… (anywhere in a folder: same name, size and content hash; renamed files by size + content hash; same name and size alone only when unambiguous — look-alikes are skipped and reported; one undo step), photo menu → Locate Missing File…; unreadable files show "!" in the grid and a reason in the loupe; undo never moves files. Missing Photos in the sidebar (counted on a worker thread); it covers library photos only — Local browse records are never checked, for the count, the view, `library.missing` and Find Missing alike | +| LRC-LIB-MISSING | Missing files & relink | P1 | ✅ | `cmd:library.missing`, `cmd:photo.relink`, `cmd:library.findMissing`, `cmd:file.findMissing`, `cmd:photo.locate`, `crates/engine/src/cmd/missing.rs` (`checked_path`) | File → Find Missing Photos… (anywhere in a folder: same name, size and content hash; renamed files by size + content hash; same name and size alone only when unambiguous — look-alikes are skipped and reported; searched on a worker thread, one undo step), photo menu → Locate Missing File…; unreadable files show "!" in the grid and a reason in the loupe; undo never moves files. Missing Photos in the sidebar (counted on a worker thread; the view, the Info panel, the grid's context menu and thumbnails read file availability from a cache a worker thread fills, so an offline NAS never stalls a frame); it covers library photos only — Local browse records are never checked, for the count, the view, `library.missing` and Find Missing alike | | LRC-LIB-CONVERT | Convert to DNG | P2 | ✅ | `cmd:photo.convertToDng`, `crates/engine/src/cmd/convert.rs` | Photo ▸ Convert to DNG: lossless DNG next to the raw with the settings embedded, photo (and its virtual copies) relinked; originals kept; undoable; the DNG is decoded and compared with the raw data, written to a temp file, synced, read back and only then given a free name (never replacing a file) — a failed write leaves no DNG and the photo on its raw (issue #106) | | LRC-LIB-PREVIEWS | Build / discard previews | P1 | ✅ | `cmd:library.buildPreviews`, `cmd:library.previewProgress`, `cmd:library.cancelPreviews`, `cmd:library.clearPreviews`, `crates/engine/src/cmd/previews.rs`, `crates/preview/src/lib.rs` | File ▸ Previews: build standard-sized (Settings → Performance size) or 1:1 previews of the selected / visible photos in the background (progress toasts, stop), discard the cache; disk thumbnail + view cache; no smart previews (offline proxies) | | LRC-LIB-SLIDESHOW-IMPROMPTU | Impromptu slideshow | P2 | ✅ | `cmd:view.slideshow` | see LR-VIEW-SLIDESHOW | diff --git a/docs/whats-new.md b/docs/whats-new.md index ce7cd6c1b..84f28efc6 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -42,6 +42,10 @@ - The control port (`--control`) closes a connection as soon as it receives anything that isn't a JSON request (issue #94): an HTTP request from a web page can no longer carry a command in its body. Lines are capped at 4 MiB and connections at 16. +- A sleeping NAS, a dropped network share or a USB drive spinning up no longer freezes the window (issue #104): + whether originals are there is checked on a worker thread (grid thumbnails, the Info panel, the photo menu, Missing + Photos); importing (also by drag and drop), Find Missing Photos, Build / Discard Smart Previews, the Rename preview, + the Local folder tree and Auto Import read the disk on worker threads too. The import progress window has Cancel. - Canon CR2 photos from the EOS 7D, 50D, 60D, 550D, 600D, 1200D, 1300D, 5D Mark II and 1D Mark IV (and other models whose sensor starts on a green-blue row) no longer come out magenta (issue #85): the colour-filter layout is read from each file instead of assumed.