pshock13/lightcraft
1
0 Fork 0
mirror of https://github.com/storytold/lightcraft.git synced 2026-10-14 19:27:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge PR #834

Browse Source
This commit is contained in:
Claude committed 2026-10-11 12:00:20 +00:00
2 parents 5ba413baf2 a66a133041
commit 477c078b35
6 files changed
+145 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/engine/src/cmd/browse.rs
+1 -1
View File
@@ -107,7 +107,7 @@ impl Session {
let smart = self.media.smart_dir.clone();
let plan = self
.catalog
.forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| d.join(crate::smart::file_name(p)).exists()));
.forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| crate::smart::file_path(d, p).exists()));
if dry_run {
return plan;
}
crates/engine/src/cmd/previews.rs
+19 -5
View File
@@ -243,6 +243,9 @@ fn smart_run(
break;
}
let _one = SMART_WRITE.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
// Read either the new shard or a pre-upgrade flat file, without making new work
// depend on having the original online.
let path = crate::smart::existing_file_path(dir, path);
if discard {
if std::fs::remove_file(&path).is_ok() {
n.removed += 1;
@@ -266,7 +269,14 @@ fn smart_run(
.ok_or_else(|| "nothing to build from".to_string())
.and_then(|s| s.load_source())
.and_then(|src| crate::smart::encode(&src.image, src.info_or(Default::default()).camera_tone.as_ref()))
.and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display())));
.and_then(|b| {
let parent = path.parent().ok_or_else(|| format!("{}: no parent folder", path.display()))?;
// Only create the leaf shard: do not recreate a missing external drive.
if !parent.is_dir() {
std::fs::create_dir(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
}
lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display()))
});
match r {
Ok(()) => {
n.built += 1;
@@ -321,7 +331,7 @@ fn smart(s: &mut Session, p: &Value) -> Result<Value> {
if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) {
continue;
}
let path = dir.join(crate::smart::file_name(&ph));
let path = crate::smart::sharded_file_path(&dir, &ph);
let source = (!discard).then(|| s.media.source_ref(&ph, crate::media::SourceLevel::Preview));
jobs.push((id, path, source, ph.relative_wb()));
}
@@ -367,7 +377,7 @@ pub(crate) fn refresh_stale_smart_previews(s: &mut Session) -> Option<std::threa
let max_edge = crate::media::SourceLevel::Preview.max_edge();
let photos: Vec<_> = s.catalog.photos().filter(|p| matches!(p.source, lightcraft_catalog::Source::File { .. }) && p.relative_wb()).collect();
let jobs: Vec<SmartJob> =
photos.iter().map(|p| (p.id, dir.join(crate::smart::file_name(p)), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect();
photos.iter().map(|p| (p.id, crate::smart::sharded_file_path(&dir, p), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect();
std::thread::Builder::new()
.name("lc-smart-look".into())
.spawn(move || refresh_stale(&dir, jobs))
@@ -378,7 +388,11 @@ pub(crate) fn refresh_stale_smart_previews(s: &mut Session) -> Option<std::threa
/// The body of [`refresh_stale_smart_previews`]: rebuild the stale proxies among `jobs`.
#[cfg(not(target_arch = "wasm32"))]
fn refresh_stale(dir: &std::path::Path, jobs: Vec<SmartJob>) {
let stale: Vec<SmartJob> = jobs.into_iter().filter(|(_, path, _, _)| crate::smart::is_stale(path)).collect();
let stale: Vec<SmartJob> = jobs
.into_iter()
.map(|(id, path, source, look)| (id, crate::smart::existing_file_path(dir, path), source, look))
.filter(|(_, path, _, _)| crate::smart::is_stale(path))
.collect();
if !stale.is_empty() {
// `custom`: the folder is there already, and must not be recreated if it went away since
match smart_run(dir, true, false, stale, None) {
@@ -461,7 +475,7 @@ fn smart_location(s: &mut Session, p: &Value) -> Result<Value> {
/// Whether photo `id` has a smart preview.
pub fn has_smart_preview(s: &Session, id: lightcraft_catalog::PhotoId) -> bool {
match (&s.media.smart_dir, s.catalog.photo(id)) {
(Some(dir), Some(p)) => crate::smart::is_valid(&dir.join(crate::smart::file_name(p))),
(Some(dir), Some(p)) => crate::smart::is_valid(&crate::smart::file_path(dir, p)),
_ => false,
}
}
crates/engine/src/media.rs
+3 -1
View File
@@ -635,7 +635,9 @@ impl MediaCache {
// be offline is tried first and its smart preview is the render worker's fallback.
#[cfg(not(target_arch = "wasm32"))]
if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) {
let sp = dir.join(crate::smart::file_name(p));
let sharded = crate::smart::sharded_file_path(dir, p);
let legacy = dir.join(crate::smart::file_name(p));
let sp = if self.availability.exists(&sharded.to_string_lossy()) == Some(true) { sharded } else { legacy };
// (a proxy cut short fails to load, like the offline original; the worker's fallback
// in `SourceRef::load` only uses a proxy that passes `smart::is_valid`)
if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) {
crates/engine/src/smart.rs
+106 -19
View File
@@ -21,6 +21,30 @@ pub fn file_name(p: &Photo) -> String {
format!("{:032x}.lcsp", h.0)
}
/// The default sharded location of a proxy. A two-character hex prefix bounds each
/// folder to about 1/256 of the library's smart previews.
pub fn sharded_file_path(dir: &Path, p: &Photo) -> PathBuf {
let name = file_name(p);
dir.join(&name[..2]).join(name)
}
/// Read an existing proxy in either layout. New proxies use the sharded location; previews
/// made by older releases still load and can be moved with smartPreviewsLocation.
pub fn file_path(dir: &Path, p: &Photo) -> PathBuf {
existing_file_path(dir, sharded_file_path(dir, p))
}
/// Resolve a sharded path to an older flat file when present. Call only off the UI thread:
/// checking a removable drive synchronously could stall drawing.
pub fn existing_file_path(dir: &Path, sharded: PathBuf) -> PathBuf {
if sharded.exists() {
return sharded;
}
let Some(name) = sharded.file_name() else { return sharded };
let legacy = dir.join(name);
if legacy.exists() { legacy } else { sharded }
}
/// The most header [`is_valid`] reads (the JSON line before the JPEG).
const HEADER_MAX: u64 = 4096;
@@ -104,12 +128,45 @@ pub fn check_writable(dir: &Path) -> Result<(), String> {
Ok(())
}
/// The owned proxy files in both layouts. Do not follow symlinked directories or
/// traverse unrelated folders inside a user-selected smart preview directory.
fn files(dir: &Path) -> Vec<PathBuf> {
let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
let mut found = Vec::new();
for e in entries.flatten() {
let Ok(kind) = e.file_type() else { continue };
if kind.is_file() && e.path().extension().is_some_and(|ext| ext == "lcsp") {
found.push(e.path()); // legacy layout
} else if kind.is_dir() {
let shard_name = e.file_name();
let Some(shard) = shard_name.to_str() else { continue };
if shard.len() != 2 || !shard.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) {
continue;
}
let Ok(children) = std::fs::read_dir(e.path()) else { continue };
for child in children.flatten() {
let name = child.file_name();
let Some(name) = name.to_str() else { continue };
let Some(stem) = name.strip_suffix(".lcsp") else { continue };
if stem.len() == 32
&& stem.starts_with(shard)
&& stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
&& child.file_type().is_ok_and(|kind| kind.is_file())
{
found.push(child.path());
}
}
}
}
found
}
/// The number of smart previews in `dir` and their total size in bytes.
pub fn stats(dir: &Path) -> (usize, u64) {
let Ok(rd) = std::fs::read_dir(dir) else { return (0, 0) };
rd.flatten()
.filter(|e| e.path().extension().is_some_and(|x| x == "lcsp"))
.fold((0, 0), |(n, b), e| (n + 1, b + e.metadata().map_or(0, |m| m.len())))
files(dir).into_iter().fold((0usize, 0u64), |(count, bytes), path| {
let size = std::fs::metadata(path).map_or(0, |m| m.len());
(count.saturating_add(1), bytes.saturating_add(size))
})
}
/// What to do with the smart previews in the old folder when the location changes.
@@ -140,24 +197,30 @@ pub fn migrate(from: &Path, to: &Path, what: Existing) -> (usize, Vec<String>) {
if what == Existing::Leave {
return (done, failed);
}
let Ok(rd) = std::fs::read_dir(from) else { return (done, failed) };
for e in rd.flatten().filter(|e| e.path().extension().is_some_and(|x| x == "lcsp")) {
let src = e.path();
for src in files(from) {
let Some(name) = src.file_name() else { continue };
let dst = name.to_str().and_then(|n| n.strip_suffix(".lcsp")).filter(|stem| {
stem.len() == 32 && stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
}).map_or_else(|| to.join(name), |stem| to.join(&stem[..2]).join(name));
let r = match what {
Existing::Discard => std::fs::remove_file(&src),
_ => {
let dst = to.join(e.file_name());
if is_valid(&dst) {
// named by content: the complete copy already there is the same preview
std::fs::remove_file(&src)
} else {
// across drives: copied atomically (never a partial proxy at the new place)
std::fs::rename(&src, &dst).or_else(|_| {
std::fs::read(&src)
.and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b))
.and_then(|()| std::fs::remove_file(&src))
})
}
let parent = dst.parent().unwrap_or(to);
// Create only the shard, never missing parent directories on an unplugged drive.
let prepare = if parent.is_dir() { Ok(()) } else { std::fs::create_dir(parent) };
prepare.and_then(|()| {
if is_valid(&dst) {
// named by content: the complete copy already there is the same preview
std::fs::remove_file(&src)
} else {
// across drives: copied atomically (never a partial proxy at the new place)
std::fs::rename(&src, &dst).or_else(|_| {
std::fs::read(&src)
.and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b))
.and_then(|()| std::fs::remove_file(&src))
})
}
})
}
};
match r {
@@ -288,6 +351,30 @@ mod tests {
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn sharded_and_legacy_proxies_are_counted_and_moved() {
let base = temp("shards");
let from = base.join("from");
let to = base.join("to");
std::fs::create_dir_all(from.join("a1")).unwrap();
std::fs::create_dir(&to).unwrap();
let old = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2.lcsp";
let new = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1.lcsp";
std::fs::write(from.join(old), b"legacy").unwrap();
std::fs::write(from.join("a1").join(new), b"sharded").unwrap();
std::fs::write(from.join("a1").join("notes.txt"), b"user file").unwrap();
std::fs::write(from.join("notes.lcsp.txt"), b"keep").unwrap();
assert_eq!(stats(&from), (2, 13));
let (moved, failed) = migrate(&from, &to, Existing::Move);
assert_eq!((moved, failed), (2, Vec::<String>::new()));
assert_eq!(stats(&from), (0, 0));
assert_eq!(stats(&to), (2, 13));
assert_eq!(std::fs::read(to.join("b2").join(old)).unwrap(), b"legacy");
assert_eq!(std::fs::read(to.join("a1").join(new)).unwrap(), b"sharded");
assert!(from.join("a1").join("notes.txt").exists());
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn damaged_proxies_are_invalid_and_never_kept_over_good_ones() {
let base = temp("valid");
crates/engine/src/tests_import.rs
+11 -4
View File
@@ -462,14 +462,14 @@ fn damaged_smart_previews_are_rebuilt_and_failed_writes_leave_none() {
s.execute("library.import", &json!({"paths": [src.join("a.png").to_string_lossy()]})).unwrap();
let id = s.active().unwrap();
let dir = s.media.smart_dir.clone().unwrap();
let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap()));
let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap());
{
let _fault = lightcraft_catalog::safe_file::fail_writes_after(64);
let r = s.execute("library.smartPreviews", &json!({})).unwrap();
assert_eq!((r["built"].as_u64(), r["failed"].as_array().map(Vec::len)), (Some(0), Some(1)), "{r}");
}
assert!(!file.exists(), "no partial proxy");
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "no temp file left");
assert_eq!(std::fs::read_dir(file.parent().unwrap()).unwrap().count(), 0, "no temp file left in shard");
assert_eq!(s.execute("photo.smartPreview", &json!({})).unwrap()["smartPreview"], false);
let r = s.execute("library.smartPreviews", &json!({})).unwrap();
@@ -795,6 +795,13 @@ fn smart_previews_folder_is_chosen_per_library() {
assert_eq!(loc["path"], lib.join("Smart Previews").to_string_lossy().as_ref());
s.execute("library.smartPreviews", &json!({})).unwrap();
assert_eq!(crate::smart::stats(&lib.join("Smart Previews")).0, 1);
// A flat proxy from older builds remains discoverable and migrates to the new sharded layout.
let dir = lib.join("Smart Previews");
let photo = s.catalog.photo(id).unwrap();
let shard = crate::smart::sharded_file_path(&dir, photo);
let flat = dir.join(crate::smart::file_name(photo));
std::fs::rename(&shard, &flat).unwrap();
assert_eq!(s.execute("photo.smartPreview", &json!({})).unwrap()["smartPreview"], true);
// relative paths and missing drives are refused, and nothing is created on this drive
assert!(s.execute("library.smartPreviewsLocation", &json!({"path": "rel/dir"})).is_err());
@@ -1300,8 +1307,8 @@ fn smart_previews_from_an_older_look_fit_are_refreshed() {
assert!(s.catalog.photo(id).unwrap().relative_wb() && !s.catalog.photo(png).unwrap().relative_wb());
s.execute("library.select", &json!({"ids": [id.0, png.0], "active": id.0})).unwrap();
let dir = s.media.smart_dir.clone().unwrap();
let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap()));
let png_file = dir.join(crate::smart::file_name(s.catalog.photo(png).unwrap()));
let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap());
let png_file = crate::smart::sharded_file_path(&dir, s.catalog.photo(png).unwrap());
let (old, new) = (test_tone(1.5), test_tone(3.5));
let calls = std::sync::Arc::new(AtomicUsize::new(0));
let plain = s.media.file_loader.clone().unwrap();
crates/ui-egui/src/panels/right.rs
+5 -1
View File
@@ -699,7 +699,11 @@ fn info(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
.media
.smart_dir
.as_ref()
.is_some_and(|d| avail.exists(&d.join(lightcraft_engine::smart::file_name(&p)).to_string_lossy()) == Some(true));
.is_some_and(|d| {
let shard = lightcraft_engine::smart::sharded_file_path(d, &p);
let legacy = d.join(lightcraft_engine::smart::file_name(&p));
avail.exists(&shard.to_string_lossy()) == Some(true) || avail.exists(&legacy.to_string_lossy()) == Some(true)
});
if !online || smart {
let text = match (online, smart) {
(false, true) => "Original offline · editing the smart preview",