Compare commits

...
Author SHA1 Message Date
FrancoisandClaude d999e382ba #1949: pt-BR for Include selected photos and the mask overlay modes (#2033)
Claude-Session: https://claude.ai/code/session_01AbegtYaPuwqZ81pGfELsFK

Co-authored-by: Claude <[email protected]>
2026-10-18 16:59:33 -06:00
Dylan M. Taylor fb7ac08b9a ci: configure Dependabot for Cargo and GitHub Actions
Co-authored-by: Dylan M. Taylor <[email protected]>
2026-10-18 12:31:40 -06:00
Dylan M. Taylor 31a677fe3d docs: drop references to competing commercial software (#2213) 2026-10-18 10:41:49 -04:00
Brandon Thomas a945533f63 Merge pull request #2236 from storytold/claude/integrate-batch-42
Integration batch 42: 31 community PRs
2026-10-18 10:39:21 -04:00
Brandon Thomas d43dd1fc8f Merge PR #1778: Model downloads: Time out and cancel the name lookup (#1532) 2026-10-18 10:05:47 -04:00
Brandon Thomas 6f5d662df3 Merge PR #2235: Georgian: a symbol Noto Sans Georgian cannot paint 2026-10-18 09:53:40 -04:00
Brandon ThomasandClaude Opus 5.5 70265ed17a Batch 42: #1893's heap test passes the Copy DecodeOptions without clone (clippy 1.99)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 09:53:39 -04:00
Michael Anderson 19008bdee9 Test cancelling a name lookup while it waits 2026-10-18 08:36:24 -05:00
Brandon Thomas 56c9b49634 Merge PR #1919: Hasselblad FFF: colour matrix from the embedded 3FR header
# Conflicts:
#	docs/parity-checklist.md
#	docs/raw-parity.md
2026-10-18 09:22:57 -04:00
Brandon Thomas 2c60d2ab27 Merge PR #1778: Model downloads: Time out and cancel the name lookup (#1532) 2026-10-18 09:22:41 -04:00
Brandon Thomas 4b6db59ade Merge PR #1903: LRC-PRINT-PAGE: Page Options print page numbers, page info and crop marks (gap 15)
# Conflicts:
#	docs/gaps.md
#	docs/parity-checklist.md
2026-10-18 09:22:40 -04:00
Brandon ThomasandClaude Opus 5.5 cbdbd50f84 Merge main into #1919 (main already dropped the drop(write) line)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 09:21:51 -04:00
Brandon ThomasandClaude Opus 5.5 fe7edd7b26 Model downloads: the What's New note gets its own heading
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 09:21:34 -04:00
Brandon ThomasandClaude Opus 5.5 1e4aa93d58 LRC-PRINT-PAGE: the preview draws no Page Options for a package, as the print doesn't
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 09:21:12 -04:00
Brandon Thomas 7614fb14f1 Merge PR #1902: docs: refresh README features, counts and setup requirements 2026-10-18 09:20:46 -04:00
Brandon Thomas 2983a40385 Merge PR #1960: Merged develop settings are kept in range, as XMP import keeps them 2026-10-18 09:20:45 -04:00
Brandon Thomas 583e3ceefc Merge PR #2091: Reject trailing text after a bracketed IPv6 host in Url::parse 2026-10-18 09:20:43 -04:00
Brandon Thomas 5b66a4825e Merge PR #2150: Keep face index records aligned after a torn or failed append 2026-10-18 09:20:42 -04:00
Brandon Thomas dee8358c38 Merge PR #1838: Drop out-of-range GPS coordinates read from XMP 2026-10-18 09:20:40 -04:00
Brandon Thomas 8d41bf69f7 Merge PR #1894: HEIF: max_pixels on uncropped SPS sizes and grid totals (#1518) 2026-10-18 09:20:39 -04:00
Brandon Thomas f4e7d07e9b Merge PR #1989: TIFF: read strips and tiles larger than 128 MiB 2026-10-18 09:20:37 -04:00
Brandon Thomas 4e77935e6e Merge PR #2070: Show the applied profile when hovering it in the profile browser (#2057) 2026-10-18 09:20:36 -04:00
Brandon Thomas 7134099101 Merge PR #1691: LR-EXP-TYPE: omit redundant alpha from opaque 8-bit exports (#1670) 2026-10-18 09:20:34 -04:00
Brandon Thomas ad22d059db Merge PR #1844: fix(import): keep Lightroom luminance ranges open at white, report unmapped local sliders
# Conflicts:
#	contributors/people.toml
2026-10-18 09:20:33 -04:00
Brandon Thomas 29384a0404 Merge PR #1898: LR-MERGE-HEADLESS: merge dialogs remember the last settings per kind 2026-10-18 09:20:21 -04:00
Brandon Thomas 70100a34c5 Merge PR #2144: Strip XML 1.0-forbidden characters from XMP sidecar output 2026-10-18 09:20:19 -04:00
Brandon Thomas 363dbbdd9f Merge PR #2224: X-Trans CMOS 4 bodies take the X-T3's spectral matrices 2026-10-18 09:20:17 -04:00
Brandon Thomas 80e2d0e64a Merge PR #1884: AI denoise: support 102 MP Bayer raws (#1880) 2026-10-18 09:20:16 -04:00
Brandon Thomas d3b6266d07 Merge PR #1964: Contributors: add Marco Arnulfo
# Conflicts:
#	contributors/people.toml
2026-10-18 09:20:09 -04:00
Brandon Thomas c9f383b368 Merge PR #2040: Ignore zero or negative ISO when importing XMP 2026-10-18 09:19:45 -04:00
Brandon Thomas 389be0f6a9 Merge PR #2105: l10n: unify zh-hant library term and contact sheet wording 2026-10-18 09:19:43 -04:00
Brandon Thomas f819c8439b Merge PR #2151: Make TIFF chunk-length inference and header-plan merging fast 2026-10-18 09:19:42 -04:00
Brandon Thomas e6a7501adc Merge PR #1416: Localize Info Overlay mode-change notices 2026-10-18 09:19:29 -04:00
Brandon Thomas 0a0cb19011 Merge PR #1893: JPEG XL: check dimensions during loading & bound tracked allocations (#1517) 2026-10-18 09:19:27 -04:00
Brandon Thomas 7b65a6a59f Merge PR #1911: Contributors: credit nledford by real name 2026-10-18 09:19:26 -04:00
Brandon Thomas d43889145b Merge PR #2046: Start crop drags in the zone the cursor showed at the press 2026-10-18 09:19:24 -04:00
Brandon ThomasandClaude Opus 5.5 6985cfa54c Drop the README notes: toast language is not README material
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 09:18:48 -04:00
Brandon Thomas 592cb335cb Merge PR #1935: raw: alias Canon EOS R6 Mark II to EOS R6m2 2026-10-18 09:18:21 -04:00
Brandon Thomas d6a9a2e8f9 Merge PR #1845: Match f:/aperture:/focal: search tokens at f32 precision 2026-10-18 09:18:19 -04:00
Brandon Thomas 11308a5c8a Merge PR #2035: Reject out-of-range start in photo.rename and photo.renamePreview 2026-10-18 09:18:17 -04:00
Brandon Thomas 9bf08f6ce6 Merge PR #2083: Reject signed hex digits in #rrggbb colour parsing 2026-10-18 09:18:16 -04:00
Brandon Thomas 5c85bc3ee4 Merge PR #2149: Refuse catalog snapshots with an out-of-range seq 2026-10-18 09:18:14 -04:00
Hamza 0c3c500ea4 Georgian: a symbol Noto Sans Georgian cannot paint
`cargo xtask ci` fails with the craft-fonts input supplied, which CLAUDE.md says
releases always build with:

    ka in Proportional (faces: ["Noto Sans Georgian"]) lack glyphs: ✕

Two strings in ka.json use ✕ (U+2715). Noto Sans Georgian has no glyph for it
and egui's bundled fonts do not either, so a Georgian reader gets a blank where
the text describes the activity stack's stop control. Without the fonts input
those twenty tests skip, so a plain `cargo xtask ci` is green and this stayed
invisible.

The two Georgian values use × (U+00D7) instead: it reads the same at text size
and every face has it. The English keys are untouched — they are the source text
from docs/whats-new.md, and the test checks values, which is the right scope. An
English reader's font covers ✕; the problem is a translation carrying it into a
script whose face does not.

`translated_catalogs_do_not_use_missing_menu_triangles` already forbids ▸ and ▾
in locale JSON for the same reason, so ✕ joins them in one list. That matters
because ✕ is in the English source of those two strings: every language that
translates them next would hit the same failure, and a cheap test now catches it
instead of the font-dependent one.
2026-10-18 18:12:18 +05:00
NetterSilverandClaude Haiku 5.5 601ddce6ef Match f:/aperture:/focal: search tokens at f32 precision
Found while testing crates/catalog/src/query.rs:130

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-18 13:49:17 +02:00
Marco Arnulfo b9159b0fce TIFF: keep the default 128 MiB intermediate buffer for small pixel budgets 2026-10-18 13:18:49 +02:00
Yorgos S. 863adcac6f X-Trans CMOS 4 bodies take the X-T3's spectral matrices
An X-T30 RAF shot in Acros embeds a black-and-white camera JPEG, so the
per-file look fits only the tone curve, and the colour came from the
neutral "camera RGB ~ sRGB" fallback: a grey sky and muted colours. The
raw decode (lossless compressed) was correct.

Add a hand-kept SAME_SENSOR alias list next to the generated
rawtoaces-data table: the 26.1 MP X-Trans CMOS 4 bodies (X-T30, X-T30 II,
X-Pro3, X100V, X-S10, X-E4) use the X-T3's measured matrices, and their
white balance reads in Kelvin through them.
2026-10-18 11:46:07 +02:00
Brandon ThomasandClaude Opus 5.5 4f0b110bcb Release 0.9.1: the release web build installs the wasm-bindgen CLI
Since the web zip moved from trunk to cargo xtask web (aeaf4154), packaging/web/package.sh needs the
wasm-bindgen CLI at Cargo.lock's version, and the shared release workflow installs only trunk, so
0.9.0's Web (wasm) job failed. In CI, install it first (as FilmCraft and EffectCraft do).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-17 23:16:37 -04:00
Brandon ThomasandClaude Opus 5.5 20ccedf56e Release 0.9.0
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-17 21:42:57 -04:00
Alex Chambers 43a47e5a3b Issue #1952: clip non-finite float image samples safely (#2071) 2026-10-17 18:40:14 -06:00
NetterSilverandClaude Haiku 5.5 cebb2be4d4 Keep the zone when shifting a capture time without seconds (#2146)
Found while testing crates/catalog/src/dates.rs:52 (shift_iso, tail = iso.trim().get(19..)); used by crates/engine/src/cmd/manage.rs:222

Co-authored-by: Claude Haiku 5.5 <[email protected]>
2026-10-17 18:39:42 -06:00
NetterSilverandClaude Haiku 5.5 f48b2ad895 Search words with an unknown field prefix as plain text (#2099)
Found while testing crates/catalog/src/query.rs:146-177 (token_matches)

Co-authored-by: Claude Haiku 5.5 <[email protected]>
2026-10-17 18:39:08 -06:00
NetterSilverandClaude Haiku 5.5 0d2c77c258 Fix Assisted Culling bursts that cross midnight at month end (#2095)
Found while testing crates/engine/src/cmd/cull.rs:17-20 (secs, used by the burst grouping at cull.rs:45)

Co-authored-by: Claude Haiku 5.5 <[email protected]>
2026-10-17 18:38:33 -06:00
NetterSilverandClaude Haiku 5.5 5589827178 Fix offline reverse geocoding for cities at longitude exactly 180 (#2061)
Found while testing crates/geocode/src/lib.rs:105-108 (cell) vs lib.rs:189-192 (reverse)

Co-authored-by: Claude Haiku 5.5 <[email protected]>
2026-10-17 18:37:42 -06:00
skylar taylor-barrick 2218d97e89 Merge pull request #2199 from skylartaylor/open-original-media-gate
Open Original: only file types LightCraft reads reach the system opener
2026-10-17 17:07:18 -06:00
skylar taylor-barrick f9b0c8c8a1 Merge pull request #2175 from skylartaylor/linked-path-checks
Follow catalog paths only on this computer or the catalog's share
2026-10-17 12:14:13 -06:00
skylar taylor-barrick 2b173f0bee Build: take craft-docpath from the merged craft-libs rev
craft-libs#169 is merged, so the fork pin is no longer needed. Pinned to
storytold/craft-libs 82a24b006eefa036b43c09a67fb9a3fbd17ffe63 (main after the merge) rather than the older 12281d07,
which predates resolve_absolute and the Windows colon refusal.
2026-10-17 12:03:10 -06:00
skylar taylor-barrickandClaude e6587c3f82 chore: clear the rustc 1.96 clippy lints that fail cargo xtask ci
Unrelated to the #2135 security fix in the commit before this one: both lines
predate this branch and are rustc/clippy 1.96 lint churn. `cargo clippy
--workspace --all-targets` only warns about them, but `cargo xtask ci` runs it
with `-D warnings` (xtask/src/main.rs), so the gate fails before any test runs.

- `clippy::nonminimal_bool` in `slide_overlay::shadow` (reported twice, two
  overlapping suggestions on the one expression): `!(A && B) || !C || !D` is
  `!(A && B && C && D)` by De Morgan, which is what the line now says. The
  operands are pure float comparisons and `is_finite` guards with no side
  effects, so this is an exact rewrite — a NaN or infinite `mx0`/`my0` still
  makes `shadow` return without drawing, as before. (Clippy's second suggestion
  drops the `my0.is_finite()` term and is *not* equivalent; it is not used.)
- `clippy::drop_non_drop` on `drop(write)` in
  `same_folder_as_original_refuses_photos_without_a_folder`. The line is not
  dead: `write` is a closure holding a mutable borrow of `written`, and the
  `drop` ended that borrow so the next assertion could read it. The closure now
  lives in a block of its own instead, which ends the borrow at the brace
  rather than relying on `drop` of a non-`Drop` type. Every assertion of the
  test is unchanged.

Co-Authored-By: Claude <[email protected]>
2026-10-17 10:33:47 -06:00
skylar taylor-barrickandClaude d54673bb95 #2135: only file types LightCraft reads reach the system opener
Photo ▸ Edit In ▸ Open Original in Other Editor handed a photo's stored path
straight to `open` / `xdg-open` / `explorer.exe`, with `Path::is_file()` as the
only check, and Lightroom catalog import kept original paths exactly as the
catalog had them. A crafted `.lrcat` could therefore put `payload.exe`,
`start.lnk`, `run.bat`, `page.hta`, `Evil.app` or `go.command` into the library
and have it run on one user click.

- `lightcraft_engine::open_path`: one gate for every path that leaves
  LightCraft for the OS. The extension must be one LightCraft reads
  (`import::EXTENSIONS`, which already covers the Edit In copy formats
  `tif`/`tiff`/`psd`), it must be a regular file (so a `.app` bundle and any
  other folder are out), what a link resolves to must be a read type too, and
  what is handed over is the absolute path. A refusal is the sentence shown to
  the user.
- Open Original and Edit In go through it; a refused photo is listed in
  `skipped` with the reason, and the menu item is greyed out for a record that
  can never be opened (extension only, so no disk or share is touched while a
  menu is drawn).
- `lightroom_catalog::refuse_non_media` drops such images where the catalog is
  read and again where its photos become records, with a warning each, so a
  refused entry never becomes an openable library record. A virtual copy of a
  refused original goes with it.

Tests: a catalog naming a program (and a virtual copy of it) imports neither
while the catalog's real photos do; a library record naming a program is never
handed to the stub launcher, which records no invocation, and a program
selected beside a photo leaves only the photo, by absolute path. Both fail
before this change.

Co-Authored-By: Claude <[email protected]>
2026-10-17 10:18:47 -06:00
adrdsouzaandCodex a8bc882092 Correct JPEG XL test target guard & tracked-budget claims (#1517)
Pitch-Impact: none

Co-authored-by: Codex <[email protected]>
2026-10-17 21:13:49 +05:30
adrdsouzaandCodex 9c4fce176f Record CI-generated codecs test dependency lockfile
Co-authored-by: Codex <[email protected]>
2026-10-17 20:50:59 +05:30
adrdsouzaandCodex 16beb4db40 Bound JPEG XL input feeds through the final frame (#1517)
Retain the pre-frame size check and allocation budget while avoiding untracked input-sized buffers. Add the reported heap regression and container/progressive compatibility checks.

Co-authored-by: Codex <[email protected]>
2026-10-17 20:38:18 +05:30
NetterSilverandClaude Haiku 5.5 f6d465c72c Make TIFF chunk-length inference and header-plan merging fast
Fixes #2128

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-17 14:50:59 +02:00
NetterSilverandClaude Haiku 5.5 fafa70724b Keep face index records aligned after a torn or failed append
Fixes #2129

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-17 14:33:33 +02:00
NetterSilverandClaude Haiku 5.5 035d572ac8 Refuse catalog snapshots with an out-of-range seq
Fixes #2130

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-17 13:57:37 +02:00
NetterSilverandClaude Haiku 5.5 e0c02d5e89 Strip XML 1.0-forbidden characters from XMP sidecar output
Fixes #2131

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-17 13:12:52 +02:00
wly 8e268431e0 l10n: unify zh-hant library term and contact sheet wording
Use 照片庫 for "library" everywhere (it was split between 照片庫 and
圖庫), and 縮圖目錄 for the contact sheet progress label to match the
File menu item. Only translations change; English keys and format
placeholders are untouched.
2026-10-17 15:08:07 +08:00
aiongg 62eb6627a6 Build: use craft-docpath from craft-libs 2026-10-17 02:21:02 +00:00
NetterSilverandClaude Haiku 5.5 edc369565e Reject trailing text after a bracketed IPv6 host in Url::parse
Found while testing crates/fetch/src/http.rs:87-102

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-17 04:17:46 +02:00
NetterSilverandClaude Haiku 5.5 80e439f6dc Reject signed hex digits in #rrggbb colour parsing
Found while testing crates/slideshow/src/settings.rs:296-300 (parse_color); same pattern in crates/engine/src/slideshow.rs:117-122 (fadeColor)

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-17 00:45:25 +00:00
MarekSpodBiedry 5be931e48a Show the applied profile when hovering it in the profile browser (#2057) 2026-10-17 00:23:38 +02:00
aiongg 7b934ca2b2 Sidecars: write XMP only beside an existing local original 2026-10-16 17:57:30 +00:00
aiongg 346681561a Lightroom import: follow catalog paths only on this computer or the catalog's share 2026-10-16 17:44:52 +00:00
NetterSilverandClaude Haiku 5.5 3accb56196 Start crop drags in the zone the cursor showed at the press
Fixes #1930

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-16 16:32:19 +00:00
NetterSilverandClaude Haiku 5.5 0a9675c6d9 Ignore zero or negative ISO when importing XMP
Found while testing crates/meta/src/xmp.rs:509-512

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-16 14:54:06 +00:00
NetterSilverandClaude Haiku 5.5 312dd04830 Reject out-of-range start in photo.rename and photo.renamePreview
Found while testing crates/engine/src/rename.rs:919 (start parsed at crates/engine/src/cmd/manage.rs:12)

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-16 16:21:24 +02:00
Marco Arnulfo b67bb01952 TIFF: read strips and tiles larger than 128 MiB (fixes #1983) 2026-10-16 07:04:45 +02:00
skylar taylor-barrick f34c7a8a89 Merge pull request #967 from dylanmtaylor/refactor/dry-github-actions
ci: refactor workflows to use storytold/craft-common-actions
2026-10-15 21:34:12 -06:00
skylartaylor 3885b61e6b ci: retain LightCraft installer and package checks 2026-10-16 05:25:18 +02:00
Dylan M. Taylor 0802892132 ci: refactor workflows to use storytold/craft-common-actions 2026-10-15 21:19:37 -04:00
Marco Arnulfo c8f741fd0b Contributors: add my name for the About window 2026-10-16 02:20:02 +02:00
Marco Arnulfo c5deea05e4 Merged develop settings are kept in range, as XMP import keeps them (fixes #1946) 2026-10-16 00:23:44 +02:00
FredApps 9bec9db541 LR-1670: refresh opaque RGB export against main 2026-10-15 22:24:56 +02:00
skybluu fa715c8c94 Localize Info Overlay mode notices (#911) 2026-10-15 15:10:36 -05:00
Claude 6938be39e4 Engine tests: drop the drop() of a closure (clippy drop_non_drop)
Clippy 1.97 rejects `drop(write)` in the same-folder export test: the
closure has no Drop, so its borrow of `written` already ends at its last
use and the call did nothing. Removed, with a note on why the following
read of `written` compiles.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01J2NDVXA8jQQWeJb4YVTb6b
2026-10-15 19:17:57 +00:00
naryzach dde403946b raw: alias Canon EOS R6 Mark II to EOS R6m2
Canon EOS R6 Mark II writes its Exif model as 'Canon EOS R6 Mark II', which failed the spectral camera table lookup because the table registers the model as 'EOS R6m2'. Add the alias mapping and regression unit tests in lightcraft_raw and lightcraft_catalog.
2026-10-15 12:17:25 -07:00
Claude b5dc500c54 Hasselblad 3FR / FFF: uncompressed RGB embedded previews
These files carry no JPEG: their preview is an uncompressed 8-bit RGB
IFD (1217 x 913 in the H4D-40 FFF, 218 x 218 in the CFV 3FR). The
largest such image is handed on by embedded_preview as a small baseline
TIFF, which lightcraft_codecs::decode reads, so 3FR / FFF show a preview
in the grid and loupe before they are developed. No Orientation tag is
written: the raw file's applies, as for JPEG previews without one. The
FFF's 16-bit copy (with a TransferFunction of unknown meaning) is not
used.

decode_raw_preview applied a container-declared colour space by forcing
the JPEG decoder on any owned preview; it now checks for JPEG bytes, so
a wrapped TIFF (or a JPEG XL preview) goes to its own decoder.

Checked on the CC0 H4D-40 FFF and CFV 3FR through
embedded_preview_srgb: right colours, the 3FR upright through its
Rotate90. Tests cover the wrap, the choice among RGB images, a short
strip and truncated files.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01J2NDVXA8jQQWeJb4YVTb6b
2026-10-15 18:58:05 +00:00
Claude 0f67225e83 Hasselblad FFF: colour matrix from the embedded 3FR header
An FFF (a 3FR converted on the desktop) has no colour matrix of its own:
the camera's stays in the original 3FR header the FFF carries as a
complete TIFF in private IFD0 tag 0xC51B. The new vendor::hasselblad
reader decodes 3FR / FFF through dng::decode_as and, when the file has
no matrix, takes the matrices from that header, so an FFF no longer
develops with the "camera RGB = sRGB" fallback (H4D-40 as-shot white
moves from xy 0.279, 0.356 to 0.325, 0.336).

RGGB checked against the camera preview on the CC0 H4D-40 FFF and CFV
3FR (raw.pixls.us 931 and 709). Synthetic tests cover the embedded
matrix, an own matrix winning, a missing or damaged header, and
truncated files.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01J2NDVXA8jQQWeJb4YVTb6b
2026-10-15 18:19:15 +00:00
Nathaniel LedfordandClaude d020c55062 Contributors: credit nledford by real name
Co-Authored-By: Claude <[email protected]>
2026-10-15 11:39:17 -04:00
Michael Anderson 25d466fd69 Merge remote-tracking branch 'upstream/main' into fix/1532-dns-deadline
# Conflicts:
#	docs/whats-new.md
2026-10-15 09:14:06 -05:00
Lucas LepskiandGPT-6.1 Sol 7ac6dd81fe DOC-README: refresh current features and setup requirements
Align feature descriptions, counts, exports and language options with the
current code. Distinguish model-dependent features and implemented Print
and slideshow workflows from their remaining gaps.

Co-Authored-By: GPT-6.1 Sol <[email protected]>
2026-10-15 15:40:53 +02:00
Marco Arnulfo 79d726f856 LRC-PRINT-PAGE: Page Options print page numbers, page info and crop marks (gap 15)
The Print dialog's Page section gets Lightroom's Page Options for Single Image
and Contact Sheet: the page number at the bottom right, the print settings line
(sharpening, profile, resolution) at the bottom left, and crop marks at the
corners of each photo. They are `print.set {page: {pageNumbers, pageInfo,
cropMarks}}`, saved with templates, drawn in the preview and composed into the
PDF; marks and text turn white on a dark page.
2026-10-15 15:38:03 +02:00
Matthew James SivakandClaude Opus 5.5 a43cb68593 LR-MERGE-HEADLESS: merge dialogs remember the last settings per kind
The HDR / Panorama / HDR Panorama dialogs now open with the options of the
last merge of that kind instead of the defaults, and those options are kept
in the library's prefs.json (Session::last_merge) so they survive a restart.
"With Last Settings" and HDR for Each Bracket Set read the same store.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 14:04:14 +02:00
Adrian D'souzaandClaude Opus 5.5 8a4d6f2ebf HEIF: hold uncropped SPS sizes and a grid's tiles together to max_pixels (#1518)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 17:23:19 +05:30
Adrian D'souzaandClaude Opus 5.5 112e202fed JPEG XL: check the size from the header before reading frames, under an allocation budget (#1517)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 17:23:18 +05:30
justinjohn0306 de038ca3d4 #1880: allow 102 MP AI denoise inputs 2026-10-15 16:31:06 +05:30
Brandon Thomas 0da998f24d Merge pull request #1867 from storytold/claude/integrate-batch-40
Integration batch 40: 3 rebased community PRs (GPU face detection, soft-proof zoom windows, faster folder sync) and restored tracker rows
2026-10-15 04:00:13 -04:00
Brandon ThomasandClaude Opus 5.5 e8856c828c Batch 40: raw-parity action 3 (verify popular Nikon, Fujifilm, Canon CR2 and Pentax bodies) is done (#1045, #1081)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:45:11 -04:00
Brandon ThomasandClaude Opus 5.5 b17483a3ce Batch 40: tracker rows that earlier batches' doc merges dropped are back
Batches 36's doc conflicts kept main's side of rows both sides had touched; for some, main had not changed the row,
so the PR's update was lost. Restored: LR-VIEW-FILMSTRIP (#1482), KEYC-COMPARE (#1491 + #1555), LR-IMP-LOCAL
(#1547 + #1820), LR-LIB-STACK (#1556), gaps 28 (#1555), 2 (#1029) and 23 (#1038), ROADMAP UI/UX (#1038). The
raw-parity by-maker and by-use tables are re-measured with coverage.py (545 verified, ~94% / ~85%).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:45:00 -04:00
Brandon ThomasandClaude Opus 5.5 e6dceb9f89 Batch 40: #1027's LR-VIEW-ZOOM row (✅, mask and proof zoom windows) kept; parity summary refreshed
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:43:21 -04:00
Brandon Thomas 5e9f7f1a65 Merge PR #1027 (rebased)
# Conflicts:
#	ROADMAP.md
#	docs/gaps.md
#	docs/parity-checklist.md
#	docs/target-app-parity.md
2026-10-15 03:40:37 -04:00
Brandon Thomas 9ddb8d1824 Merge PR #1663 (rebased)
# Conflicts:
#	crates/ui-egui/locales/de.json
#	crates/ui-egui/locales/es.json
#	crates/ui-egui/locales/fr.json
#	crates/ui-egui/locales/ja.json
#	crates/ui-egui/locales/pt-br.json
#	crates/ui-egui/locales/ru.json
#	crates/ui-egui/locales/uk.json
#	crates/ui-egui/locales/zh-hans.json
#	crates/ui-egui/locales/zh-hant.json
2026-10-15 03:40:17 -04:00
Brandon Thomas 8291f08ab7 Merge PR #1602 (rebased) 2026-10-15 03:40:15 -04:00
Brandon ThomasandClaude Opus 5.5 1d7699056e Merge PR #1027
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:36:18 -04:00
Brandon ThomasandClaude Opus 5.5 2179473536 Merge PR #1663
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:24:51 -04:00
Brandon Thomas 132b0e3ac8 Merge pull request #1865 from storytold/claude/integrate-batch-39
Integration batch 39: 4 rebased community PRs (Georgian, slider scrub and reset hints, straight brush lines) and the Windows ARM64 font pin
2026-10-15 03:08:47 -04:00
Brandon ThomasandClaude Opus 5.5 00dced3dae Batch 39: #1829's straight brush lines carry pen pressure as the author merged them with #1033 (the whole masks.rs from the PR head)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:07:35 -04:00
Brandon ThomasandClaude Opus 5.5 4212907656 Merge PR #1602
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:02:56 -04:00
Brandon Thomas fd80e4eb2f Merge PR #1829 (updated head) 2026-10-15 03:01:12 -04:00
Brandon ThomasandClaude Opus 5.5 d6f63f244f Batch 39: parity summary refreshed
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 03:00:59 -04:00
Brandon ThomasandClaude Opus 5.5 fd9d045c75 CI: the packaging scripts check out the craft-fonts revision release.yml pins (3d9ab3a, with Noto Sans Thai)
Windows ARM64 on main failed since batch 36: #1479's scripts pinned 8dcdacd, which predates the Thai face #1311
made required.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:57:33 -04:00
Brandon ThomasandClaude Opus 5.5 a22f9a6d46 Batch 39: hang guards of the slideshow PDF export and slow-import tests allow a loaded machine (both pass in seconds alone)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:51:35 -04:00
Brandon ThomasandClaude Opus 5.5 51d8018e27 Merge PR #1823 (rebased)
Georgian next to Italian in the language table, menus, catalogs list and docs; ka-formats.json lists the format
strings other PRs added, in English until translated.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:39:01 -04:00
Brandon ThomasandClaude Opus 5.5 f0b5f0339a Batch 39: #1474's hint test expects the value hint #1604 gave the scrubbable value
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:38:20 -04:00
Brandon Thomas 88ca6ed572 Merge PR #1474 (rebased)
# Conflicts:
#	crates/ui-egui/src/widgets.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 02:32:41 -04:00
Brandon Thomas 780872b158 Merge PR #1604 (rebased)
# Conflicts:
#	docs/parity-checklist.md
2026-10-15 02:32:16 -04:00
Brandon Thomas 809259a37d Merge PR #1829 (rebased)
# Conflicts:
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 02:32:15 -04:00
Brandon ThomasandClaude Opus 5.5 bc39923608 Merge PR #1604
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:28:24 -04:00
Brandon ThomasandClaude Opus 5.5 d9ed118c1d Merge PR #1474
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:13:45 -04:00
Michael Anderson fc6719d1db Merge remote-tracking branch 'upstream/main' into feat/brush-shift-click
# Conflicts:
#	docs/whats-new.md
2026-10-15 01:07:50 -05:00
Michael Anderson 888a54f52a Merge remote-tracking branch 'upstream/main' into fix/1532-dns-deadline
# Conflicts:
#	docs/whats-new.md
2026-10-15 01:05:38 -05:00
Brandon Thomas a82a001d20 Merge pull request #1857 from storytold/claude/integrate-batch-38
Integration batch 38: 6 rebased community PRs (Nikon camera profiles, lens-corrected colour, HDR frame order, focus points)
2026-10-15 02:00:39 -04:00
Brandon Thomas 2f1acef58f Merge PR #1595 (updated head) 2026-10-15 02:00:04 -04:00
Brandon ThomasandClaude Opus 5.5 a39e85da72 Merge PR #1829
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 02:00:02 -04:00
Brandon ThomasandClaude Opus 5.5 a94fc6bc84 Batch 38: What's New lists the Nikon profile and lens-corrected colour notes once
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:59:50 -04:00
Brandon Thomas 3f69729d69 Merge PR #1595 (updated head) 2026-10-15 01:59:36 -04:00
Brandon Thomas 0270d30eda Merge PR #1592 (updated head) 2026-10-15 01:59:36 -04:00
Brandon Thomas 956420ea5f Merge PR #1586 (updated head) 2026-10-15 01:59:36 -04:00
Brandon ThomasandClaude Opus 5.5 e44e7157fc Batch 38: rustfmt and parity summary after the merges
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:58:56 -04:00
Michael Anderson c877e8e9ee Merge remote-tracking branch 'origin/feat/nikon-z6ii-camera-profile' into feat/nikon-z8-d5300-profiles
# Conflicts:
#	README.md
#	README.zh-hans.md
#	crates/engine/src/camera_preview.rs
#	crates/engine/src/tests_xmp.rs
#	docs/camera-preview-colour.md
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 00:58:42 -05:00
Michael Anderson 296e2ce73c Merge remote-tracking branch 'origin/fix/colour-fit-lens-geometry' into feat/nikon-z6ii-camera-profile
# Conflicts:
#	crates/engine/src/camera_preview.rs
#	crates/engine/src/tests_xmp.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 00:55:18 -05:00
Michael Anderson a77eafb32a Merge remote-tracking branch 'upstream/main' into feat/brush-shift-click
# Conflicts:
#	docs/whats-new.md
2026-10-15 00:54:23 -05:00
Michael Anderson aed214da7e Merge remote-tracking branch 'upstream/main' into fix/1532-dns-deadline
# Conflicts:
#	docs/whats-new.md
2026-10-15 00:50:50 -05:00
Brandon ThomasandClaude Opus 5.5 73ee54fe62 Batch 38: format catalogs list the strings this batch adds, in English until translated
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:48:46 -04:00
Brandon Thomas 11351ef141 Merge PR #1834 (rebased)
# Conflicts:
#	crates/ui-egui/locales/cs-formats.json
#	crates/ui-egui/locales/pl-formats.json
#	crates/ui-egui/src/lib.rs
2026-10-15 01:48:46 -04:00
Brandon ThomasandClaude Opus 5.5 202fcbb525 Merge PR #1823
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:48:43 -04:00
Michael Anderson 9ad2c1766b Merge remote-tracking branch 'upstream/main' into fix/colour-fit-lens-geometry
# Conflicts:
#	crates/engine/src/camera_preview.rs
#	crates/engine/src/files.rs
#	docs/whats-new.md
2026-10-15 00:48:09 -05:00
Brandon Thomas d5b2cd1474 Merge PR #1595 (rebased)
# Conflicts:
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 01:48:05 -04:00
Brandon Thomas 97959610be Merge PR #1592 (rebased)
# Conflicts:
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 01:47:46 -04:00
Brandon Thomas 99eba2e069 Merge PR #1586 (rebased)
# Conflicts:
#	crates/engine/src/camera_preview.rs
#	crates/engine/src/files.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 01:45:53 -04:00
Brandon Thomas 0fb809c671 Merge PR #1614 (rebased)
# Conflicts:
#	crates/merge/src/hdr.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-15 01:44:24 -04:00
Brandon Thomas e4519aa710 Merge PR #1487 (rebased)
# Conflicts:
#	docs/parity-checklist.md
2026-10-15 01:43:02 -04:00
Brandon ThomasandClaude Opus 5.5 26e43cd691 Merge PR #1595
Stacked on rb/1592: LOOK_VERSION 8 -> 9; the PR's corpus-profile helper is the base's corpus_uses_the_bundled_profile.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:42:11 -04:00
Brandon ThomasandClaude Opus 5.5 480e96c030 Merge PR #1592
Stacked on rb/1586: LOOK_VERSION 7 -> 8; README lists the base's ILCA-99M2 profile too.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:42:11 -04:00
Brandon ThomasandClaude Opus 5.5 2756414f46 Merge PR #1586
Resolved against rb-base-36: LOOK_VERSION 6 -> 7; starting_colour_tags passes no lens (merge frames arrive without it).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:42:11 -04:00
Brandon ThomasandClaude Opus 5.5 3c0300fcbb Merge PR #1614
Resolved against rb-base-36: docs combine the 1 px clip neighbourhood and frame order with the base's moving-highlight rule and 2048 px deghost.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:42:11 -04:00
Brandon ThomasandClaude Opus 5.5 b5733c7649 Merge PR #1487
Resolved against rb-base-36: Scanned keeps the base's rewrite flag next to magic and tag; Exif keeps the Panasonic lens fallback; locale catalogs merged (one duplicate key dropped).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:42:11 -04:00
Brandon Thomas 3d003e8c64 Merge pull request #1852 from storytold/claude/integrate-batch-37
Integration batch 37: 20 rebased community PRs (print module, decode speed-ups, camera coverage, Italian)
2026-10-15 01:37:33 -04:00
Brandon ThomasandClaude Opus 5.5 dec7fd4eb2 Merge PR #1834
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 01:26:47 -04:00
Michael Anderson dc562c654a Merge remote-tracking branch 'upstream/main' into feat/brush-shift-click
# Conflicts:
#	crates/engine/src/cmd/masks.rs
#	crates/ui-egui/src/panels/detail.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-14 23:45:14 -05:00
Ege Gursel 31f4ad43fe contributors: re-add my people.toml entry lost in the merge 2026-10-15 00:29:43 -04:00
Michael Anderson ef216d2110 Merge remote-tracking branch 'upstream/main' into fix/1532-dns-deadline
# Conflicts:
#	docs/whats-new.md
2026-10-14 23:29:27 -05:00
Michael Anderson bcffde1a06 Merge remote-tracking branch 'upstream/main' into feat/nikon-z8-d5300-profiles
# Conflicts:
#	README.md
#	README.zh-hans.md
#	crates/engine/src/camera_preview.rs
#	docs/camera-preview-colour.md
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-14 23:28:25 -05:00
Michael Anderson 68ef6ff3b3 Merge remote-tracking branch 'upstream/main' into feat/nikon-z6ii-camera-profile
# Conflicts:
#	README.md
#	README.zh-hans.md
#	crates/engine/src/camera_preview.rs
#	docs/camera-preview-colour.md
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-14 23:26:01 -05:00
Ege Gursel f906998174 Merge branch 'main' into fix/xmp-import-local-grain-luminance-range 2026-10-15 00:25:35 -04:00
Ege Gursel 1130bda456 fix(import): keep Lightroom luminance ranges open at white, report unmapped local sliders
Two of the three problems reported in #1840:

- A luminance range at full strength up to white (LumRange "a b 1 1",
  a typical highlights mask) ended at our scene white (0 EV) with no
  feather, so anything brighter (the sun, speculars, a sky after a
  positive exposure) fell out of the mask along a hard edge. That end
  now stays open.
- Correction sliders we have no counterpart for (LocalGrain, ...) were
  dropped silently. They are now listed in `unmapped` as
  "Mask: LocalGrain"; LocalCurveRefineSaturation 100 counts as neutral.

Adobe base looks, the third problem, are handled by #1469.

Also adds my entry to contributors/people.toml.
2026-10-15 00:23:41 -04:00
Michael Anderson f9076514e7 Merge remote-tracking branch 'upstream/main' into fix/colour-fit-lens-geometry
# Conflicts:
#	crates/engine/src/camera_preview.rs
#	docs/parity-checklist.md
#	docs/whats-new.md
2026-10-14 23:22:42 -05:00
NetterSilverandClaude Haiku 5.5 48d69bfae5 Drop out-of-range GPS coordinates read from XMP
Found while testing crates/meta/src/xmp.rs:364 (parse_gps_coord, used at xmp.rs:539-544)

Co-Authored-By: Claude Haiku 5.5 <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 05:21:00 +02:00
Michael Anderson 2953b63576 Merge remote-tracking branch 'upstream/main' into feat/brush-shift-click
# Conflicts:
#	docs/parity-checklist.md
2026-10-14 21:05:05 -05:00
Michael Anderson 15becc9eb4 Merge remote-tracking branch 'upstream/main' into feat/nikon-z8-d5300-profiles
# Conflicts:
#	docs/parity-checklist.md
2026-10-14 20:56:06 -05:00
Michael Anderson a68da45fc3 Merge remote-tracking branch 'upstream/main' into fix/colour-fit-lens-geometry
# Conflicts:
#	docs/parity-checklist.md
2026-10-14 20:55:01 -05:00
Michael Anderson 35f347397d Merge remote-tracking branch 'upstream/main' into feat/nikon-z6ii-camera-profile
# Conflicts:
#	docs/parity-checklist.md
2026-10-14 20:54:03 -05:00
Nathaniel LedfordandClaude Opus 5.5 62da9e8581 LRC-LIB-LIBFOLDERS: drag photos from the grid onto a Folders row to move them there
Photos dragged from the grid onto a Folders row move there on disk and in the library
(`photo.moveToFolder`: files and XMP sidecars, virtual copies follow, never overwrites, one undo
step), as Move Selected Photos Here in the row's menu does.
- The row lights up, and the label following the pointer says where they go: "Move 2 photos to
  “Porto”" (onto an album the same gesture only adds them, so the two read apart).
- A closed folder held under the pointer for 0.7 s opens, so a folder inside it can be reached.
- Refused, with the reason on release and a not-allowed pointer: a row that covers several disks,
  and Local photos (not in the library). Photos already there say "Already in “Porto”".
- A drop asks first, as Lightroom Classic does: "Move 2 photos to “Porto”?" (their files and
  XMP sidecars move on disk; Edit → Undo moves them back), with Don't ask again, kept in Settings
  as "Confirm before moving photos to another folder on disk" (on by default). The menu's Move
  Selected Photos Here, chosen by name, goes at once. A question left open while another library
  opens (the menu bar and agents reach past a dialog) moves nothing: photo ids are per library.
- The toast says how many moved and that Edit → Undo moves them back; the menu item says the same.
- Photos that leave the view leave the selection, so a Delete or edit next can't reach them unseen.

For NAS and network shares: the copy across volumes now streams (it read the whole file into
memory, a problem for a large raw or video) into a free hidden name that takes the photo's name only
once complete (a share dropping mid-copy left a truncated file under the photo's name, which a
retry reported as the photo already there). New engine tests cover a share going away:
Undo with the share offline or read-only is refused with the reason and kept until it can run,
nothing is deleted and the library stays linked; a connection lost partway through a move puts
back what moved; a file the share can't give back rolls the whole Undo back; and a real
move across devices where the machine has a second file system (`/dev/shm`).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-14 20:58:08 -04:00
Michael Anderson e5c6e17bdb LR-MASK-BRUSH: Shift-click draws a straight brush stroke from the last one
A Shift-click with the brush paints a straight line from where the last stroke ended, tapering size, feather and flow when they changed in between, as one undo step. mask.brushStroke takes connect: true for it.
2026-10-14 19:11:34 -05:00
Davit MatchakhelidzeandClaude Opus 5.5 fb2a460532 i18n(ka): bring the Georgian catalog up to date
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-15 00:38:48 +04:00
Davit MatchakhelidzeandClaude Opus 5.5 9b9a96a1b5 i18n(ka): bring the Georgian catalog up to date
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-14 23:22:36 +04:00
Michael Anderson e65b97e239 Model downloads: Time out and cancel the name lookup (#1532)
The host name lookup blocked with no deadline before the cancel check, and each address then got the full connect timeout. The lookup now runs on a helper thread under the connect deadline, which all addresses share.
2026-10-14 13:56:34 -05:00
Davit MatchakhelidzeandClaude Opus 5.5 af0c1638fe i18n: Georgian (ka) interface language
Adds Georgian (ქართული, code ka, system tags ka / ka-GE / ka_GE) with
complete plain and formats catalogs, the Edit > Language command and the
Settings choice. Sentence-cased messages go through i18n::upper_first,
which leaves Georgian letters alone (upper-casing Mkhedruli would turn it
into Mtavruli). The Settings language choices wrap now that eleven
endonyms no longer fit on one line, and the face-model Remove button goes
through tr().

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01YMTAbGNRxnMEzRvYnQGphF
2026-10-14 22:50:28 +04:00
Claude ac17c580f5 Merge storytold/lightcraft main into LR-MERGE-HDR (frame order, clipped pixels)
Brings in #1423 (HDR merges keep the lens corrections), which landed in batch 35 / v0.8.0. hdr.rs merged cleanly:
the result's lens corrections come from the reference frame after the brightness ordering. Conflicts only in docs:
the LR-MERGE-HDR row keeps main's lens-correction text plus this branch's two items, and this branch's What's New
entry moves to the 2026-10-14 section.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_015gRtdWF1E9358FfzhzYqCS
2026-10-14 16:22:37 +00:00
Jason Baldwin 2b766142bc Merge origin/main into focus-point (tracker header/summary/change log, whats-new entry moved under 2026-10-14) 2026-10-14 14:33:49 +02:00
Michael Anderson ead24aded9 LR-PROF-CAMERACOLOR: Add a bundled Nikon D5300 camera profile
Fitted with lightcraft-cli calibrate to 264 of my own D5300 photos.
2026-10-14 07:32:56 -05:00
Michael Anderson 975b08c334 LR-PROF-CAMERACOLOR: Add a bundled Nikon Z 6II camera profile
Fitted with lightcraft-cli calibrate to 296 of my own Z 6II photos.
2026-10-14 07:32:56 -05:00
Michael Anderson 2bc2f60277 LR-PROF-CAMERACOLOR: Add a bundled Nikon Z 8 camera profile
Fitted with lightcraft-cli calibrate to 261 of my own Z 8 photos.
2026-10-14 07:32:56 -05:00
Michael Anderson c854e172e4 LR-PROF-CAMERACOLOR: Match the camera look against lens-corrected JPEGs
The fit compared the uncorrected raw with the corrected JPEG. It now warps the raw with the file's own lens correction when that lines up better.
2026-10-14 07:32:47 -05:00
Nathaniel LedfordandClaude Opus 5.5 540a281ccd PERF-SYNC: a Synchronize Folder scan reads off the folder's listing whether a photo and its sidecar are there
The scan asked the disk about every library photo in the folder: is its
file there, and has it a sidecar under any of four names. On a network
share each question is a round trip, and the scan had just listed the
folder. Measured on a hard disk over SMB, for unchanged folders of 1,600
to 2,400 photos: 4.6 to 7.1 s of questions after a listing of 4 to 5 s.

- The walk hands each folder's listing over (`walk::files_in_noting`:
  every entry's name and kind, hidden ones too, and whether that is all of
  them). The scan walks the folder once, then checks the photos beside
  the probing of new files.
- `sync::Listing` answers "is it there" / "is it a file" where that holds
  on any file system: a name the folder lists is there; a plain ASCII name
  is not there when the folder was listed to its end, all its names are
  ASCII and none is that name in another case. Everything else is asked
  of the disk as before (a link, a name Windows reads another way, a
  folder that was not walked, and every name not listed in a folder that
  holds a name outside ASCII).

Now under 0.05 s for those folders. Two things differ on purpose: a file
the folder lists is there though it can't be opened by its path (a folder
that may be listed but not searched, an over-long path), where such
photos were listed as missing; and the answers are the folder as the
walk saw it.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-14 07:45:43 -04:00
Andrew FrahnandClaude Opus 5.5 61ec5c596c Docs: restore upstream's header notes in gaps and parity checklist
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-14 20:02:09 +11:00
Andrew FrahnandClaude Opus 5.5 f625c74c7a Faces: whole networks on the GPU, batched faces, CPU-only switch, paced GPU use
Activations stay on the device between layers (net/program.rs lowers the graph; every
operator is a WGSL kernel, one submission and read-back per run); faces of a photo go
in batches of up to 16; Settings > Faces shows each model's device and a CPU-only
switch (faces.gpu); the scan's pace limits GPU use; the GPU check is an activity row.
YuNet 58 -> 21 ms, SFace 40 -> 9.6 ms, AuraFace 295 -> 71 ms (heavily loaded machine).

Sources: own design.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-14 19:43:44 +11:00
FredAppsandCodex 2b86ca0537 LR-EXP-TYPE: omit redundant alpha from opaque 8-bit exports (#1670)
Preserve real transparency, pixels and embedded metadata. Cover RGB PNG,
TIFF compression modes and WebP, plus real-alpha round trips.

Co-authored-by: Codex <[email protected]>
2026-10-14 10:23:40 +02:00
Andrew FrahnandClaude Opus 5.5 d598c304aa Merge upstream/main into pr/zoom-detail
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-14 19:20:53 +11:00
Andrew FrahnandClaude Opus 5.5 0242a75b5a Faces: run the networks' convolutions on the GPU (wgpu) when it agrees with the CPU and is faster
An accelerator hook in the face interpreter takes Conv nodes; lightcraft-gpu implements it with a WGSL
implicit-GEMM convolution (any stride, padding, grouping; split-K for small maps; a direct kernel for depthwise
layers), each model's weights resident on a device of its own. After loading, detection and recognition check
the GPU in the background (agreement within 1e-3 and faster than the CPU) and otherwise stay on the CPU, which
remains the reference; the first GPU error drops it. Sources: own design.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-14 12:22:50 +11:00
Andrew FrahnandClaude Opus 5.5 80a8acaa30 Merge upstream/main into pr/zoom-detail
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-14 07:52:45 +11:00
good-news-everyoneandClaude Opus 5.5 ab64e9a563 KEY-SLIDER-SCRUB: drag a slider's value sideways to scrub it
As over Lightroom's values, a press on the number next to a slider's name
that moves sideways now changes the value as dragging the thumb would: the
pointer's travel over the track's width, a tenth as far with Shift, on the
control's steps and range, as one undo step. A press that moves less than
3 points sideways is still a click that opens the value for typing
(KEY-SLIDER-TYPE), and the pointer over the value is the horizontal-resize
cursor. The tooltip says both ("Click to type a value, drag to change it"),
translated in every catalog.

Tests: scrub distance and direction, Shift, clamping, steps, one interaction
per scrub, no field or draft left behind, a jittery click still types, a
disabled slider doesn't scrub. The readout test that drags away from the
value now drags straight down, so it still checks that a cancelled press
keeps neither a value nor the keyboard.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-13 19:55:17 +03:00
Jason Baldwin fdad5a4ad4 LR-VIEW-FOCUSPOINT: read focus points for photos already in the library, header-only in the background (#1480)
Photos imported before focus points were read got one only through Reload from Disk, which
hashes whole files on the UI thread. Now:

- lightcraft_meta::focus::read_header reads only what the decoder looks at: IFD0 (Make,
  Model), the Exif IFD, the maker note's IFD and the focus tags' values (a JPEG: its segments
  up to the Exif one), walked round by round and decoded with the whole-file code from a
  buffer of the file's length holding just those ranges. Bounded rounds, bytes (2 MB) and
  size (1 GB). One 32 KB read per α1 / A7S III ARW; on 109 real files (83 with a point)
  it gives the whole-file read's point for every one.
- A background job (crates/engine/src/focus_scan.rs) reads photos with no point from a
  supported body (by the stored camera; nothing else is opened), ARW or JPEG, one at a
  time in path order, with a "Reading focus points" activity row (Cancel::Flag). The UI's
  per-frame focus.pump gives it the face scan's pace (paused while dragging or typing) and
  starts it once per library opening. Commits every 200 photos or 3 s as system ops (no
  undo step, no sidecar write), only meta.focus, only on photos that still have none at the
  same path. "Looked, none" goes to focus-scanned.bin beside the library, headed by the
  decoder's tag so a decoder knowing more bodies starts afresh; unreadable files are retried
  at the next opening.
- photo.readFocusPoints {ids?, wait?} (Photo > Read Focus Points) runs it on demand.

Measured cold on a 5400 rpm USB drive: 200 α1 / A7S III ARWs in 1.9 s (~100 photos/s,
32 KB each), so 58,796 raws take about 10 minutes instead of re-reading 2.25 TB.
2026-10-13 16:06:06 +02:00
Jason Baldwin ac84cd31ee LR-VIEW-FOCUSPOINT: show the camera's focus point (Sony) and zoom to it (#1480)
Read the autofocus point from Sony maker notes (FocusLocation 0x2027, FocusFrameSize
0x2037, FocusMode 0x201b, AFAreaModeSetting 0x201c, AFTracking 0x2021 and the area mode
used from the enciphered 0x9402 block), for the ILCE-1 and ILCE-7SM3, whose layout was
checked against ExifTool's output on 333 real ARWs. Manual focus, centre-only Multi
reports and unchecked bodies give no point. The box is stored on the upright photo as
Meta.focus (omitted when unknown; no catalog format change), filled at import and by
Reload from Disk.

View > Show Focus Point (Alt+F) draws it in Detail, Compare and Survey through rotation
and crop; View > Zoom to Focus Point (on by default) makes Toggle Zoom centre on it; the
Info panel shows "AF-C · Wide · Animal Eye Tracking". photo.inspect returns meta.focus.
2026-10-13 13:14:24 +02:00
Claude 951d52788a LR-MERGE-HDR: pixels next to clipped ones don't tint HDR merges
Measured against the darkest frame on real ARW brackets, each frame is linear to 1 % up to the clip far from
clipped pixels, but within 2 px of them the +2 EV frame's green reads 3–5 % low: demosaicing interpolates it from
clipped photosites. That frame carries the most weight (k = 4) and those pixels were unclipped themselves, so they
kept it, and bright windows took a magenta tinge.

A frame's weight now goes by its brightest channel within 1 px (3 × 3; wider margins changed nothing more). On two
real interior brackets (ILCE-7M3, 0 / −2 / +2 EV), bright pixels whose green against red and blue is over 10 % below
the darkest frame's, in camera RGB: dining room 0.32 % → 0.08 %, kitchen 2.86 % → 0.28 %. Rendered at −3 EV with
main's colour tags (#1123), window pixels with G < 0.85·min(R, B) go 50 → 3 and 16 → 10 (the darkest frame alone:
2 and 205); most of the visible magenta those brackets showed before came from the missing colour tags. Test: a
bracket whose +2 EV frame has 5 % less green next to its clipped pixels merges without the deficit (fails before:
−1.96 %).

Docs: docs/merge.md, the LR-MERGE-HDR row, docs/whats-new.md (also for the frame-order fix).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_015gRtdWF1E9358FfzhzYqCS
2026-10-13 09:51:11 +00:00
Atmik Shetty cb8b797cd4 KEY-SLIDER-RESET: a slider's name and thumb say a double-click resets it (fixes #1436)
A double-click on a slider's name or thumb already reset it, but nothing said so: only the value had a
tooltip. Resting the pointer on the name or on the thumb now shows "Exposure — double-click to reset
it", the wording (and translations) the curve channels already use. The rest of the track, which is
clicked and dragged, gets no tooltip, and the value keeps "Click to type a value".

The first click of a double-click on the thumb moved the value to where the pointer landed, a few
points off its centre: one history step, then the reset another. A click on the thumb now leaves the
value alone, so the reset is one undo step, as from the name. A disabled slider's name no longer
resets it.

No per-slider reset button: Lightroom has none, and the double-click it documents is now shown where
it works. Tests: tests_slider_reset.rs (name and thumb reset Exposure and Contrast in one undo step,
the hints, a click on the thumb keeps the value, typing a value still works) and
widgets/tests_slider_reset.rs (a disabled slider's name).
2026-10-13 13:16:08 +05:30
Claude 146cb7b975 LR-MERGE-HDR: the merge no longer depends on the order the frames come in
The same bracket merged in another order gave a different result: on real ARW brackets, the app's selection
order and the CLI's file order differed on 0.6 % of the pixels (up to 102 levels), while the same order gave
byte-identical files. Alignment seeds were the frames' input positions, and sums, deghost masks and brightness
ties followed them too.

merge_hdr now puts the frames in brightness order first (ties by median, then a content fingerprint, never by
position), runs everything in that order, and returns the per-frame results (EV, alignment, reference index)
in the caller's order. Test: every permutation of a bracket gives the same radiance and deghost overlay bit for
bit, and the same per-frame results (fails before: "the radiance differs").

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_015gRtdWF1E9358FfzhzYqCS
2026-10-13 05:57:13 +00:00
Andrew FrahnandClaude Opus 5.5 c78b457c0d LR-VIEW-ZOOM: soft proofing at 1:1 renders a zoom window; 1:1 pixels tested equal to export
Soft proof is per-pixel, so the After window carries the proof like the
whole-frame render under it (the proof key is part of the tile's look).
New tests: a 1:1 window of a 24 MP photo equals the full-size export to
within one level; soft proofing at 1:1 is a canvas-sized frame plus a
native window. profile_slider_drag gains a soft-proof case. Gap 4 closed,
checklist row to done with measured Linux latencies.

Sources: own design.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-12 03:37:35 +11:00
Andrew FrahnandClaude Opus 5.5 76f12dcabd Issue #989: fill_box reads its buffer with as_chunks::<3>() (clippy 1.99)
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-12 03:37:35 +11:00
205 changed files with 19797 additions and 1209 deletions

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
version: 2
updates:
# Maintain Rust / Cargo dependencies
- package-ecosystem: "cargo"
directory: "/"
schedule:
interval: "daily"
time: "06:00"
timezone: "America/New_York"
# Consolidate updates into a single PR rather than one PR per crate
groups:
cargo-dependencies:
patterns:
- "*"
commit-message:
prefix: "chore(deps)"
# Maintain GitHub Actions workflows
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
time: "06:00"
timezone: "America/New_York"
groups:
actions:
patterns:
- "*"
commit-message:
prefix: "ci(deps)"
+19 -38
View File
@@ -1,16 +1,23 @@
# Builds and tests the workspace on FreeBSD, inside a FreeBSD VM on a Linux runner
# (GitHub has no FreeBSD runners). GPU tests skip themselves when the VM has no adapter.
# FreeBSD build and package check in a QEMU VM.
name: FreeBSD
on:
push:
branches: [main]
pull_request:
paths:
- Cargo.toml
- Cargo.lock
- "**/Cargo.toml"
- .github/workflows/freebsd.yml
- ".github/workflows/freebsd.yml"
- "packaging/freebsd/**"
- "Cargo.toml"
- "Cargo.lock"
- "crates/**"
pull_request:
branches: [main]
paths:
- ".github/workflows/freebsd.yml"
- "packaging/freebsd/**"
- "Cargo.toml"
- "Cargo.lock"
- "crates/**"
workflow_dispatch:
concurrency:
@@ -22,34 +29,8 @@ permissions:
jobs:
freebsd:
name: FreeBSD 14 (build + test)
runs-on: ubuntu-latest
timeout-minutes: 300
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Optional font build input, so the Japanese font tests run (they skip without it).
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
path: craft-fonts
- name: Build and test in a FreeBSD VM
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
with:
release: "14.3"
usesh: true
cpu: 4
mem: 12288
# Nothing to bring back; copying the target dir out of the VM fails on large builds.
copyback: false
prepare: |
pkg install -y git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal
run: |
. "$HOME/.cargo/env"
# The checkout is synced in under another owner; let tests that call git read it.
git config --global --add safe.directory "*"
rustc --version
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked
CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/freebsd.yml@main
with:
require-craft-fonts: true
craft-fonts-ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
+6 -62
View File
@@ -1,7 +1,5 @@
# Quick checks for the release pipeline (no Rust build): workflow syntax, packaging
# shell/PowerShell scripts, and the Linux desktop/AppStream/Flatpak metadata (seconds), plus a
# Windows job that builds and installs throwaway per-user MSI probes for the install-folder
# searches (a few minutes).
# Quick static checks for the release pipeline (seconds, no Rust build): workflow syntax,
# packaging shell/PowerShell scripts, and the Linux desktop/AppStream/Flatpak metadata.
name: Packaging lint
on:
@@ -21,64 +19,10 @@ concurrency:
jobs:
lint:
timeout-minutes: 15
runs-on: ubuntu-latest
env:
ACTIONLINT_VERSION: 1.7.12
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install tools
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends shellcheck desktop-file-utils appstream libxml2-utils python3-yaml
curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" \
| tar -xz -C "$RUNNER_TEMP" actionlint
- name: actionlint
run: |
"$RUNNER_TEMP/actionlint" -color
- name: shellcheck
run: shellcheck packaging/*.sh packaging/*/*.sh
- name: PowerShell
shell: pwsh
run: |
foreach ($f in Get-ChildItem packaging/windows/*.ps1) {
$errors = $null
[System.Management.Automation.Language.Parser]::ParseFile($f.FullName, [ref]$null, [ref]$errors) | Out-Null
if ($errors) { $errors | ForEach-Object { Write-Output "::error file=$($f.Name)::$_" }; exit 1 }
Write-Output "ok $($f.Name)"
}
- name: XML, desktop entry, AppStream, Flatpak manifest
run: |
xmllint --noout packaging/windows/lightcraft.wxs packaging/macos/*.plist* packaging/linux/*.xml*
# Windows Installer needs Icon ids ending in .ico or .exe, or shortcuts show a blank icon.
if grep -oE '<Icon Id="[^"]*"' packaging/windows/lightcraft.wxs | grep -vE '\.(ico|exe)"$'; then
echo "::error::Icon Id must end in .ico or .exe"; exit 1
fi
# The interactive installer asks where to install and ends on a completion page (issues #18, #399).
for want in '<UIRef Id="LightcraftUI" />' 'Value="InstallDirDlg"' '<Property Id="WIXUI_INSTALLDIR" Value="INSTALLFOLDER" />' \
'Key="Software\LightCraft" Name="InstallFolder"' 'Value="LaunchLightcraft"'; do
grep -qF "$want" packaging/windows/lightcraft.wxs || { echo "::error::lightcraft.wxs lost: $want"; exit 1; }
done
# The completion page the Publish events point at must actually exist: a Publish alone
# does not pull the dialog fragment into the MSI, and without the DialogRef the
# installer finishes silently (issues #790, #892).
grep -qF '<DialogRef Id="ExitDialog" />' packaging/windows/lightcraft.wxs \
|| { echo "::error::ExitDialog is not referenced; installs and removes end without a completion dialog"; exit 1; }
desktop-file-validate packaging/linux/ai.storyteller.lightcraft.desktop
sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.lightcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
python3 - <<'EOF'
import yaml
src = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.yml"))
bundle = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.bundle.yml"))
for m in (src, bundle):
assert m["id"] == "ai.storyteller.lightcraft" and m["command"] == "lightcraft", m["id"]
for key in ("runtime", "runtime-version", "sdk", "finish-args"):
assert src[key] == bundle[key], f"Flatpak manifests disagree on {key}: {src[key]} vs {bundle[key]}"
print("Flatpak manifests ok")
EOF
- name: FreeBSD tarball layout (stub binaries)
run: bash packaging/freebsd/package.sh --dry-run
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/packaging-lint.yml@main
with:
extra-checks-script: packaging/check-app-specific.sh
windows-install-folder:
runs-on: windows-latest
+25 -479
View File
@@ -1,13 +1,5 @@
# Release pipeline: every push to `release` builds platform artifacts (signed where credentials
# are configured) and creates or updates the draft GitHub Release "LightCraft v<version>".
# See docs/releasing.md.
#
# The signing jobs (macOS, Windows) and the draft-release job run in the `release` environment
# (restricted to the `release` branch), which holds the signing secrets. Every secret is optional:
# missing ones produce unsigned artifacts and a warning, never a failed build. The jobs that sign
# nothing (version, Linux, Linux riscv64, Flatpak, FreeBSD, web) run without it, so a dispatch on
# any branch is a dry run of them; the release job needs the signed jobs, so a branch run never
# drafts a release.
# Release pipeline: every push to `release` builds signed installers for every platform and
# creates or updates the draft GitHub Release "LightCraft v<version>". See docs/releasing.md.
name: Release
on:
@@ -21,479 +13,33 @@ on:
default: ""
permissions:
contents: read
contents: write
concurrency:
group: release-${{ github.ref }}
# Never cancel a run mid-notarization or mid-upload; a newer push waits its turn.
cancel-in-progress: false
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
RUST_BACKTRACE: 1
LIGHTCRAFT_BUILD_SHA: ${{ github.sha }}
INPUT_VERSION: ${{ inputs.version }}
jobs:
version:
timeout-minutes: 10
name: version
runs-on: ubuntu-latest
# No `environment: release` here or on the other jobs that sign nothing: no release secrets.
outputs:
version: ${{ steps.v.outputs.version }}
date: ${{ steps.v.outputs.date }}
prerelease: ${{ steps.v.outputs.prerelease }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- id: v
name: Resolve version
run: |
. packaging/env.sh
v="${INPUT_VERSION#v}"
v="${v:-$VERSION}"
if ! printf '%s' "$v" | grep -Eq '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'; then
echo "::error::'$v' is not a version like 1.2.3 or 1.2.3-rc.1"
exit 1
fi
case "$v" in *-*) pre=true ;; *) pre=false ;; esac
{
echo "version=$v"
echo "date=$(date -u +%Y-%m-%d)"
echo "prerelease=$pre"
} >>"$GITHUB_OUTPUT"
echo "### LightCraft v$v (\`${GITHUB_SHA::9}\`)" >>"$GITHUB_STEP_SUMMARY"
macos:
timeout-minutes: 90
name: macOS universal
needs: version
runs-on: macos-15-xlarge # M2 Pro larger runner; use only in release.yml (actions_macos_xl budget caps spend)
environment: release
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: aarch64-apple-darwin, x86_64-apple-darwin
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-macos-universal
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
- name: Import signing certificate
env:
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
run: packaging/macos/import-cert.sh
- name: Build, sign, notarize
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
run: packaging/macos/package.sh --arch universal
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-macos
path: dist/release/*
if-no-files-found: error
- name: Remove temporary keychain
if: always()
run: |
if [ -n "${MACOS_KEYCHAIN:-}" ]; then security delete-keychain "$MACOS_KEYCHAIN" || true; fi
windows:
timeout-minutes: 90
name: Windows ${{ matrix.arch }}
needs: version
runs-on: windows-latest
environment: release
strategy:
fail-fast: false
matrix:
include:
- { arch: x64, target: x86_64-pc-windows-msvc }
- { arch: x86, target: i686-pc-windows-msvc }
# Windows on ARM: cross-compiled on the x64 runner, so signing and WiX run as above.
# .github/workflows/windows-arm64.yml installs and runs it on ARM64 hardware.
- { arch: arm64, target: aarch64-pc-windows-msvc }
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: ${{ matrix.target }}
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-windows-${{ matrix.arch }}
- name: Apply version override
if: inputs.version != ''
shell: bash
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
# package.ps1 adds the matching WixToolset.UI/Util.wixext (5.0.2) to the global extension cache.
- name: Install WiX
shell: pwsh
run: dotnet tool install --global wix --version 5.0.2
- name: Build, sign, package
shell: pwsh
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }}
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
run: ./packaging/windows/package.ps1 -Arch ${{ matrix.arch }}
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-windows-${{ matrix.arch }}
path: dist/release/*
if-no-files-found: error
linux:
timeout-minutes: 90
name: Linux ${{ matrix.arch }}
needs: version
# The oldest GitHub-hosted image, so the binaries only need glibc >= 2.35 (Ubuntu 22.04+,
# Debian 12+, Fedora 36+, RHEL 10). See docs/releasing.md.
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- { arch: x86_64, runner: ubuntu-22.04, deb: amd64 }
- { arch: aarch64, runner: ubuntu-22.04-arm, deb: arm64 }
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
NFPM_VERSION: 2.47.0
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- name: Install build and packaging tools
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev \
desktop-file-utils appstream python3-yaml zsync
curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" \
"https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_${{ matrix.deb }}.deb"
sudo dpkg -i "$RUNNER_TEMP/nfpm.deb"
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-linux-${{ matrix.arch }}
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
- name: Build and package (AppImage, deb, rpm, tar.gz)
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: packaging/linux/package.sh
- name: Check the packages
run: |
dpkg-deb --info dist/release/*.deb
dpkg-deb --contents dist/release/*.deb | grep -E 'usr/bin/lightcraft$|metainfo|applications'
ldd target/release/lightcraft
python3 -c "import yaml; m = yaml.safe_load(open('packaging/linux/flatpak/ai.storyteller.lightcraft.yml')); assert m['id'] == 'ai.storyteller.lightcraft'"
for appimage in dist/release/*.AppImage; do
chmod +x "$appimage"
APPIMAGE_EXTRACT_AND_RUN=1 "$appimage" --version
# Update information is embedded, and the .zsync for delta updates sits beside it.
"$appimage" --appimage-updateinformation | grep -F 'gh-releases-zsync|'
test -s "$appimage.zsync"
done
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-linux-${{ matrix.arch }}
path: dist/release/*
if-no-files-found: error
flatpak:
timeout-minutes: 90
name: Flatpak ${{ matrix.arch }}
needs: [version, linux]
# Repackages the Linux job's tarball (the same binaries as the AppImage/deb/rpm); no Rust
# build here. 24.04 for a newer flatpak-builder (1.4) than 22.04 ships.
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- { arch: x86_64, runner: ubuntu-24.04 }
- { arch: aarch64, runner: ubuntu-24.04-arm }
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: release-linux-${{ matrix.arch }}
path: dist/linux
- name: Install flatpak-builder
run: |
sudo apt-get update
# librsvg2-common: the SVG pixbuf loader. flatpak-builder runs the host's
# `appstreamcli compose`, which renders the scalable icon and fails without it.
sudo apt-get install -y --no-install-recommends flatpak flatpak-builder librsvg2-common
# Ubuntu 24.04's AppArmor blocks the unprivileged user namespaces bubblewrap needs.
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
- name: Build the Flatpak bundle
run: packaging/linux/flatpak-bundle.sh "dist/linux/lightcraft-$LIGHTCRAFT_VERSION-linux-${{ matrix.arch }}.tar.gz"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-flatpak-${{ matrix.arch }}
path: dist/release/*.flatpak
if-no-files-found: error
linux-riscv64:
timeout-minutes: 90
name: Linux riscv64
needs: version
# Cross-compiled on x86_64 (no hosted RISC-V runners) against the runner's own Ubuntu release
# from ports.ubuntu.com, so the binaries need that release's glibc (24.04: 2.39). 22.04's
# riscv64 ports lag its amd64 updates, which breaks the multiarch install. tar.gz only; the
# CLI is smoke-tested under QEMU.
runs-on: ubuntu-24.04
# No `environment: release`: this job signs nothing, so it gets no release secrets.
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: riscv64gc-unknown-linux-gnu
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-linux-riscv64
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
- name: Install cross-compilation dependencies
run: |
. /etc/os-release
sudo dpkg --add-architecture riscv64
# Pin the existing (archive.ubuntu.com) sources to amd64; riscv64 comes from ports.
if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
sudo sed -i '/^Types: deb/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
fi
if [ -f /etc/apt/sources.list ]; then
sudo sed -i 's/^deb /deb [arch=amd64] /g' /etc/apt/sources.list
sudo sed -i 's/^deb-src /deb-src [arch=amd64] /g' /etc/apt/sources.list
fi
for suite in "$VERSION_CODENAME" "$VERSION_CODENAME-updates" "$VERSION_CODENAME-security"; do
echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports $suite main universe restricted multiverse"
done | sudo tee /etc/apt/sources.list.d/riscv64.list
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
gcc-riscv64-linux-gnu binutils-riscv64-linux-gnu libc6-dev-riscv64-cross libc6-dev:riscv64 qemu-user \
libxkbcommon-dev:riscv64 libwayland-dev:riscv64 libx11-dev:riscv64 libxrandr-dev:riscv64 \
libxi-dev:riscv64 libgl1-mesa-dev:riscv64
- name: Build and package (tar.gz)
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER: riscv64-linux-gnu-gcc
PKG_CONFIG_SYSROOT_DIR: /
PKG_CONFIG_PATH: /usr/lib/riscv64-linux-gnu/pkgconfig:/usr/share/pkgconfig
PKG_CONFIG_ALLOW_CROSS: "1"
CROSS_ARCH: riscv64
CROSS_TARGET: riscv64gc-unknown-linux-gnu
CROSS_COMPILE: riscv64-linux-gnu-
EMULATOR: qemu-riscv64
QEMU_LD_PREFIX: /usr/riscv64-linux-gnu
run: packaging/linux/package.sh --formats tar
- name: Check the tarball
run: |
tar -tzf dist/release/lightcraft-*-linux-riscv64.tar.gz \
| grep -E '/bin/lightcraft$|/bin/lightcraft-cli$|/share/applications/ai\.storyteller\.lightcraft\.desktop$'
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-linux-riscv64
path: dist/release/*.tar.gz
if-no-files-found: error
freebsd:
name: FreeBSD x86_64
needs: version
# GitHub has no FreeBSD runners: build in a FreeBSD 14.3 VM, the same image and packages as
# .github/workflows/freebsd.yml.
runs-on: ubuntu-latest
timeout-minutes: 180
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- name: Build and package in a FreeBSD VM (tar.gz)
# vmactions/freebsd-vm v1.5.9, pinned by commit (it runs our build).
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
with:
release: "14.3"
usesh: true
cpu: 4
mem: 12288
# Passed into the VM by name; never templated into the script below.
envs: "LIGHTCRAFT_VERSION LIGHTCRAFT_BUILD_DATE LIGHTCRAFT_BUILD_SHA INPUT_VERSION"
# The FreeBSD CI packages, plus bash for the packaging script.
prepare: |
pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
run: |
set -e
. "$HOME/.cargo/env"
git config --global --add safe.directory "$PWD"
# Build outside the checkout so only dist/ is copied back out of the VM (copying the
# target dir back fails on large builds). 4 jobs: more runs the VM out of memory.
export CARGO_TARGET_DIR=/var/tmp/lightcraft-target CARGO_BUILD_JOBS=4
if [ -n "$INPUT_VERSION" ]; then cargo xtask version set "$LIGHTCRAFT_VERSION"; fi
# craft-fonts was checked out next to the sources and synced into the VM with them.
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
bash packaging/freebsd/package.sh
tar -tzf dist/release/lightcraft-*-freebsd-x86_64.tar.gz \
| grep -E '/bin/lightcraft$|/bin/lightcraft-cli$|/share/applications/ai\.storyteller\.lightcraft\.desktop$|/share/doc/lightcraft/LICENSE-MIT$'
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-freebsd-x86_64
path: dist/release/*
if-no-files-found: error
web:
timeout-minutes: 90
name: Web (wasm)
needs: version
runs-on: ubuntu-latest
env:
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
WASM_BINDGEN_VERSION: 0.2.129
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-web
- name: Install wasm-bindgen CLI
run: |
# Must match the wasm-bindgen version in Cargo.lock exactly (cargo xtask web checks it).
if ! { command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION"; }; then
cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
fi
wasm-bindgen --version
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
- name: Build and zip
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: packaging/web/package.sh
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-web
path: dist/release/*
if-no-files-found: error
release:
timeout-minutes: 10
name: Draft GitHub Release
needs: [version, macos, windows, linux, linux-riscv64, flatpak, freebsd, web]
runs-on: ubuntu-latest
environment: release
permissions:
contents: write
env:
VERSION: ${{ needs.version.outputs.version }}
PRERELEASE: ${{ needs.version.outputs.prerelease }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: release-*
path: artifacts
merge-multiple: true
- name: Checksums
working-directory: artifacts
run: |
sha256sum -- * | tee SHA256SUMS.txt
- name: Create or update the draft release
working-directory: artifacts
run: |
tag="v$VERSION"
title="LightCraft v$VERSION"
pre=()
if [ "$PRERELEASE" = true ]; then pre=(--prerelease); fi
if draft="$(gh release view "$tag" --json isDraft --jq .isDraft 2>/dev/null)"; then
if [ "$draft" != "true" ]; then
echo "::error::Release $tag is already published. Bump the version (cargo xtask version set) before pushing to release."
exit 1
fi
echo "Updating draft $tag: replacing its assets"
gh release view "$tag" --json assets --jq '.assets[].name' | while read -r asset; do
gh release delete-asset "$tag" "$asset" --yes
done
gh release edit "$tag" --draft --title "$title" --target "$GITHUB_SHA" --prerelease="$PRERELEASE"
gh release upload "$tag" --clobber -- *
else
echo "Creating draft $tag"
gh release create "$tag" --draft --title "$title" --target "$GITHUB_SHA" --generate-notes "${pre[@]}" -- *
fi
{
echo "Draft release **$title** at \`${GITHUB_SHA::9}\`:"
echo
sed 's/^/ /' SHA256SUMS.txt
} >>"$GITHUB_STEP_SUMMARY"
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/release.yml@main
with:
app-name: lightcraft
app-title: LightCraft
craft-fonts-ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
version: ${{ inputs.version }}
secrets:
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }}
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
+24 -79
View File
@@ -1,22 +1,27 @@
# Windows on ARM: package the ARM64 release the way release.yml does (cross-compiled on an x64
# runner, unsigned here), then install that MSI on a Windows 11 ARM64 runner, check the installed
# binaries are ARM64, run the CLI natively, and uninstall. (A native ARM64 `cargo test` job is left
# out for now: PdfCraft saw an intermittent access violation in egui UI tests there.)
# Windows ARM64 build and smoke test.
#
# Release packages on arm64-windows are produced on x64 runners via cross-compilation
# (target: aarch64-pc-windows-msvc) using MSVC's ARM64 cross tools, then installed and run
# on GitHub Actions' native windows-11-arm runner.
name: Windows ARM64
on:
push:
branches: [main]
pull_request:
paths:
- Cargo.toml
- Cargo.lock
- "**/Cargo.toml"
- "**/build.rs"
- .github/workflows/windows-arm64.yml
- packaging/windows/**
schedule:
- cron: "41 4 * * 1" # weekly, so drift in the runner images shows up
- ".github/workflows/windows-arm64.yml"
- "packaging/windows/**"
- "Cargo.toml"
- "Cargo.lock"
- "crates/**"
pull_request:
branches: [main]
paths:
- ".github/workflows/windows-arm64.yml"
- "packaging/windows/**"
- "Cargo.toml"
- "Cargo.lock"
- "crates/**"
workflow_dispatch:
concurrency:
@@ -26,70 +31,10 @@ concurrency:
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
jobs:
package:
name: package (cross-compiled on x64)
runs-on: windows-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: aarch64-pc-windows-msvc
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: windows-arm64-package
- name: Install WiX
shell: pwsh
run: dotnet tool install --global wix --version 5.0.2
- name: Build and package (unsigned)
shell: pwsh
env:
DIST: ${{ github.workspace }}\dist\arm64
run: ./packaging/windows/package.ps1 -Arch arm64
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: lightcraft-windows-arm64
path: dist/arm64/*
if-no-files-found: error
retention-days: 7
install:
name: install and run on ARM64
needs: package
runs-on: windows-11-arm
timeout-minutes: 30
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: lightcraft-windows-arm64
path: dist
- name: Install the MSI, check it, run it, uninstall it
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
$msi = (Get-ChildItem dist/*.msi | Select-Object -First 1).FullName
$p = Start-Process msiexec.exe -ArgumentList '/i', "`"$msi`"", '/qn', '/l*v', 'install.log' -Wait -PassThru
if ($p.ExitCode -ne 0) { Get-Content install.log -Tail 60; throw "msiexec /i exited $($p.ExitCode)" }
$dir = Join-Path $env:ProgramFiles 'LightCraft'
foreach ($exe in 'lightcraft.exe', 'lightcraft-cli.exe') {
$path = Join-Path $dir $exe
if (-not (Test-Path $path)) { throw "$exe not installed in $dir" }
$bytes = [System.IO.File]::ReadAllBytes($path)
$machine = [BitConverter]::ToUInt16($bytes, [BitConverter]::ToInt32($bytes, 0x3C) + 4)
if ($machine -ne 0xAA64) { throw "$exe is for machine 0x$('{0:X}' -f $machine), not ARM64" }
Write-Output "ok $path (ARM64)"
}
# The installed CLI runs natively on ARM64.
$cli = Join-Path $dir 'lightcraft-cli.exe'
$out = & $cli --version 2>&1 | Out-String
if ($LASTEXITCODE -ne 0) { Write-Output $out; throw "lightcraft-cli --version exited $LASTEXITCODE" }
Write-Output "ok lightcraft-cli --version: $($out.Trim())"
$p = Start-Process msiexec.exe -ArgumentList '/x', "`"$msi`"", '/qn' -Wait -PassThru
if ($p.ExitCode -ne 0) { throw "msiexec /x exited $($p.ExitCode)" }
if (Test-Path (Join-Path $dir 'lightcraft.exe')) { throw 'uninstall left lightcraft.exe behind' }
Write-Output 'ok uninstalled'
windows-arm64:
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/windows-arm64.yml@main
with:
app-name: lightcraft
app-dir: LightCraft
Generated
+41 -33
View File
@@ -1072,6 +1072,11 @@ dependencies = [
"libc",
]
[[package]]
name = "craft-docpath"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=82a24b006eefa036b43c09a67fb9a3fbd17ffe63#82a24b006eefa036b43c09a67fb9a3fbd17ffe63"
[[package]]
name = "craft-ui"
version = "0.1.0"
@@ -3073,7 +3078,7 @@ dependencies = [
[[package]]
name = "lightcraft"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"eframe",
"egui",
@@ -3098,7 +3103,7 @@ dependencies = [
[[package]]
name = "lightcraft-catalog"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"crc32fast",
"lightcraft-develop",
@@ -3115,7 +3120,7 @@ dependencies = [
[[package]]
name = "lightcraft-cli"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"dhat",
"lightcraft-codecs",
@@ -3131,10 +3136,11 @@ dependencies = [
[[package]]
name = "lightcraft-codecs"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"avif-serialize",
"bytemuck",
"dhat",
"image",
"image-webp",
"jixel",
@@ -3161,7 +3167,7 @@ dependencies = [
[[package]]
name = "lightcraft-color"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-geom",
"proptest",
@@ -3170,7 +3176,7 @@ dependencies = [
[[package]]
name = "lightcraft-denoise"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"faer-core",
"lightcraft-denoise-core",
@@ -3187,7 +3193,7 @@ dependencies = [
[[package]]
name = "lightcraft-denoise-core"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-raster",
"miniz_oxide 0.8.9",
@@ -3202,7 +3208,7 @@ dependencies = [
[[package]]
name = "lightcraft-develop"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-color",
"lightcraft-geom",
@@ -3214,9 +3220,10 @@ dependencies = [
[[package]]
name = "lightcraft-engine"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"ab_glyph",
"craft-docpath",
"harfrust",
"jxl-oxide",
"libc",
@@ -3256,7 +3263,7 @@ dependencies = [
[[package]]
name = "lightcraft-faces"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-testkit",
"matrixmultiply",
@@ -3271,7 +3278,7 @@ dependencies = [
[[package]]
name = "lightcraft-fetch"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"log",
"rustls",
@@ -3282,7 +3289,7 @@ dependencies = [
[[package]]
name = "lightcraft-geocode"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"miniz_oxide 0.8.9",
"serde",
@@ -3290,7 +3297,7 @@ dependencies = [
[[package]]
name = "lightcraft-geom"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"proptest",
"serde",
@@ -3298,12 +3305,13 @@ dependencies = [
[[package]]
name = "lightcraft-gpu"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"bytemuck",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
"lightcraft-faces",
"lightcraft-geom",
"lightcraft-pipeline",
"lightcraft-raster",
@@ -3317,7 +3325,7 @@ dependencies = [
[[package]]
name = "lightcraft-heif"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"heic-rs",
"rayon",
@@ -3325,14 +3333,14 @@ dependencies = [
[[package]]
name = "lightcraft-inpaint"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"rayon",
]
[[package]]
name = "lightcraft-lensdb"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-develop",
"lightcraft-geom",
@@ -3341,7 +3349,7 @@ dependencies = [
[[package]]
name = "lightcraft-mcp"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-codecs",
"lightcraft-engine",
@@ -3352,7 +3360,7 @@ dependencies = [
[[package]]
name = "lightcraft-merge"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-codecs",
"lightcraft-color",
@@ -3368,7 +3376,7 @@ dependencies = [
[[package]]
name = "lightcraft-meta"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-geom",
"lightcraft-tiff",
@@ -3380,7 +3388,7 @@ dependencies = [
[[package]]
name = "lightcraft-pipeline"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"image",
"lightcraft-color",
@@ -3398,7 +3406,7 @@ dependencies = [
[[package]]
name = "lightcraft-preview"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"jpeg-encoder",
"lightcraft-raster",
@@ -3410,7 +3418,7 @@ dependencies = [
[[package]]
name = "lightcraft-raster"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"libc",
"lightcraft-color",
@@ -3422,7 +3430,7 @@ dependencies = [
[[package]]
name = "lightcraft-raw"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"dhat",
"jpeg-encoder",
@@ -3446,7 +3454,7 @@ dependencies = [
[[package]]
name = "lightcraft-scenes"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"image",
"lightcraft-color",
@@ -3457,7 +3465,7 @@ dependencies = [
[[package]]
name = "lightcraft-segment"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"candle-core",
"candle-nn",
@@ -3471,7 +3479,7 @@ dependencies = [
[[package]]
name = "lightcraft-slideshow"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"lightcraft-raster",
"serde",
@@ -3480,15 +3488,15 @@ dependencies = [
[[package]]
name = "lightcraft-sysmem"
version = "0.8.0"
version = "0.9.1"
[[package]]
name = "lightcraft-testkit"
version = "0.8.0"
version = "0.9.1"
[[package]]
name = "lightcraft-tiff"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"proptest",
"serde",
@@ -3497,7 +3505,7 @@ dependencies = [
[[package]]
name = "lightcraft-ui-egui"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"craft-ui",
"egui",
@@ -3526,7 +3534,7 @@ dependencies = [
[[package]]
name = "lightcraft-web"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"crc32fast",
"eframe",
@@ -6996,7 +7004,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
[[package]]
name = "xtask"
version = "0.8.0"
version = "0.9.1"
dependencies = [
"brotli",
"flate2",
+2 -1
View File
@@ -3,7 +3,7 @@ resolver = "3"
members = ["crates/*", "apps/*", "xtask"]
[workspace.package]
version = "0.8.0"
version = "0.9.1"
edition = "2024"
license = "MIT OR Apache-2.0"
rust-version = "1.90"
@@ -11,6 +11,7 @@ repository = "https://github.com/storytold/lightcraft"
[workspace.dependencies]
craft-ui = { git = "https://github.com/storytold/craft-libs", rev = "5c874d51eeb3de70e735c8ce87a59a7dedd47675", features = ["serde"] }
craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "82a24b006eefa036b43c09a67fb9a3fbd17ffe63" }
# internal
lightcraft-denoise-core = { path = "crates/denoise-core" }
lightcraft-denoise = { path = "crates/denoise", default-features = false }
+47 -27
View File
@@ -18,7 +18,7 @@
</p>
<p align="center">
<b>Photo library and raw development; an open-source, clean-room reimplementation of Adobe Lightroom, rebuilt in pure Rust.</b><br>
<b>Photo library and raw development; an open-source, non-destructive photo library and raw developer, built in pure Rust.</b><br>
Native on macOS, Windows and Linux. In the browser via WebAssembly. Drivable end to end by AI agents over MCP.
</p>
@@ -129,7 +129,7 @@ Split-tone shadows, midtones and highlights independently with drag-anywhere col
## Before & after
Hold <kbd>\\</kbd> to peek at the original, press <kbd>Y</kbd> for side by side, or <kbd>Shift</kbd>+<kbd>Y</kbd> for a
Press <kbd>\\</kbd> to toggle the Before view, <kbd>Y</kbd> for side by side, or <kbd>Shift</kbd>+<kbd>Y</kbd> for a
split view. Every image below is a real screenshot of LightCraft, captured automatically by an agent through the
[control channel](#built-for-agents).
@@ -150,9 +150,13 @@ split view. Every image below is a real screenshot of LightCraft, captured autom
Paint with a **Brush** (size, feather, flow, density, erase), drop **Linear** and **Radial Gradients** with draggable
pins, or select by **Luminance Range**, **Color Range**, **Sky**, **Subject** and **Background**. Combine components
with **Add / Subtract / Intersect**, invert any of them, and dial in 15 local adjustments per mask (Temp, Tint,
Exposure, Contrast, Highlights, Shadows, Whites, Blacks, Texture, Clarity, Dehaze, Hue, Saturation, Sharpness, Noise)
plus an overall Amount.
with **Add / Subtract / Intersect**, invert any of them, and dial in 17 local adjustments per mask (Temp, Tint,
Exposure, Contrast, Highlights, Shadows, Whites, Blacks, Texture, Clarity, Dehaze, Hue, Saturation, Sharpness, Noise,
Moiré, Defringe), plus an overall Amount.
Desktop **Object** and **Describe** masks use SAM 3 for click-based and text-prompt selection, with a separately
installed model; see [model setup and terms](docs/ai-masks.md). Subject, Sky and Background selections still use
classical heuristics. Existing model-generated masks render and export without the model installed.
<p align="center">
<img src="docs/images/masking.jpg" alt="Masking panel with a linear Sky mask and a radial Sun glow mask; the radial gradient is drawn as a red overlay around the sun on a lake scene" width="100%">
@@ -164,7 +168,7 @@ plus an overall Amount.
## Presets, profiles & the Color Mixer
Eighteen hand-built presets ship in the box (*Golden Hour, Teal & Orange, Faded Matte, Selenium Tone, Crisp
Forty-one hand-built presets ship in the box (*Golden Hour, Teal & Orange, Faded Matte, Selenium Tone, Crisp
Landscape* and more), each with an **Amount** slider from 0 to 200 %. Save your own from any group of settings, mark
favourites, and copy, paste or sync edits across a whole selection with exactly the groups you choose.
@@ -209,6 +213,10 @@ A library that stays out of your way: **All Photos**, **Recently Added**, **Pick
import date, edit date, name, rating, size or at random (a stable shuffle; View → Sort → Reshuffle for a new one). The justified **Photo Grid** and **Square Grid** views are virtualized,
so they stay smooth whether you have forty photos or forty thousand.
The **People** view groups named faces, including names read from XMP; face boxes can be edited in the loupe.
On desktop, automatic detection and recognition use optional models installed through **Settings → Faces**; no weights are
bundled. See [face models and setup](docs/faces.md).
<table>
<tr>
<td width="50%"><img src="docs/images/grid-demo.jpg" alt="Justified Photo Grid of 24 demo photos with ratings and flags, and a sidebar of nested albums under a Travel 2026 folder"><br><sub>Photo Grid with albums nested in a folder, ratings and flags. <i>Demo library.</i></sub></td>
@@ -281,7 +289,13 @@ lightcraft --control 7980 ~/Pictures/trip
print.cell.add width=5 height=7 print.cell.add width=2.5 height=3.5 print.layout.autoLayout print.exportPdf path=Package.pdf
```
Printer colour management and page overlays (identity plate, watermark, page numbers) are not there yet.
The Print Job panel supports printer/paper ICC profiles, rendering intent, black point compensation,
print adjustment, sharpening and 8/16-bit PDF output. Direct printing through an OS print dialog and page
overlays (identity plate, watermark, page numbers) are not there yet.
- **Slideshows:** **View → Slideshow** plays the photos in view full screen, with fades, repeat and random order.
**Settings → Slideshow** configures layouts, templates, backgrounds, overlays and title screens. Native exports
support PDF, JPEG sequences and AV1 video in MP4 or WebM; music and a dedicated Slideshow module editor remain
missing. See [docs/slideshow.md](docs/slideshow.md).
- **Every widget is addressable** (`ui.widgets`) and clickable by name, so agents operate the real UI, not a
side door.
- The screenshots in this README were produced end to end by the [`docs/showcase/`](docs/showcase/) scripts.
@@ -293,7 +307,7 @@ lightcraft --control 7980 ~/Pictures/trip
- **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2/CR3, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans,
Panasonic RW2 / Leica RWL, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP,
PSD composites and JPEG XL open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
PSD composites, JPEG XL, GIF (first frame) and BMP open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
as in PhotoCraft).
- **Scene-referred & wide-gamut.** Linear Rec.2020 float internally, Bradford-adapted white balance, gamut mapping
instead of clipping, a filmic shoulder for raw and pixel-exact pass-through for JPEGs you haven't touched.
@@ -319,17 +333,21 @@ lightcraft --control 7980 ~/Pictures/trip
LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (details in the [roadmap](ROADMAP.md) and
[docs/target-app-parity.md](docs/target-app-parity.md)):
- **By feature count we're at ~81%** of Lightroom (core features 98%), tracked row by row in
- **By feature count we're at ~82%** (core features 98%), tracked row by row in
[docs/parity-checklist.md](docs/parity-checklist.md).
- **As a day-to-day Lightroom replacement we're nearer 61% for the whole product, ~51% for a typical working photographer and ~65% for
- **As a day-to-day tool we're nearer 61% for the whole product, ~51% for a typical working photographer and ~65% for
casual use.** It's great for JPEG/DNG and most Sony / Panasonic /
Fujifilm / Nikon / older-Canon raws on one machine; see [docs/raw-parity.md](docs/raw-parity.md) for your camera.
- **The biggest gaps:**
- **camera colour calibration:** Sony, Nikon, Panasonic, Fujifilm and Canon CR3 raws have guarded estimates from their camera JPEGs, with built-in ILCA-99M2, ILCE-7CR, ILCE-7M4, X-H2S and X-T4 profiles; measured calibration is missing, and other raws or rejected fits retain a neutral matrix;
- **camera colour calibration:** Sony, Nikon, Panasonic, Fujifilm and Canon CR3 raws have guarded estimates from their camera JPEGs, with built-in ILCA-99M2, ILCE-7CR, ILCE-7M4, X-H2S, X-T4, Nikon Z 6II, Z 8 and D5300 profiles; measured calibration is missing, and other raws or rejected fits retain a neutral matrix;
- **compressed Olympus raws and unsupported CR3 variants:** these use embedded JPEG previews when present. Fujifilm lossless/lossy compressed RAF now decodes sensor data; [verification and existing-library reload instructions](docs/raf-compression.md);
- **AI masks and denoise:** subject and sky selection are classical heuristics;
- **AI quality and model setup:** Subject and Sky selection are classical heuristics; Object/Describe use an
optional SAM 3 model. AI Denoise supports Bayer RAW with separately installed weights, while X-Trans and
linear-RGB denoise and Super Resolution remain missing; see [AI masks](docs/ai-masks.md) and [AI Denoise](docs/denoise.md);
- **network libraries:** a preview reads the raw's header — or, for a CR3 or a TIFF raw whose preview tag sits past it, exactly the ranges the sparse planner names — and its embedded JPEG only (56.5 MB → 5.9 MB, 10 s → 1 s on the test share), but the *developed* picture still needs the sensor data over the wire; see [docs/raw-preview-reads.md](docs/raw-preview-reads.md);
- **on-screen HDR display, video and the Classic Print / Book / Map modules.**
- **on-screen HDR display, video import/editing, and the Book / Map module views.** Print layouts and
colour-managed PDF output work; direct printer output and page overlays remain missing. Slideshow playback
and export work, with a dedicated module editor and music still missing.
- **What's next:** see [the ranked gaps](docs/gaps.md) and [the roadmap](docs/roadmap.md).
| Area | Status |
@@ -337,16 +355,16 @@ LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (det
| Library: albums, folders, smart albums, stacks (incl. auto-stack), virtual copies, ratings, flags, labels, filter bar, search, sort, grids, filmstrip | ✅ |
| Culling: Compare (synced zoom) and Survey views, auto-advance, instant previews | ✅ |
| Light, Color, Effects (vignette styles), Tone Curve (+ refine saturation, targeted adjustment), Color Mixer (+ targeted), Point Color, Color Grading, Calibration, B&W | ✅ |
| Masking: brush, linear, radial, luminance/colour range, add/subtract/intersect | ✅ (AI subject/sky use classical heuristics for now) |
| Masking: brush, linear, radial, luminance/colour range, add/subtract/intersect; Object and Describe selections | ✅ · Object/Describe require optional SAM 3; Subject/Sky remain heuristic ([details](docs/ai-masks.md)) |
| Crop, straighten tool + auto straighten, flip, rotate, aspect ratios, overlays | ✅ |
| Profiles (Color, Neutral, Vivid, Landscape, Portrait, Monochrome: our own looks), presets, versions, history, copy/paste/sync settings | ✅ |
| Camera colour: DNG files use their own matrices | ✅ DNG · 🚧 own Sony/Fujifilm profiles; measured calibration database missing |
| Camera colour: DNG matrices, guarded camera-JPEG look fits and built-in profiles for selected cameras | 🚧 measured calibration and per-model coverage remain incomplete ([raw parity](docs/raw-parity.md)) |
| Native macOS menu bar (generated from the command registry), control channel + every widget addressable, headless UI snapshots | ✅ |
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed, incl. lossy after split), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed, 12.8 bpp), Samsung SRW (uncompressed); embedded previews for every format incl. CR3 | 🚧 · other CR3 variants, compressed ORF / SRW and Nikon HE NEF decode ⬜ ([raw parity](docs/raw-parity.md)) |
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise 🚧 (Bayer, user-installed weights), Super Resolution ⬜ |
| Remove / Heal / Clone spots (auto source), Visualize Spots, Red Eye and Pet Eye (auto pupil detection, catchlight) | ✅ · content-aware fill, spot pin editing 🚧 |
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ |
| HDR: HDR editing with a headroom limit, SDR rendition, Visualize HDR; export as ISO 21496-1 gain map JPEG, PQ AVIF or 32-bit float TIFF | ✅ · HDR display ⬜ |
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise 🚧 (Bayer, user-installed weights; [setup](docs/denoise.md)), Super Resolution ⬜ |
| Remove / Heal / Clone spots (auto source), non-generative Fill, editable spot pins, Visualize Spots, Red Eye and Pet Eye | ✅ · Fill uses PatchMatch; no generative fill |
| Export: JPEG / PNG / TIFF / flat PSD / WebP / AVIF / JPEG XL / raw or rendered DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ |
| HDR: HDR editing with a headroom limit, SDR rendition, Visualize HDR; gain map JPEG, PQ/HLG AVIF, PQ PNG/JPEG XL and float TIFF export | ✅ · HDR display ⬜ |
| Library persistence (crash-safe op log + snapshots, background compaction, failed saves reported), disk thumbnail cache | ✅ |
| Import: Add in place / Copy / Move, rename and folder templates, devices, duplicate detection, watched folders; Local folder browsing | ✅ |
| MCP server (headless or live app, persistent libraries), CLI, control channel | ✅ |
@@ -354,7 +372,11 @@ LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (det
| Optics (distortion, vignetting, auto + manual CA, defringe, lens corrections embedded in DNG files, Panasonic / Leica RW2 / RWL distortion data and Nikon Z 6II / Z 8 / Z f distortion data), Geometry (transforms, Constrain Crop), Upright (Auto/Level/Vertical/Full/Guided) | ✅ · camera lens profiles (our own) ⬜ |
| Photo Merge: HDR (auto-align, deghost), Panorama (spherical/cylindrical/perspective, boundary warp, auto crop), HDR Panorama → DNG | ✅ |
| GPU pipeline (wgpu compute, CPU-exact within 1/255), CPU fallback on device limits / errors | ✅ · WebGPU in the browser 🚧 |
| AI: segmentation masks, AI denoise, super resolution, faces; video | 🚧 (see [gaps](docs/gaps.md)) |
| People view, XMP face names and editable face boxes; automatic face detection and recognition | ✅ · detection/recognition require optional models ([setup](docs/faces.md)) |
| GPX tracklog auto-tagging and offline reverse geocoding | ✅ · place names require the GeoNames dataset; Map view ⬜ |
| Print layouts, picture packages, templates and colour-managed PDF export | ✅ layouts/PDF · 🚧 direct printer output and page overlays |
| Slideshow playback, layouts, overlays, templates and PDF/JPEG/AV1 export | ✅ playback · 🚧 dedicated module editor and music ([details](docs/slideshow.md)) |
| Video import/editing and Book module | ⬜ |
| Web build (same UI in the browser via WASM): persistent library in OPFS/IndexedDB, Web Worker rendering, export downloads | ✅ · WebGPU, Safari/Firefox testing 🚧 |
<sub>✅ works today · 🚧 in progress · ⬜ not started</sub>
@@ -373,7 +395,7 @@ cargo run --release -p lightcraft -- --memory # a throwaway in-memory
cargo run --release -p lightcraft -- --control 7980 # with the automation channel
cargo xtask web --serve # the same app in the browser: http://127.0.0.1:8080/
cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light.exposure=0.5
cargo xtask ci # fmt, clippy, tests, layering, wasm checks
cargo xtask ci # fmt, clippy, HEIF, tests, parity, layers, assets, WASM
```
CI defaults to line-table debug information and one build job per 1.5 GB of RAM
@@ -426,7 +448,7 @@ CRAFT_FONTS_DIR=../craft-fonts cargo run --release -p lightcraft
Without it LightCraft builds and runs the same, but Chinese and Japanese text have no glyphs. Fonts are never committed to this
repo — the standard for what goes there lives with it: see the [craft-fonts README](https://github.com/storytold/craft-fonts#readme).
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語, Português (Brasil), Español, Deutsch, Русский, Français, Українська, Čeština) or **Settings → General →
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語, Português (Brasil), Español, Deutsch, Русский, Français, Українська, Polski, Čeština, Italiano, ქართული) or **Settings → General →
Language**; the choice applies immediately and persists. See [docs/localization.md](docs/localization.md).
**Logs:** the desktop app writes its log to standard error and to `logs/lightcraft.log` in its settings folder
@@ -491,9 +513,9 @@ egui frontend is one swappable crate; nothing below it knows a UI exists. `mcp`
Humans and agents follow the same rules, so read [AGENTS.md](AGENTS.md) first. The short version:
- **Clean-room.** Never read Adobe binaries or GPL raw/photo code (darktable, RawTherapee, LibRaw, rawspeed, dcraw…);
- **Clean-room.** Never read proprietary binaries or GPL raw/photo code (darktable, RawTherapee, LibRaw, rawspeed, dcraw…);
work from public specs and black-box observation.
- **No Adobe assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from Adobe products. Every
- **No proprietary assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from commercial products. Every
image, icon and font in the repo is original, public domain, Creative Commons, OFL or permissively licensed, and has
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit. New fonts go to
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), not here.
@@ -536,7 +558,7 @@ option to start it. For unattended installs: `msiexec /i lightcraft-<ver>-window
| Format | x86_64 | aarch64 (ARM64) | Notes |
|---|---|---|---|
| AppImage | `lightcraft-<ver>-linux-x86_64.AppImage` | `lightcraft-<ver>-linux-aarch64.AppImage` | Runs anywhere; updates itself with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) |
| AppImage | `lightcraft-<ver>-linux-x86_64.AppImage` | `lightcraft-<ver>-linux-aarch64.AppImage` | Portable; updates with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) |
| Flatpak | `lightcraft-<ver>-linux-x86_64.flatpak` | `lightcraft-<ver>-linux-aarch64.flatpak` | Sandboxed; `flatpak install --user <file>` |
| Debian/Ubuntu | `lightcraft-<ver>-linux-x86_64.deb` | `lightcraft-<ver>-linux-aarch64.deb` | |
| Fedora/RHEL/openSUSE | `lightcraft-<ver>-linux-x86_64.rpm` | `lightcraft-<ver>-linux-aarch64.rpm` | |
@@ -623,8 +645,6 @@ ArtCraft Team and are not covered by this license. They may be used only unmodif
part of this repository and LightCraft, under [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt).
Forks and modified versions must remove them.
<sub>Adobe, Photoshop, Illustrator, Premiere Pro, Lightroom, Acrobat, After Effects and InDesign are trademarks or registered trademarks of Adobe Inc. in the United States and/or other countries. LightCraft is an independent, open-source project and is not affiliated with, sponsored by or endorsed by Adobe Inc.; these names are used only to describe the workflows it is compatible with.</sub>
<p align="center">
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
<sub>Made by the <a href="https://getartcraft.com/">ArtCraft</a> team and community.</sub>
+6 -8
View File
@@ -18,7 +18,7 @@
</p>
<p align="center">
<b>照片图库与 raw 显影;Adobe Lightroom 的开源净室实现,用纯 Rust 重建。</b><br>
<b>照片图库与 raw 显影;开源、非破坏性的照片图库与 raw 显影工具,用纯 Rust 构建。</b><br>
在 macOS、Windows 和 Linux 上原生运行,也可通过 WebAssembly 在浏览器中使用,并能由 AI 智能体经 MCP 端到端驱动。
</p>
@@ -299,12 +299,12 @@ lightcraft --control 7980 ~/Pictures/trip
LightCraft 很年轻,而且进展很快。**我们如实所处的位置**(细节见[路线图](ROADMAP.md#where-we-stand)):
- **按功能数量算,我们达到了 Lightroom 的约 79%**(核心功能 98%),逐行记录在
- **按功能数量算,我们达到了约 79%**(核心功能 98%),逐行记录在
[docs/parity.md](docs/parity.md)。
- **作为日常替代 Lightroom 的工具,我们更接近 60–70%。** 在单台机器上处理 JPEG/DNG 以及大多数尼康/索尼/
- **作为日常使用的工具,我们更接近 60–70%。** 在单台机器上处理 JPEG/DNG 以及大多数尼康/索尼/
较老佳能的 raw 都很出色。
- **最大的缺口:**
- **相机色彩校准:** 索尼、尼康、松下、富士和佳能 CR3 的 raw 会从各自的相机 JPEG 得到受保护的估计值,内置 ILCA-99M2、ILCE-7CR、ILCE-7M4、X-H2S 和 X-T4 配置文件;实测校准仍然缺失,其他 raw 或被拒绝的拟合会保留中性矩阵;
- **相机色彩校准:** 索尼、尼康、松下、富士和佳能 CR3 的 raw 会从各自的相机 JPEG 得到受保护的估计值,内置 ILCA-99M2、ILCE-7CR、ILCE-7M4、X-H2S、X-T4、尼康 Z 6II、Z 8 和 D5300 配置文件;实测校准仍然缺失,其他 raw 或被拒绝的拟合会保留中性矩阵;
- **压缩的奥林巴斯 raw 和不受支持的 CR3 变体:** 这些在有内嵌 JPEG 预览时使用预览。富士无损/有损压缩 RAF 现在能解码传感器数据;[验证与已有图库重新加载说明](docs/raf-compression.md);
- **AI 蒙版与降噪:** 主体和天空选择目前是经典启发式算法;
- **屏幕上的 HDR 显示、视频,以及经典版式中的打印/画册/地图模块。**
@@ -453,9 +453,9 @@ egui 前端是一个可替换的 crate;它下面没有任何东西知道 UI
人和智能体遵循同样的规则,所以请先读 [AGENTS.md](AGENTS.md)。简短版本:
- **净室。** 绝不阅读 Adobe 的二进制文件或 GPL 的 raw/照片代码(darktable、RawTherapee、LibRaw、rawspeed、dcraw……);
- **净室。** 绝不阅读专有软件的二进制文件或 GPL 的 raw/照片代码(darktable、RawTherapee、LibRaw、rawspeed、dcraw……);
只依据公开规范和黑盒观察来工作。
- **绝不使用 Adobe 资源:** 不使用 Adobe 产品的图标、截图、预设、配置文件、LUT 或字体。仓库中的每一个
- **绝不使用专有资源:** 不使用商业产品的图标、截图、预设、配置文件、LUT 或字体。仓库中的每一个
图像、图标和字体都是原创、公有领域、知识共享、OFL 或宽松许可的,并在同一提交中
于 [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) 添加条目。新字体提交到
[storytold/craft-fonts](https://github.com/storytold/craft-fonts),而不是这里。
@@ -579,8 +579,6 @@ ArtCraft 团队的商标,不在本许可范围内。它们只能在未修改
本仓库和 LightCraft 的一部分,按 [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt) 使用。
分支和修改版本必须移除它们。
<sub>Adobe、Photoshop、Illustrator、Premiere Pro、Lightroom、Acrobat、After Effects 和 InDesign 是 Adobe Inc. 在美国和/或其他国家的商标或注册商标。LightCraft 是一个独立的开源项目,与 Adobe Inc. 无隶属关系,也未获其赞助或认可;这些名称仅用于说明它所兼容的工作流。</sub>
<p align="center">
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
<sub>由 <a href="https://getartcraft.com/">ArtCraft</a> 团队和社区制作。</sub>
+5 -3
View File
@@ -46,7 +46,7 @@ Hours are calibrated on this repo's PR history; see
|---|---:|---:|---:|---|
| Features | ~61% | ~77% (checklist 81.1%) | 600–1,100 h | [target-app-parity](docs/target-app-parity.md#features) |
| ↳ RAW cameras, colour, lens profiles | ~60% | 83% of 1,446 Adobe models decode, 38% verified | 255–480 h | [raw-parity](docs/raw-parity.md) |
| UI/UX fidelity | ~70% | ~85% | 34–58 h | [ui-parity](docs/ui-parity.md) |
| UI/UX fidelity | ~70% | ~85% | 32–54 h | [ui-parity](docs/ui-parity.md) |
| File formats (non-raw) | ~70% | ~85% | 30–55 h | [file-format-parity](docs/file-format-parity.md) |
| Hardware | ~40% | ~50% | 40–80 h | [hardware-parity](docs/hardware-parity.md) |
| Localization | ~55% of the 12 key languages | 7 of 12 present | 130–210 h (+145–255 h new scripts) | [localization-parity](docs/localization-parity.md) |
@@ -96,7 +96,7 @@ while lower-layer errors stay English. Detail: [localization-parity.md](docs/loc
| Korean | ko | 0% | none (no Hangul face) |
| Vietnamese | vi | 0% | none |
Also shipped: Traditional Chinese (87%), Russian (74%), Ukrainian (99%), Italian (92%).
Also shipped: Traditional Chinese (87%), Russian (74%), Ukrainian (99%), Italian (92%), Georgian (100%).
## Upcoming
@@ -105,7 +105,7 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
1. Camera colour of our own: measured calibration for the popular models (50–90 h)
2. CR3 for every Canon body (25–45 h)
3. Render-fidelity suite against Lightroom, then tuning (30–60 h)
4. Full-resolution zoom (8–15 h)
4. ~~Full-resolution zoom~~ (done 2026-10-12)
5. AI masks: Subject / Sky / People (40–70 h + model decision)
6. Compressed ORF and HE NEF (28–55 h)
7. Lens profile database of our own (40–80 h)
@@ -191,6 +191,8 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
| 2026-10-14 | minor | 17 uncompressed Olympus and Samsung models verified on one CC0 file each (254 models verified, 18%); headline numbers unchanged |
| 2026-10-12 | minor | Mamiya MEF decoded and imported: ZD verified (241); headline numbers unchanged |
| 2026-10-12 | minor | Epson ERF decoded: 3 more verified models (240); headline numbers unchanged |
| 2026-10-12 | minor | Georgian (`ka`) interface language added (every catalog message) |
| 2026-10-14 | minor | Full-resolution zoom (gap 4) closed: soft proofing gets zoom windows; headline numbers unchanged |
| 2026-10-12 | minor | UI/UX remaining 40–70 h → 34–58 h (accessibility audit landed); headline numbers unchanged |
| 2026-10-14 | minor | Camera-coverage sweep (#1327): 30 models verified on 40 CC0 files (coverage.py 515 → 545 verified); sensor verification does not establish colour parity (#1277); headline numbers unchanged |
| 2026-10-12 | minor | 22 more CR3 bodies verified, EOS R6 Mark III white balance: RAW decoding 77% → 83%, RAW ~56% → ~60%; mainstream 49% → 51%, essentials 64% → 65%; full unchanged at ~61% |
+4
View File
@@ -15,6 +15,9 @@ covers a whole directory), or if a licence file referenced in the Licence column
| `assets/camera-profiles/ILCA-99M2.json` | Sony ILCA-99M2 camera colour profile (matrix + hue/saturation/value table), built into LightCraft | LightCraft contributors; source photographs supplied by @PiMinusMax | original work: fitted by `lightcraft-cli calibrate` to 144 usable ILCA-99M2 ARWs (of 150, seven shooting days, Standard creative style) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-11 | none |
| `assets/camera-profiles/ILCE-7CR.json` | Sony ILCE-7CR camera colour profile (matrix + hue/saturation/value table), built into LightCraft | LightCraft contributors; source photographs supplied by @AlexChambers | original work: fitted by `lightcraft-cli calibrate` to 125 usable ILCE-7CR ARWs (of 131, September 2025 to September 2026, Standard creative style) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-09 | none |
| `assets/camera-profiles/X-H2S.json`, `assets/camera-profiles/X-T4.json` | Fujifilm camera colour profiles (matrix + hue/saturation/value table), built into LightCraft | LightCraft contributors; source photographs supplied by Sebastian Pfluegelmeier | original work: fitted by `lightcraft-cli calibrate` to 201 X-H2S and 545 X-T4 RAFs and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-08 | none |
| `assets/camera-profiles/NIKON_Z_6_2.json` | Nikon Z 6II camera colour profile (matrix + hue/saturation/value table), built into LightCraft | Michael Anderson (@endthestart) | original work: fitted by `lightcraft-cli calibrate` to 296 usable Z 6II NEFs of my own (of 299, 2024 to 2026, Standard and Auto picture controls; NIKKOR Z 20mm f/1.8 S, Z 24-70mm f/4 S and AF-S 70-200mm f/2.8G on the FTZ) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-13 | none |
| `assets/camera-profiles/NIKON_Z_8.json` | Nikon Z 8 camera colour profile (matrix + hue/saturation/value table), built into LightCraft | Michael Anderson (@endthestart) | original work: fitted by `lightcraft-cli calibrate` to 261 usable Z 8 NEFs of my own (of 278, 2023 and 2026, nearly all Standard picture control, lossless 14-bit; NIKKOR Z 24-120mm f/4 S and AF-S 70-200mm f/2.8G on the FTZ) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-13 | none |
| `assets/camera-profiles/NIKON_D5300.json` | Nikon D5300 camera colour profile (matrix + hue/saturation/value table), built into LightCraft | Michael Anderson (@endthestart) | original work: fitted by `lightcraft-cli calibrate` to 264 usable D5300 NEFs of my own (of 273, 2014 to 2017, mostly Standard picture control, lossy 12-bit, mostly the AF-S DX 35mm f/1.8G) and their in-camera JPEGs; aggregate colour statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 | 2026-10-13 | none |
| `crates/raw/src/spectral_table.rs` | Camera colour matrices (DNG ColorMatrix1/2 and ForwardMatrix1/2 for Standard Illuminant A and D65) for 52 camera models, and the model alias list | Derived by LightCraft contributors from rawtoaces-data (Academy Software Foundation): camera spectral sensitivities measured by Weta Digital (44 cameras), scitech (7) and Jonathon Irons at the National Physical Laboratory (Nikon D5100); training reflectances; CIE 1931 colour-matching functions (CIE, CC-BY-4.0, as shipped in rawtoaces-data) | https://github.com/AcademySoftwareFoundation/rawtoaces-data, tag v1.1.0, commit 5cfa53707b053594b2d0d48e8019fb56e6582d6c | Apache-2.0 (`LICENSE-APACHE`); the colour-matching functions CC-BY-4.0 | 2026-10-08 | generated by `tools/spectral_camera_table.py`: a 3×3 matrix per illuminant fitted to the spectral data (no measurement copied verbatim); aliases copied from `data/aliases.json`; no Adobe or third-party camera matrix |
| `assets/camera-tone/sony-dro-off.json` | Sony global tone curve with the Dynamic Range Optimizer off (32 knots), built into LightCraft for ARWs shot with DRO on | LightCraft contributors; source photographs by raw.pixls.us contributors | original work: tone curve quantile-matched over the pooled pixel pairs of 12 CC0 raw.pixls.us ARWs (DRO off, Standard creative style; ILCE-3500, -5000, -6600, -7CR, -7M3 ×3, -7S, -9M2, ILCA-77M2, SLT-A57, NEX-F3) and their embedded camera JPEGs; the file lists each source URL and SHA-256 and how the curve was derived (see also `docs/camera-preview-colour.md`); aggregate tone statistics only, no image content; no Adobe or third-party profile data | MIT OR Apache-2.0 (sources CC0) | 2026-10-09 | none |
| `crates/scenes/` (generated images) | Procedural demo photographs | LightCraft contributors | original work (generated at runtime, no source imagery) | MIT OR Apache-2.0 | 2026-09-30 | n/a |
@@ -30,6 +33,7 @@ covers a whole directory), or if a licence file referenced in the Licence column
| `crates/ui-egui/locales/uk.json`, `crates/ui-egui/locales/uk-formats.json` | Ukrainian interface catalogs | @dmatviichuk with Codex; PhotoCraft, FilmCraft and VectorCraft translation contributors for reused labels | original translations from English source; 337 initial English-key matches reviewed from [PhotoCraft 69c41f3](https://github.com/storytold/photocraft/blob/69c41f3523e147eb215da8c7b4c29b18b35d1c63/crates/ui-egui/src/i18n/uk.tsv), [FilmCraft 4854c27](https://github.com/storytold/filmcraft/blob/4854c277536fe8118257362304b6a7d82d8e704b/crates/ui-egui/src/i18n/uk.tsv), [VectorCraft 4b44265](https://github.com/storytold/vectorcraft/blob/4b44265965fdd6ffedfd6182a892c1a6c360d344/crates/ui-egui/src/i18n/uk.tsv) | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`, also the source repositories' licences) | 2026-10-09 | adapted to LightCraft photo-development context, completed catalogs, reviewed spelling and placeholders |
| `crates/ui-egui/locales/cs.json`, `crates/ui-egui/locales/cs-formats.json` | Czech interface catalogs | @Losminek with AI assistance (Codex) | original work: translations of English source phrases, reviewed against the app UI conventions; no external translation material | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) | 2026-10-11 | catalog coverage, Czech dates and count-neutral wording; independent scope/terminology review; native-speaker proofreading pending |
| `crates/ui-egui/locales/it.json`, `crates/ui-egui/locales/it-formats.json` | Italian interface catalogs | @GabrielePecchioli with Claude Code; PhotoCraft, VectorCraft, PdfCraft and WordCraft translation contributors for reused labels | original translations from English source; 410 initial English-key matches reviewed from [PhotoCraft da3da09](https://github.com/storytold/photocraft/blob/da3da0942c14c1159f40c652afcf33584332c733/crates/ui-egui/src/i18n/it.tsv), [VectorCraft 5432821](https://github.com/storytold/vectorcraft/blob/543282148007bcb5633c493a5374cd26b0339c8d/crates/ui-egui/src/i18n/it.tsv), [PdfCraft e8e275c](https://github.com/storytold/pdfcraft/blob/e8e275cbd40ccd0dc6b56f82b16a575571043702/crates/ui-egui/src/i18n/it.tsv) and the Italian WordCraft catalog ([storytold/wordcraft#598](https://github.com/storytold/wordcraft/pull/598)); 46 of them retranslated where the sibling's meaning didn't fit a photo app | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`, also the source repositories' licences) | 2026-10-12 | adapted to LightCraft photo-development context, count-neutral labels for Italian plural agreement, reviewed placeholders |
| `crates/ui-egui/locales/ka.json`, `crates/ui-egui/locales/ka-formats.json` | Georgian interface catalogs | Davit Matchakhelidze with Claude; WordCraft, GridCraft, PhotoCraft, FilmCraft and VectorCraft Georgian catalogs for reused labels | original translations from the English source (meaning-based, no vendor localization used); 450 English-key matches reused from the sibling Craft apps' Georgian catalogs | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) | 2026-10-12 | adapted to LightCraft photo-library and raw-development context; count-neutral numerals; placeholders checked |
Notes
- Fonts authored or published by Adobe (Source Sans/Serif/Code, Source Han, …) are not used, even though some are
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+12
View File
@@ -45,3 +45,15 @@ sync_github_name = true
[people.Silvertree2010]
display_name = "Netter"
sync_github_name = false
[people.nledford]
real_name = "Nathaniel Ledford"
[people.marcoarnulfo]
real_name = "Marco Arnulfo"
display_name = "Marco Arnulfo"
sync_github_name = false
[people.egegl]
real_name = "Ege Gursel"
sync_github_name = true
+12 -1
View File
@@ -49,7 +49,9 @@ pub fn shift_iso(iso: &str, secs: i64) -> Option<String> {
if !(first..=last).contains(&t) {
return None;
}
let tail = iso.trim().get(19..).unwrap_or("");
let iso = iso.trim();
// the seconds are optional (`10:00+02:00`): without them the tail starts right after the minutes
let tail = iso.get(if iso.get(16..17) == Some(":") { 19 } else { 16 }..).unwrap_or("");
Some(format!("{}{tail}", civil(t)))
}
@@ -222,6 +224,15 @@ mod tests {
assert_eq!(normalize_iso("2026-02-30"), None);
}
#[test]
fn shift_keeps_the_zone_of_a_time_without_seconds() {
assert_eq!(shift_iso("2026-01-01T10:00+02:00", 3600).as_deref(), Some("2026-01-01T11:00:00+02:00"));
assert_eq!(shift_iso("2026-01-01T10:00Z", 60).as_deref(), Some("2026-01-01T10:01:00Z"));
assert_eq!(shift_iso("2026-01-01T10:00", 3600).as_deref(), Some("2026-01-01T11:00:00"));
assert_eq!(shift_iso("2026-01-01T10:00:00.5-05:00", 3600).as_deref(), Some("2026-01-01T11:00:00.5-05:00"));
assert_eq!(iso_seconds("2026-01-01T10:00+02:00"), iso_seconds("2026-01-01T10:00"));
}
#[test]
fn runs_by_day_month_year_keep_stacks_together() {
let mut c = Catalog::new();
+11 -4
View File
@@ -271,7 +271,7 @@ fn resync(line: &[u8], last: u64) -> Option<(usize, (u64, Op))> {
const START: &[u8] = b"{\"seq\":";
(1..line.len().saturating_sub(START.len() - 1)).filter(|&i| line[i..].starts_with(START)).find_map(|i| {
let rec = std::str::from_utf8(&line[i..]).ok().map(|l| l.trim_end_matches('\r')).and_then(decode_record)?;
(rec.0 <= last + 1).then_some((i, rec))
(rec.0 <= last.saturating_add(1)).then_some((i, rec))
})
}
@@ -298,6 +298,9 @@ fn write_snapshot(store: &mut dyn Store, seq: u64, catalog: &Catalog) -> std::io
Ok(SnapshotTiming { serialize_ms, write_sync_ms: (ms_since(t0) - serialize_ms).max(0.0), bytes, ..Default::default() })
}
/// The highest `seq` a snapshot may hold; beyond it the counter is treated as corrupt, far from overflow.
const MAX_SEQ: u64 = u64::MAX / 2;
/// Just the identification of `catalog.snap` (read before the catalog itself).
#[derive(serde::Deserialize)]
struct SnapHeader {
@@ -331,6 +334,10 @@ impl Journal {
return Err(newer(format!("{SNAPSHOT} is catalog format v{}", h.version)));
}
let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
// a seq this high can't come from real use; the next record's seq would wrap
if s.seq > MAX_SEQ {
return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: seq {} is out of range", s.seq)));
}
let mut catalog = s.catalog;
report.repaired = catalog.repair();
for r in &report.repaired {
@@ -392,7 +399,7 @@ impl Journal {
let _ = op;
report.stale += 1;
}
Some((seq, op)) if seq == j.seq + 1 => {
Some((seq, op)) if j.seq.checked_add(1) == Some(seq) => {
if catalog.apply(op).is_err() {
report.failed += 1;
} else {
@@ -476,7 +483,7 @@ impl Journal {
let mut buf = String::new();
let mut seq = self.seq;
for op in ops {
seq += 1;
seq = seq.checked_add(1).ok_or_else(|| CatalogError::Corrupt(format!("{LOG}: op counter overflow")))?;
buf.push_str(&encode_record(seq, op));
buf.push('\n');
}
@@ -527,7 +534,7 @@ impl Journal {
log::warn!("catalog: background snapshot failed ({e}); writing one now");
}
let t0 = web_time::Instant::now();
let seq = self.seq + unlogged;
let seq = self.seq.checked_add(unlogged).ok_or_else(|| CatalogError::Corrupt(format!("{SNAPSHOT}: op counter overflow")))?;
let mut timing = match write_snapshot(self.store.as_mut(), seq, catalog) {
Ok(t) => t,
Err(e) => {
+6
View File
@@ -199,6 +199,11 @@ pub struct Meta {
/// Left out of the catalog JSON when empty (most photos), so older catalogs read unchanged.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub regions: Vec<lightcraft_meta::Region>,
/// Where the camera focused (read from the file's maker notes at import or Reload from Disk; Sony
/// bodies whose files have been checked), on the upright (EXIF-oriented) photo. Read-only: never
/// written to XMP. Left out of the catalog JSON when unknown, so older catalogs read unchanged.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub focus: Option<lightcraft_meta::FocusPoint>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -638,6 +643,7 @@ mod edited_tests {
("NIKON CORPORATION NIKON D850", "NEF", false),
("Canon Canon EOS 5D Mark III", "CR2", false),
("Canon Canon EOS R6m2", "CR3", false),
("Canon Canon EOS R6 Mark II", "CR3", false),
("Canon Canon EOS R7", "CR3", true),
("FUJIFILM X-T4", "RAF", false),
("ILCE-7M3", "ARW", false),
+18 -4
View File
@@ -157,13 +157,20 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
Some((_, v)) => (x - v).abs() < 1e-9,
None => false,
};
// Fields stored as `f32` compare at f32 precision: 2.8f32 widened to f64 is not 2.8.
let cmp32 = |x: f32, s: &str| match num(s) {
Some(('>', v)) => x > v as f32,
Some(('<', v)) => x < v as f32,
Some((_, v)) => x == v as f32,
None => false,
};
return match field {
"rating" | "stars" => cmp(p.rating as f64, val),
"flag" => Flag::parse(val) == Some(p.flag),
"label" | "color" => p.label.is_some_and(|l| format!("{l:?}").eq_ignore_ascii_case(val)),
"iso" => p.meta.iso.is_some_and(|i| cmp(i as f64, val)),
"f" | "aperture" => p.meta.aperture.is_some_and(|a| cmp(a as f64, val)),
"focal" => p.meta.focal_mm.is_some_and(|a| cmp(a as f64, val)),
"f" | "aperture" => p.meta.aperture.is_some_and(|a| cmp32(a, val)),
"focal" => p.meta.focal_mm.is_some_and(|a| cmp32(a, val)),
"camera" => p.meta.camera.to_lowercase().contains(val),
"lens" => p.meta.lens.to_lowercase().contains(val),
"keyword" | "kw" => p.meta.keywords.iter().any(|k| crate::keywords::is_under(k, val)),
@@ -173,11 +180,18 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
"date" => p.captured.as_deref().is_some_and(|c| c.starts_with(val)),
"copy" | "virtual" => (val == "true" || val == "yes") == p.copy_of.is_some(),
"name" | "file" => p.file_name.to_lowercase().contains(val),
_ => false,
// Not a known field: the colon is part of a plain word ("10:30", "Re:").
_ => text_matches(p, &t),
};
}
text_matches(p, &t)
}
/// Whether the lower-cased word `t` is in the file name, title, caption, camera, lens, location,
/// format or a keyword of `p`.
fn text_matches(p: &Photo, t: &str) -> bool {
let hay = [&p.file_name, &p.meta.title, &p.meta.caption, &p.meta.camera, &p.meta.lens, &p.meta.location, &p.format];
hay.iter().any(|h| h.to_lowercase().contains(&t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(&t))
hay.iter().any(|h| h.to_lowercase().contains(t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(t))
}
/// Whether `p` has a named face region called `name` (case-insensitive).
+2 -1
View File
@@ -294,7 +294,8 @@ pub fn iso_seconds(s: &str) -> Option<i64> {
if !(1..=12).contains(&m) || !(1..=31).contains(&d) {
return None;
}
let (hh, mm, ss) = if s.len() >= 16 { (num(11, 13)?, num(14, 16)?, num(17, 19).unwrap_or(0)) } else { (0, 0, 0) };
let (hh, mm, ss) =
if s.len() >= 16 { (num(11, 13)?, num(14, 16)?, if s.get(16..17) == Some(":") { num(17, 19).unwrap_or(0) } else { 0 }) } else { (0, 0, 0) };
// days from civil (proleptic Gregorian)
let y2 = if m <= 2 { y - 1 } else { y };
let era = y2.div_euclid(400);
+64
View File
@@ -123,6 +123,48 @@ fn filter_search_sort() {
assert_eq!(c.keywords(), vec![("ocean".to_string(), 1), ("summer".to_string(), 1)]);
}
/// A search word with a colon whose left part is not a known field ("10:30", "Re:") is plain
/// text, not a fielded token that matches nothing.
#[test]
fn free_text_with_a_colon_searches_the_text_fields() {
let mut c = Catalog::new();
let a = photo(&mut c, "meeting.jpg", "2026-05-01T10:00:00");
let _b = photo(&mut c, "other.jpg", "2026-04-01T10:00:00");
let mut m = c.photo(a).unwrap().meta.clone();
m.caption = "Meeting at 10:30 with Re: notes".into();
m.iso = Some(1600);
c.apply(Op::SetMeta { id: a, meta: Box::new(m) }).unwrap();
let q = |text: &str| c.query(&Filter { text: text.into(), ..Default::default() }, &Sort::default());
assert_eq!(q("meeting"), vec![a]);
assert_eq!(q("10:30"), vec![a], "a free-text token with a colon");
assert_eq!(q("re:"), vec![a]);
assert_eq!(q("RE: notes"), vec![a], "case-insensitive, one token per word");
assert_eq!(q("iso:>800"), vec![a], "known fields stay fielded");
assert_eq!(q("iso:>2000"), Vec::<PhotoId>::new());
assert_eq!(q("11:45"), Vec::<PhotoId>::new(), "an unknown field matching no text finds nothing");
}
/// `f:`/`aperture:` and `focal:` are stored as `f32`; a typed value that isn't exact in binary
/// (2.8, 5.6) must still match, and `<`/`>` must not count the photo's own value.
#[test]
fn search_tokens_match_f32_aperture_and_focal() {
let mut c = Catalog::new();
let a = photo(&mut c, "a.jpg", "2026-05-01T10:00:00");
let mut m = c.photo(a).unwrap().meta.clone();
m.aperture = Some(2.8);
m.focal_mm = Some(5.6);
c.apply(Op::SetMeta { id: a, meta: Box::new(m) }).unwrap();
let q = |t: &str| c.query(&Filter { text: t.into(), ..Default::default() }, &Sort::default());
assert_eq!(q("f:2.8"), vec![a], "f:2.8");
assert_eq!(q("aperture:=2.8"), vec![a], "aperture:=2.8");
assert_eq!(q("focal:5.6"), vec![a], "focal:5.6");
assert!(q("f:2.9").is_empty(), "f:2.9");
assert!(q("f:<2.8").is_empty(), "f:<2.8");
assert!(q("f:>2.8").is_empty(), "f:>2.8");
assert!(q("focal:<5.6").is_empty(), "focal:<5.6");
assert_eq!(q("f:>2.7"), vec![a], "f:>2.7");
}
/// A `captured` string that is not ASCII (the Lightroom importer copies `captureTime` into an
/// `AddPhoto` op verbatim, and a JSON snapshot is valid UTF-8 whatever it holds) must not be
/// sliced on a byte index in `date_groups`: grouping counts the well-formed dates and ignores
@@ -425,6 +467,28 @@ fn empty_regions_are_not_serialized_and_default_when_missing() {
assert_eq!(back, with);
}
/// The camera's focus point is optional camera data: absent from the catalog JSON when unknown (so a
/// catalog from before it existed reads with none, and writes the same), kept exactly when known.
#[test]
fn focus_point_is_not_serialized_when_unknown_and_round_trips() {
let m = Meta { camera: "SONY ILCE-1".into(), ..Default::default() };
let v = serde_json::to_value(&m).unwrap();
assert!(v.get("focus").is_none(), "{v}");
let back: Meta = serde_json::from_value(v).unwrap();
assert!(back.focus.is_none());
let mut with = m.clone();
with.focus = Some(lightcraft_meta::FocusPoint {
rect: lightcraft_meta::Rect { x0: 0.36, y0: 0.33, x1: 0.38, y1: 0.36 },
mode: Some("AF-C".into()),
area: Some("Wide".into()),
tracking: None,
});
let json = serde_json::to_string(&with).unwrap();
assert!(json.contains("\"focus\"") && !json.contains("tracking"), "{json}");
let back: Meta = serde_json::from_str(&json).unwrap();
assert_eq!(back, with);
}
// ---- random sort: Given a seed, the order is a stable, reproducible shuffle of the matching photos
fn many(c: &mut Catalog, n: usize) -> Vec<PhotoId> {
+15
View File
@@ -376,3 +376,18 @@ fn a_damaged_process_value_loads_as_v1_with_the_edits() {
let d = &loaded.photo(id).unwrap().develop;
assert_eq!((d.process, d.light.exposure), (ProcessVersion::V1, 0.8));
}
/// A snapshot whose `seq` is near `u64::MAX` is refused as corrupt: the next record's seq would
/// wrap (and every later edit would be skipped as stale on reopen) or overflow-panic in debug.
#[test]
fn snapshot_with_seq_near_u64_max_is_corrupt() {
for seq in [u64::MAX, u64::MAX - 1, u64::MAX / 2 + 1] {
let m = MemStore::new();
m.set(
SNAPSHOT,
format!("{{\"format\":\"lightcraft-catalog\",\"version\":1,\"seq\":{seq},\"catalog\":{}}}\n", Catalog::new().to_snapshot()).into_bytes(),
);
let e = Journal::open(Box::new(m)).err();
assert!(matches!(e, Some(CatalogError::Corrupt(_))), "seq {seq}: {e:?}");
}
}
+3
View File
@@ -55,3 +55,6 @@ jxl-oxide = { workspace = true }
proptest = { workspace = true }
# Lossless synthetic HEIC files for the `heif` feature's tests.
lightcraft-heif = { workspace = true, features = ["testdata"] }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
dhat = { workspace = true }
+4
View File
@@ -73,6 +73,10 @@ ICC profiles are parsed with `moxcms` (v2/v4, `curv`/`para` TRCs, `chad`); sRGB,
Adobe RGB (1998), ProPhoto (ROMM) and Rec.2020 are recognised by colorants. `icc::write_matrix_trc`
emits v4 display profiles (D50 PCS, Bradford `chad`) for export embedding.
Before colour conversion, positive-infinity float samples become 1.0; NaN and negative infinity
become 0.0. This also applies to alpha. Finite samples keep their existing range: matrix/TRC paths
preserve values above 1.0 and below 0.0, while CMS inputs and alpha remain bounded to 0..1.
## Dependencies and licences
All pure Rust, verified with `cargo info` / `cargo tree`:
+14 -5
View File
@@ -44,11 +44,23 @@ impl Buf {
match self {
Buf::U8(v) => v[i] as f32 / 255.0,
Buf::U16(v) => v[i] as f32 / 65535.0,
Buf::F32(v) => v[i],
Buf::F32(v) => normalize_float(v[i]),
}
}
}
/// Clip non-finite device samples before colour conversion, preserving finite extended range.
#[inline]
fn normalize_float(value: f32) -> f32 {
if value == f32::INFINITY {
1.0
} else if value.is_finite() {
value
} else {
0.0
}
}
/// Interleaved device samples as produced by a format decoder.
pub(crate) struct Raw {
pub width: usize,
@@ -218,10 +230,7 @@ fn linearize(raw: &Raw, ch: usize, trc: &[Trc; 3]) -> Rgb32f {
par_rows(&mut img.data, w, |y, row| {
for (x, o) in row.iter_mut().enumerate() {
let i = (y * w + x) * ch;
let s = |c: usize| {
let val = v[i + c];
if val.is_finite() { val } else { 0.0 }
};
let s = |c: usize| normalize_float(v[i + c]);
*o = if gray {
let l = if lin { s(0) } else { trc[1].to_linear(s(0)) };
[l, l, l]
+53 -3
View File
@@ -8,7 +8,7 @@ use crate::convert::{Buf, Meta, Model, Raw, check_size, finish};
use crate::space::{NamedSpace, SourceSpace, SpaceOrigin, Trc};
use crate::{DecodeOptions, Decoded, Error, Format, Result};
use jxl_oxide::color::{ColourSpace, EnumColourEncoding, Primaries, RenderingIntent, TransferFunction, WhitePoint};
use jxl_oxide::{AuxBoxData, JxlImage};
use jxl_oxide::{AllocTracker, AuxBoxData, InitializeResult, JxlImage};
const F: Format = Format::Jxl;
@@ -16,10 +16,60 @@ fn err(e: impl std::fmt::Display) -> Error {
Error::Malformed(F, e.to_string())
}
/// Tracked allocation budget per allowed pixel for frames, extra channels and modular trees.
/// This bounds allocations covered by `AllocTracker`, not every allocation made by the decoder (#1517).
const TRACKED_BYTES_PER_PIXEL: u64 = 4 * 4 * 8;
/// The allocation budget never drops below this, so small images keep the decoder's fixed
/// overhead (group buffers, entropy tables) whatever `max_pixels` is.
const MIN_TRACKED_BYTES: u64 = 64 << 20;
/// Bytes handed to the decoder at a time (the size `JxlImageBuilder::read` reads in).
const INPUT_CHUNK: usize = 4096;
/// Initializes from bounded input, checks dimensions before rendering, then loads remaining frames
/// within a tracked allocation budget.
fn open(bytes: &[u8], opts: &DecodeOptions) -> Result<JxlImage> {
let budget = opts.max_pixels.saturating_mul(TRACKED_BYTES_PER_PIXEL).max(MIN_TRACKED_BYTES);
let tracker = AllocTracker::with_limit(usize::try_from(budget).unwrap_or(usize::MAX));
let mut uninit = JxlImage::builder().alloc_tracker(tracker).build_uninit();
// Initialization can load already-buffered frame data, so small feeds limit work before
// dimension checking.
let mut rest = bytes;
let mut window = INPUT_CHUNK;
let mut img = loop {
let consumed = uninit.feed_bytes(rest.get(..window).unwrap_or(rest)).map_err(err)?;
rest = rest.get(consumed..).unwrap_or_default();
match uninit.try_init().map_err(err)? {
InitializeResult::Initialized(img) => break img,
InitializeResult::NeedMoreData(more) => uninit = more,
}
if consumed == 0 {
// the parser wants more than the window holds at once
if window >= rest.len() {
return Err(err("truncated JPEG XL header"));
}
window = window.saturating_mul(2);
}
};
check_size(F, img.width() as u64, img.height() as u64, opts)?;
// Match `JxlImageBuilder::read`: bounded feeds until the final frame, not merely the first
// keyframe. `feed_bytes` copies codestream data into an untracked Vec, including any tail
// after the last frame; feeding the remaining file at once bypasses the allocation budget.
while !img.is_loading_done() && !rest.is_empty() {
let consumed = img.feed_bytes(rest.get(..INPUT_CHUNK).unwrap_or(rest)).map_err(err)?;
if consumed == 0 {
break;
}
rest = rest.get(consumed..).unwrap_or_default();
}
img.finalize().map_err(err)?;
Ok(img)
}
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
let mut img = JxlImage::builder().read(std::io::Cursor::new(bytes)).map_err(err)?;
let mut img = open(bytes, opts)?;
let (w, h) = (img.width(), img.height());
check_size(F, w as u64, h as u64, opts)?;
let header = img.image_header();
let bits = header.metadata.bit_depth.bits_per_sample() as u8;
let float = matches!(header.metadata.bit_depth, jxl_oxide::image::BitDepth::FloatSample { .. });
+5
View File
@@ -95,6 +95,11 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
fn limits(opts: &DecodeOptions) -> Limits {
let mut limits = Limits::default();
limits.decoding_buffer_size = opts.max_pixels.saturating_mul(4 * 8).min(usize::MAX as u64) as usize;
// A strip or tile whose declared byte count exceeds this is refused: the default 128 MiB refuses a
// single-strip uncompressed 16-bit export of ~22 MP (#1983). Raised to the decoded image's cap
// (32 bytes per allowed pixel), never lowered: a strip or tile can outgrow the pixels it shows
// (JPEG headers, padding to the tile size).
limits.intermediate_buffer_size = limits.intermediate_buffer_size.max(limits.decoding_buffer_size);
limits.ifd_value_size = 32 << 20;
limits
}
+128
View File
@@ -0,0 +1,128 @@
//! Non-finite samples in procedural 32-bit float TIFFs (issue #1952, item 6).
use lightcraft_codecs::{icc, *};
fn tiff(samples: &[f32], channels: u8, profile: Option<&[u8]>) -> Vec<u8> {
encode_tiff(
&EncodeImage::new((samples.len() / channels as usize) as u32, 1, channels, Samples::F32(samples)),
TiffCompression::None,
&EncodeMeta { icc: profile, ..Default::default() },
)
.unwrap()
}
#[test]
fn float_tiff_nonfinite_rgb_clips_without_losing_finite_hdr() {
let samples =
[[f32::INFINITY; 3], [f32::NAN; 3], [f32::NEG_INFINITY; 3], [-0.25; 3], [0.25, 0.5, 0.75], [2.0, 4.0, 8.0], [f32::INFINITY, 0.5, f32::NAN]];
let bytes = tiff(samples.as_flattened(), 3, None);
let decoded = decode(&bytes, DecodeOptions::default()).unwrap();
assert!(decoded.float);
assert_eq!(decoded.bit_depth, 32);
assert_eq!(decoded.image.data, [[1.0; 3], [0.0; 3], [0.0; 3], [-0.25; 3], [0.25, 0.5, 0.75], [2.0, 4.0, 8.0], [1.0, 0.5, 0.0]]);
let display = decoded.to_srgb8();
assert_eq!(display.data[0], [255; 4]);
for pixel in &display.data[1..4] {
assert_eq!(*pixel, [0, 0, 0, 255]);
}
assert_eq!(display.data[6], [255, 188, 0, 255]);
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
// A box-filtered pair must average clipped white with its ordinary neighbor.
let pair = tiff(&[f32::INFINITY, f32::INFINITY, f32::INFINITY, 0.25, 0.25, 0.25], 3, None);
let small = decode(&pair, DecodeOptions::fit(1, 1)).unwrap();
assert_eq!(small.image.data, [[0.625; 3]]);
}
#[test]
fn float_tiff_nonfinite_gray_and_icc_curves_match_clipped_samples() {
let samples = [f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.25, 1.0, 2.0, -0.25];
let clipped = [1.0, 0.0, 0.0, 0.25, 1.0, 2.0, -0.25];
let profiles = [
None,
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Linear)),
Some(icc::write_named(NamedSpace::Srgb)),
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Gamma(2.0))),
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Table(vec![0, 16384, 65535]))),
Some(moxcms::ColorProfile::new_gray_with_gamma(2.0).encode().unwrap()),
Some(b"invalid ICC profile".to_vec()),
];
for channels in [1, 3] {
let samples: Vec<_> = samples.iter().flat_map(|v| std::iter::repeat_n(*v, channels as usize)).collect();
let clipped: Vec<_> = clipped.iter().flat_map(|v| std::iter::repeat_n(*v, channels as usize)).collect();
for profile in &profiles {
let profile = profile.as_deref();
let decoded = decode(&tiff(&samples, channels, profile), DecodeOptions::default()).unwrap();
let reference = decode(&tiff(&clipped, channels, profile), DecodeOptions::default()).unwrap();
assert_eq!(decoded.grayscale, channels == 1);
assert_eq!(decoded.image, reference.image, "channels={channels}, space={:?}", decoded.space);
assert_eq!(decoded.to_srgb8(), reference.to_srgb8());
assert!(decoded.image.data[5].iter().all(|v| *v > 1.0), "finite HDR survives the curve");
assert!(decoded.image.data[6].iter().all(|v| *v < 0.0), "finite negative samples survive the curve");
}
}
}
#[test]
fn float_tiff_nonfinite_alpha_is_finite_and_bounded() {
let samples = [
[f32::INFINITY, f32::INFINITY, f32::INFINITY, f32::INFINITY],
[f32::NAN, f32::NEG_INFINITY, 0.5, f32::NAN],
[0.25, 0.5, 2.0, f32::NEG_INFINITY],
[2.0, 3.0, 4.0, 0.5],
];
let decoded = decode(&tiff(samples.as_flattened(), 4, None), DecodeOptions::default()).unwrap();
assert_eq!(decoded.alpha.as_ref().unwrap().data, [1.0, 0.0, 0.0, 0.5]);
assert_eq!(decoded.image.data, [[1.0; 3], [0.0, 0.0, 0.5], [0.25, 0.5, 2.0], [2.0, 3.0, 4.0]]);
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
// Associated alpha takes the same path before unpremultiplication.
let mut cursor = std::io::Cursor::new(Vec::new());
{
use tiff::encoder::{TiffEncoder, colortype};
let mut encoder = TiffEncoder::new(&mut cursor).unwrap();
let mut image = encoder.new_image::<colortype::RGBA32Float>(2, 1).unwrap();
image.encoder().write_tag(tiff::tags::Tag::ExtraSamples, &[1u16][..]).unwrap();
image.write_data(&[f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.5, 1.0, 2.0, 3.0, 0.5]).unwrap();
}
let decoded = decode(&cursor.into_inner(), DecodeOptions::default()).unwrap();
assert_eq!(decoded.alpha.as_ref().unwrap().data, [0.5, 0.5]);
assert_eq!(decoded.image.data, [[2.0, 0.0, 0.0], [2.0, 4.0, 6.0]]);
}
fn lut_profile() -> Vec<u8> {
use moxcms::{ColorProfile, LutMultidimensionalType, LutWarehouse, Matrix3d, ToneReprCurve, Vector3d, Xyzd};
let mut profile = ColorProfile::new_srgb();
profile.cicp = None;
profile.red_colorant = Xyzd::default();
profile.green_colorant = Xyzd::default();
profile.blue_colorant = Xyzd::default();
profile.red_trc = None;
profile.green_trc = None;
profile.blue_trc = None;
// A procedural mAB profile without a CLUT: linear RGB to D50 XYZ, then identity curves.
profile.lut_a_to_b_colorimetric = Some(LutWarehouse::Multidimensional(LutMultidimensionalType {
num_input_channels: 3,
num_output_channels: 3,
grid_points: [0; 16],
clut: None,
a_curves: vec![],
m_curves: vec![ToneReprCurve::Parametric(vec![1.0]); 3],
matrix: Matrix3d { v: NamedSpace::Srgb.to_xyz_d50().0.map(|r| r.map(|v| v * 32768.0 / 65535.0)) },
b_curves: vec![ToneReprCurve::Parametric(vec![1.0]); 3],
bias: Vector3d::default(),
}));
profile.encode().unwrap()
}
#[test]
fn float_tiff_nonfinite_cms_samples_match_clipped_samples() {
let profile = lut_profile();
let samples = [f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.25, 0.5, 0.75];
let clipped = [1.0, 0.0, 0.0, 0.25, 0.5, 0.75];
let decoded = decode(&tiff(&samples, 3, Some(&profile)), DecodeOptions::default()).unwrap();
let reference = decode(&tiff(&clipped, 3, Some(&profile)), DecodeOptions::default()).unwrap();
assert_eq!(decoded.space.origin, SpaceOrigin::IccCms);
assert_eq!(decoded.image, reference.image);
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
}
+98
View File
@@ -0,0 +1,98 @@
#![cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
use lightcraft_codecs::exif::minimal_exif;
use lightcraft_codecs::*;
use lightcraft_raster::Rgba8;
fn codestream(bytes: &[u8]) -> Vec<u8> {
if bytes.starts_with(b"\xff\x0a") {
return bytes.to_vec();
}
let mut raw = Vec::new();
let mut at = 12;
while at + 8 <= bytes.len() {
let len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
let end = if len == 0 { bytes.len() } else { at + len };
match &bytes[at + 4..at + 8] {
b"jxlc" => raw.extend_from_slice(&bytes[at + 8..end]),
b"jxlp" => raw.extend_from_slice(&bytes[at + 12..end]),
_ => {}
}
at = end;
}
assert!(raw.starts_with(b"\xff\x0a"));
raw
}
fn box_bytes(out: &mut Vec<u8>, kind: &[u8; 4], data: &[u8]) {
out.extend_from_slice(&u32::try_from(data.len() + 8).unwrap().to_be_bytes());
out.extend_from_slice(kind);
out.extend_from_slice(data);
}
#[test]
fn metadata_in_final_feed_matches_builder() {
let pixels = Rgba8::from_fn(8, 8, |x, y| [(x * 20) as u8, (y * 20) as u8, 80, 255]);
let encoded = encode_jxl(&EncodeImage::rgba8(&pixels), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
let raw = codestream(&encoded);
let exif = minimal_exif(1);
let mut exif_box = vec![0; 4]; // TIFF header offset, relative to the end of this field
exif_box.extend_from_slice(&exif);
// Split the codestream across two feeds, with Exif and XMP following its last frame.
let mut bytes = b"\0\0\0\x0cJXL \x0d\x0a\x87\x0a".to_vec();
box_bytes(&mut bytes, b"ftyp", b"jxl \0\0\0\0jxl ");
let split = raw.len() / 2;
let mut part = 0u32.to_be_bytes().to_vec();
part.extend_from_slice(&raw[..split]);
box_bytes(&mut bytes, b"jxlp", &part);
let padding = vec![0; 4096 - bytes.len() - 8];
box_bytes(&mut bytes, b"free", &padding);
let mut part = 0x8000_0001u32.to_be_bytes().to_vec();
part.extend_from_slice(&raw[split..]);
box_bytes(&mut bytes, b"jxlp", &part);
box_bytes(&mut bytes, b"Exif", &exif_box);
box_bytes(&mut bytes, b"xml ", b"<xmp/>");
assert!(bytes.len() < 8192, "metadata must fit in the completing input chunk");
let old = jxl_oxide::JxlImage::builder().read(bytes.as_slice()).unwrap();
let old_exif = match old.aux_boxes().first_exif().unwrap() {
jxl_oxide::AuxBoxData::Data(ex) => ex.payload().to_vec(),
_ => panic!("builder must preserve Exif in its final feed"),
};
let old_xml = match old.aux_boxes().first_xml() {
jxl_oxide::AuxBoxData::Data(xml) => xml.to_vec(),
_ => panic!("builder must preserve XMP in its final feed"),
};
let decoded = decode(&bytes, DecodeOptions::default()).unwrap();
assert_eq!(decoded.exif.as_deref(), Some(old_exif.as_slice()));
assert_eq!(decoded.xmp.as_deref(), Some(std::str::from_utf8(&old_xml).unwrap()));
assert_eq!(decoded.to_srgb8(), pixels);
}
#[test]
fn progressive_truncation_matches_builder() {
let pixels = Rgba8::from_fn(96, 64, |x, y| {
let mut seed = (y * 96 + x + 1) as u32;
let mut rgb = [0; 4];
for channel in &mut rgb[..3] {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
*channel = seed as u8;
}
rgb[3] = 255;
rgb
});
for lossless in [false, true] {
let encoded =
encode_jxl(&EncodeImage::rgba8(&pixels), &JxlOptions { lossless, progressive: true, ..Default::default() }, &EncodeMeta::default())
.unwrap();
let raw = codestream(&encoded);
assert!(raw.len() > 4096, "fixture must exercise post-header feeds");
for cut in [0, 1, 2, 12, 4095, 4096, 4097, raw.len() / 2, raw.len() - 1, raw.len()] {
let bytes = &raw[..cut];
// `finalize` permits partial images: rendering determines whether frame 0 is usable.
let old_ok = jxl_oxide::JxlImage::builder().read(bytes).is_ok_and(|image| image.render_frame(0).is_ok());
assert_eq!(decode(bytes, DecodeOptions::default()).is_ok(), old_ok, "lossless={lossless}, cut={cut}/{}", raw.len());
}
}
}
+43
View File
@@ -0,0 +1,43 @@
#![cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
use lightcraft_codecs::*;
use lightcraft_raster::Rgba8;
#[global_allocator]
static ALLOC: dhat::Alloc = dhat::Alloc;
fn codestream(bytes: &[u8]) -> Vec<u8> {
if bytes.starts_with(b"\xff\x0a") {
return bytes.to_vec();
}
let mut raw = Vec::new();
let mut at = 12;
while at + 8 <= bytes.len() {
let len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
let end = if len == 0 { bytes.len() } else { at + len };
match &bytes[at + 4..at + 8] {
b"jxlc" => raw.extend_from_slice(&bytes[at + 8..end]),
b"jxlp" => raw.extend_from_slice(&bytes[at + 12..end]),
_ => {}
}
at = end;
}
assert!(raw.starts_with(b"\xff\x0a"));
raw
}
#[test]
fn tiny_jxl_tail_does_not_allocate_a_tail_sized_buffer() {
let image = Rgba8::from_fn(8, 8, |x, y| [(x * 20) as u8, (y * 20) as u8, 80, 255]);
let encoded = encode_jxl(&EncodeImage::rgba8(&image), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
let mut raw = codestream(&encoded);
let opts = DecodeOptions { max_pixels: 64, ..Default::default() };
decode(&raw, opts).unwrap();
raw.resize(raw.len() + (1 << 20), 0);
let _profiler = dhat::Profiler::builder().testing().build();
let decoded = decode(&raw, opts).unwrap();
assert_eq!((decoded.width, decoded.height), (8, 8));
let peak = dhat::HeapStats::get().max_bytes;
eprintln!("peak heap bytes: {peak}");
assert!(peak < (128 << 10), "tiny image used {peak} heap bytes");
}
+99
View File
@@ -263,6 +263,90 @@ fn tiff_cmyk_and_orientation() {
assert!(red[0] > 0.99 && red[1] < 0.01 && red[2] < 0.01, "{red:?}");
}
/// An uncompressed 16-bit RGB export in a single strip, the shape of the file in #1983 (3923 × 5877,
/// 138 MB). Its one strip is larger than the `tiff` crate's default 128 MiB intermediate buffer, so
/// the thumbnail (and the full decode) failed with "decoder limits exceeded".
#[test]
fn tiff_single_strip_larger_than_128_mib() {
use tiff::encoder::{TiffEncoder, colortype};
let (w, h) = (3923u32, 5877u32);
let mut cur = std::io::Cursor::new(Vec::new());
{
let mut enc = TiffEncoder::new(&mut cur).unwrap();
let mut im = enc.new_image::<colortype::RGB16>(w, h).unwrap();
im.rows_per_strip(h).unwrap();
im.write_data(&vec![0x8000u16; (w * h * 3) as usize]).unwrap();
}
let bytes = cur.into_inner();
assert!(bytes.len() > 128 << 20);
let t = decode_thumbnail(&bytes, 256).unwrap();
assert_eq!((t.source_width, t.source_height), (w, h));
let px = t.image.get(10, 10);
assert!(px[0].abs_diff(128) <= 2 && px[0] == px[1] && px[1] == px[2], "{px:?}");
}
/// A little-endian TIFF with one IFD of single-value `(tag, type, value)` entries, followed by `data`.
fn ifd_tiff(entries: &[(u16, u16, u32)], data: &[u8]) -> Vec<u8> {
let mut b = b"II*\0".to_vec();
b.extend(8u32.to_le_bytes());
b.extend((entries.len() as u16).to_le_bytes());
for &(tag, ty, value) in entries {
b.extend(tag.to_le_bytes());
b.extend(ty.to_le_bytes());
b.extend(1u32.to_le_bytes());
b.extend(value.to_le_bytes());
}
b.extend(0u32.to_le_bytes());
b.extend_from_slice(data);
b
}
/// A strip or tile can be larger than the pixels it shows (a JPEG header, padding to the tile size), so
/// a small `max_pixels` must not shrink the intermediate buffer below the `tiff` crate's 128 MiB
/// default. Both files below were refused with "decoder limits exceeded" when the buffer was capped at
/// 32 bytes per allowed pixel. Fixtures from AlexChambers' review of #1989.
#[test]
fn tiff_small_pixel_budget_reads_headers_and_padded_tiles() {
let opts = |max_pixels| DecodeOptions { max_size: None, max_pixels };
// 1 × 1 gray, one JPEG-compressed strip (~400 bytes for one pixel), with a one-pixel budget.
let mut jpeg = Vec::new();
jpeg_encoder::Encoder::new(&mut jpeg, 90).encode(&[128], 1, 1, jpeg_encoder::ColorType::Luma).unwrap();
let at = 8 + 2 + 10 * 12 + 4;
let entries = [
(256, 4, 1),
(257, 4, 1),
(258, 3, 8),
(259, 3, 7),
(262, 3, 1),
(273, 4, at),
(277, 3, 1),
(278, 4, 1),
(279, 4, jpeg.len() as u32),
(284, 3, 1),
];
let d = decode(&ifd_tiff(&entries, &jpeg), opts(1)).unwrap();
assert_eq!((d.image.width, d.image.height), (1, 1));
// 16 × 16 gray stored in one 256 × 256 tile (64 KiB), with a budget of exactly 256 pixels.
let at = 8 + 2 + 11 * 12 + 4;
let entries = [
(256, 4, 16),
(257, 4, 16),
(258, 3, 8),
(259, 3, 1),
(262, 3, 1),
(277, 3, 1),
(284, 3, 1),
(322, 4, 256),
(323, 4, 256),
(324, 4, at),
(325, 4, 65536),
];
let d = decode(&ifd_tiff(&entries, &[128; 65536]), opts(256)).unwrap();
assert_eq!((d.image.width, d.image.height), (16, 16));
}
/// A one-strip, uncompressed, single-sample TIFF (little-endian) whose samples are `bits` wide and
/// already packed into `strip`, one padded row after another.
fn packed_tiff(width: u16, height: u16, bits: u16, photometric: u16, strip: &[u8]) -> Vec<u8> {
@@ -734,6 +818,21 @@ fn jxl_lossless_exact_with_metadata() {
assert!(max_diff(&d.image, &want.image) < 1e-5, "16-bit lossless differs by {}", max_diff(&d.image, &want.image));
}
/// #1517: the pixel limit rejects an oversized image; selected truncations never panic.
#[test]
#[cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
fn jxl_size_limit_and_truncation() {
let b = encode_jxl(&EncodeImage::rgba8(&gradient(97, 61)), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
let r = decode(&b, DecodeOptions { max_pixels: 97 * 61 - 1, ..Default::default() });
assert!(matches!(r, Err(Error::TooLarge(97, 61))), "{r:?}");
let d = decode(&b, DecodeOptions { max_pixels: 97 * 61, ..Default::default() }).unwrap();
assert_eq!((d.width, d.height), (97, 61));
for cut in [0, 1, 2, 5, 12, 40, b.len() / 2, b.len() - 1] {
let _ = decode(&b[..cut], DecodeOptions::default());
}
assert!(decode(&b[..12.min(b.len())], DecodeOptions::default()).is_err());
}
#[test]
#[cfg(feature = "jxl")]
fn jxl_lossy_quality_and_effort() {
+11 -2
View File
@@ -180,8 +180,8 @@ pub struct Rgb8(pub u8, pub u8, pub u8);
impl Rgb8 {
pub fn from_hex(s: &str) -> Option<Rgb8> {
let s = s.trim_start_matches('#');
if s.len() != 6 {
let s = s.strip_prefix('#').unwrap_or(s);
if s.len() != 6 || !s.bytes().all(|b| b.is_ascii_hexdigit()) {
return None;
}
let v = u32::from_str_radix(s, 16).ok()?;
@@ -196,6 +196,15 @@ impl Rgb8 {
mod tests {
use super::*;
#[test]
fn rgb8_from_hex_rejects_signs_and_extra_hashes() {
assert_eq!(Rgb8::from_hex("#0f0f0f"), Some(Rgb8(15, 15, 15)));
assert_eq!(Rgb8::from_hex("ff8000"), Some(Rgb8(255, 128, 0)));
for bad in ["+12345", "#+12345", "#+f+f+f", "-12345", "##123456", "#12345", ""] {
assert_eq!(Rgb8::from_hex(bad), None, "{bad}");
}
}
#[test]
fn srgb_matrix_matches_standard() {
let m = SRGB.to_xyz();
+16 -1
View File
@@ -14,7 +14,7 @@ use crate::manifest::Gain;
use crate::tiles::{starts, weight};
/// Most pixels one picture may have (guards the allocations that follow from its size).
pub const MAX_PIXELS: usize = 100_000_000;
pub const MAX_PIXELS: usize = 110_000_000;
/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
const MIN_SIGNAL: f32 = 0.003;
@@ -642,4 +642,19 @@ mod tests {
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
}
#[test]
fn accepts_102_mp_inputs_and_rejects_inputs_above_the_limit() {
let layout = Layout { dx: 0, dy: 0 };
let model = Mock { tile: 32, scale: 1.0 };
let p = params(32, 8, Gain::None);
assert!(matches!(
denoise_bayer(&[], 11_648, 8_736, layout, &model, &p, &Control::default()),
Err(Error::Input(message)) if message == "the mosaic does not have width × height samples"
));
assert!(matches!(
denoise_bayer(&[], MAX_PIXELS + 1, 1, layout, &model, &p, &Control::default()),
Err(Error::Input(message)) if message == "the picture has no size, or is too large"
));
}
}
+50 -1
View File
@@ -41,7 +41,7 @@ impl DevelopSettings {
/// Merge a partial JSON object into these settings (fields not mentioned are kept). A white balance that
/// names no [`WbScale`] keeps the meaning it always had (`Legacy`), whatever scale these settings were on: a
/// partial from before the scale existed, or one that never said, must not become Kelvin by landing on
/// Kelvin settings.
/// Kelvin settings. Out-of-range values are brought back into range ([`DevelopSettings::keep_in_range`]).
pub fn merged(&self, partial: &Value) -> Result<DevelopSettings, serde_json::Error> {
let mut v = self.to_json();
presets::deep_merge(&mut v, partial);
@@ -52,9 +52,34 @@ impl DevelopSettings {
{
out.wb.scale = WbScale::Legacy;
}
out.keep_in_range(self);
Ok(out)
}
/// What crs XMP import does to the values it reads, for settings that arrive as JSON (presets, `develop.merge`,
/// `render --settings`; issue #1946): every slider clamped through its control spec, point curves in 0..1, and a
/// crop with no area left as it was in `before`. Unrotated, the crop is clipped to the photo first; a turned crop
/// may reach past 0..1 and is left to the crop tools, which know the photo's size.
fn keep_in_range(&mut self, before: &DevelopSettings) {
for c in controls::CONTROLS {
if let Some(v) = controls::get(self, c.id) {
controls::set(self, c.id, v);
}
}
let c = &mut self.curve;
for p in [&mut c.master, &mut c.red, &mut c.green, &mut c.blue].into_iter().flatten() {
*p = lightcraft_geom::Point::new(p.x.clamp(0.0, 1.0), p.y.clamp(0.0, 1.0));
}
let g = &mut self.crop.geometry;
if g.angle == 0.0 {
g.rect = g.rect.intersect(&lightcraft_geom::Rect::UNIT);
}
if g.rect.is_empty() {
let old = before.crop.geometry.rect;
g.rect = if old.is_empty() { lightcraft_geom::Rect::UNIT } else { old };
}
}
/// Stable 64-bit hash of the settings (FNV-1a over canonical JSON), for preview cache keys.
pub fn hash64(&self) -> u64 {
let s = serde_json::to_string(self).unwrap_or_default();
@@ -255,6 +280,30 @@ mod tests {
assert_eq!(m.light.exposure, 0.0);
}
#[test]
fn merged_settings_are_kept_in_range() {
// issue #1946: `develop.merge`, `render --settings` and presets stored whatever the JSON said
let mut base = DevelopSettings::default();
base.crop.geometry.rect = lightcraft_geom::Rect::new(0.1, 0.2, 0.8, 0.9);
let m = base
.merged(&json!({"light": {"exposure": 1e308}, "vignette": {"amount": -1e308}, "geometry": {"scale": 0}, "optics": {"distortion": 1e308}}))
.unwrap();
assert_eq!((m.light.exposure, m.vignette.amount, m.geometry.scale, m.optics.distortion), (5.0, -100.0, 50.0, 100.0));
// point curves stay in 0..1, as an imported XMP curve does
let m = base.merged(&json!({"curve": {"master": [{"x": -3, "y": 0}, {"x": 1, "y": 1e308}]}})).unwrap();
assert_eq!(m.curve.master, vec![lightcraft_geom::Point::new(0.0, 0.0), lightcraft_geom::Point::new(1.0, 1.0)]);
// a crop with no area is left out, as XMP import leaves out one whose edges are inverted
for rect in
[json!({"x0": 0.9, "x1": 0.1, "y0": 0, "y1": 1}), json!({"x0": 1e308, "x1": -1e308, "y0": 0, "y1": 1}), json!({"x0": 0.5, "x1": 0.5})]
{
let m = base.merged(&json!({"crop": {"geometry": {"rect": rect}}})).unwrap();
assert_eq!(m.crop.geometry.rect, base.crop.geometry.rect, "{rect}");
}
// unrotated, the crop stays inside the photo
let m = base.merged(&json!({"crop": {"geometry": {"rect": {"x0": -1e308, "y0": 0.25, "x1": 1e308, "y1": 0.75}}}})).unwrap();
assert_eq!(m.crop.geometry.rect, lightcraft_geom::Rect::new(0.0, 0.25, 1.0, 0.75));
}
#[test]
fn unedited_and_reset() {
let mut s = DevelopSettings::default();
+2 -8
View File
@@ -184,17 +184,11 @@ fn scale_patch(base: &Value, patch: &Value, t: f64) -> Value {
}
/// Apply a partial settings object with an amount (1.0 = 100 %). Out-of-range values are clamped
/// through the control specs.
/// through the control specs, as every merge does ([`DevelopSettings::merged`]).
pub fn apply_partial(s: &DevelopSettings, partial: &Value, amount: f64) -> DevelopSettings {
let patch = if (amount - 1.0).abs() < 1e-9 { Cow::Borrowed(partial) } else { Cow::Owned(scale_patch(&s.to_json(), partial, amount)) };
// (a white balance without a scale keeps its old meaning: `DevelopSettings::merged`)
let mut out = s.merged(&patch).unwrap_or_else(|_| s.clone());
for c in crate::controls::CONTROLS {
if let Some(v) = crate::controls::get(&out, c.id) {
crate::controls::set(&mut out, c.id, v);
}
}
out
s.merged(&patch).unwrap_or_else(|_| s.clone())
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+2
View File
@@ -9,6 +9,8 @@ description = "LightCraft engine: session, command registry, history, render job
workspace = true
[dependencies]
# Paths stored in a Lightroom catalog are followed only to local files or the catalog's share.
craft-docpath = { workspace = true }
lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
+47
View File
@@ -0,0 +1,47 @@
//! What a Synchronize Folder scan costs on a folder whose photos are all in the library and
//! unchanged: the walk, and the check of every photo and its sidecar. Reads the folder; writes
//! nothing there, and probes no file (the library is made from the folder's own listing).
//!
//! `cargo run --release -p lightcraft-engine --example sync_scan_bench -- <folder>`
use std::time::Instant;
use lightcraft_catalog::{Op, Photo, Source};
use lightcraft_engine::Session;
use lightcraft_engine::sync::{SyncInput, SyncProgress, scan_with};
fn main() {
let Some(folder) = std::env::args().nth(1) else {
eprintln!("usage: sync_scan_bench <folder>");
return;
};
let mut s = Session::new();
let t = Instant::now();
let files = lightcraft_engine::import::expand(std::slice::from_ref(&folder), None);
println!("{} files listed in {:.1} s (the library is made of them)", files.len(), t.elapsed().as_secs_f64());
for f in &files {
let id = s.catalog.alloc_photo_id();
let name = std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let p = Photo::new(id, Source::File { path: f.clone() }, &name, "JPEG", 6, 4, "2026-01-01T00:00:00");
if s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).is_err() {
eprintln!("could not add {name}");
}
}
for run in 0..2 {
let input = match SyncInput::new(&mut s, &folder, true) {
Ok(input) => input,
Err(e) => {
eprintln!("{e}");
return;
}
};
let t = Instant::now();
let changes = scan_with(input, &SyncProgress::default());
println!(
"run {run}: scan {:.1} s ({} new, {} missing, {} with sidecar news)",
t.elapsed().as_secs_f64(),
changes.new.len(),
changes.missing.len(),
changes.metadata.len()
);
}
}
+1 -1
View File
@@ -73,7 +73,7 @@ impl Unit {
pub struct TaskInfo {
pub id: u64,
/// A stable id: `import`, `export`, `scan`, `previews`, `smartPreviews`, `lightroom`, `merge`, `download`,
/// `faces`, `denoise`, `findMissing`, `applePhotos`, `cache`.
/// `faces`, `denoise`, `findMissing`, `applePhotos`, `cache`, `focus`.
pub kind: &'static str,
/// English; the frontend translates it.
pub label: String,
+249 -24
View File
@@ -31,7 +31,10 @@ use lightcraft_raw::{
/// 4: a tone curve from black-and-white camera JPEGs (#577).
/// 5: dark JPEG luminance pairs for the camera toe (#1307).
/// 6: the ILCA-99M2 profile.
pub const LOOK_VERSION: u32 = 6;
/// 7: the sensor proxy follows the file's lens warp when the camera JPEG is lens-corrected.
/// 8: the bundled NIKON Z 6_2 profile.
/// 9: the bundled NIKON Z 8 and NIKON D5300 profiles.
pub const LOOK_VERSION: u32 = 9;
#[derive(Clone, Debug)]
pub(crate) struct CameraLook {
@@ -76,14 +79,18 @@ pub(crate) fn file_local_look(format: RawFormat) -> bool {
/// white (a monochrome picture style) has no colour to fit, but still gives its tone curve: with the camera profile's
/// colour in [`fit_preview`], else on top of the spectral matrices or the fallback ([`monochrome_look`]). Returns the
/// as-shot white, the camera transform at it and the fitted look.
/// `lens` is the file's own lens-correction warp ([`lens_geometry`], read before the caller removes it from
/// `OpcodeList3`): the camera applied it to its JPEG, so the sensor proxy is warped the same way before pairing.
/// `embedded` is the file's embedded lens vignetting, for the DNG tone fit.
pub(crate) fn starting_colour(
raw: &mut RawImage,
bytes: &[u8],
lens: Option<&lightcraft_develop::EmbeddedLens>,
lens: Option<&LensGeometry>,
embedded: Option<&lightcraft_develop::EmbeddedLens>,
) -> (Xy, CameraTransform, Option<CameraLook>) {
let (xy, t, look) = starting_colour_with(raw, |raw, t| fit_preview(raw, bytes, t), |raw, t| monochrome_look(raw, bytes, t));
let (xy, t, look) = starting_colour_with(raw, |raw, t| fit_preview(raw, bytes, t, lens), |raw, t| monochrome_look(raw, bytes, t, lens));
if look.is_none() && dng_tone_eligible(raw, &t) {
let look = fit_dng_tone(raw, bytes, &t, lens);
let look = fit_dng_tone(raw, bytes, &t, embedded);
return (xy, t, look);
}
(xy, t, look)
@@ -154,7 +161,8 @@ pub(crate) fn starting_colour_tags(raw: &mut RawImage, bytes: &[u8]) -> Option<C
// the white the source shows, decided before `starting_colour` may fill in measured matrices
let scale = crate::files::WbScaleOf::new(raw.format, &raw.color, raw.wb_multipliers, &raw.metadata);
let white = scale.dng_white(&raw.color, lightcraft_raw::color::as_shot_white_xy(raw));
let (_, t, look) = starting_colour(raw, bytes, None);
// (a merge frame arrives without its lens warp: `lightcraft_merge` removes it before calling)
let (_, t, look) = starting_colour(raw, bytes, None, None);
dng_colour(&raw.color, &white, &t, look.as_ref())
}
@@ -252,11 +260,11 @@ fn dng_tone_curve(tone: &CameraTone) -> Option<ToneCurve> {
}
/// The look fitted to the file's camera JPEG, and whether that JPEG is black and white ([`monochrome`]).
pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransform) -> (Option<CameraLook>, bool) {
pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, lens: Option<&LensGeometry>) -> (Option<CameraLook>, bool) {
if !transform.matrix_is_fallback || !file_local_look(raw.format) {
return (None, false);
}
let Some((sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, 0.0) else { return (None, false) };
let Some((sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, 0.0, lens) else { return (None, false) };
let black_and_white = monochrome(&reference);
// A camera profile pooled from many photos knows colours this photo shows too little of;
// try its colour first and fit tone/chroma per photo (picture styles vary).
@@ -427,7 +435,7 @@ fn fit_dng_tone(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, lens:
}
None
};
let Some((mut sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, MIN_PREVIEW_FRACTION) else { return refuse("preview") };
let Some((mut sensor, reference, clipped)) = proxies(raw, bytes, transform, PROXY, MIN_PREVIEW_FRACTION, None) else { return refuse("preview") };
// As the loader renders it: the profile's tables around the baseline exposure.
let xy = lightcraft_raw::color::as_shot_white_xy(raw);
if let Some(tables) = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy)) {
@@ -569,7 +577,14 @@ fn structure_agrees(sensor: &Rgb32f, reference: &Rgb32f) -> bool {
/// Same-size proxies of the sensor (white-balanced, baseline exposure, through `transform`'s
/// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020),
/// and per proxy pixel whether any sensor sample it covers is clipped.
fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usize, min_fraction: f64) -> Option<(Rgb32f, Rgb32f, Vec<bool>)> {
fn proxies(
raw: &RawImage,
bytes: &[u8],
transform: &CameraTransform,
size: usize,
min_fraction: f64,
lens: Option<&LensGeometry>,
) -> Option<(Rgb32f, Rgb32f, Vec<bool>)> {
let edge = (2 * size).max(384) as u32;
let decoded = crate::files::decode_raw_preview(
bytes,
@@ -604,6 +619,12 @@ fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usiz
let k = (crop.width.max(crop.height).div_ceil(edge as usize).max(2)).div_ceil(2) * 2;
let mut sensor = sensor_proxy(raw, k, edge as usize)?;
let mut clipped = clip_mask(&sensor);
// The camera may have corrected its JPEG with the lens warp the file carries (Nikon Z lenses always, at up to 9 %
// zoom; other cameras when their correction setting is on): the sensor moved into that geometry is a candidate,
// kept below when it lines up with the JPEG better than the uncorrected one.
let warped = lens
.filter(|_| raw.format != RawFormat::Cr3)
.and_then(|l| Some((into_lens_geometry(&sensor, raw, k, l)?, into_lens_geometry(&clipped, raw, k, l)?)));
let gain = 2f32.powf(transform.baseline_exposure as f32);
let to_working = |p: [f32; 3]| transform.matrix.apply_f32(std::array::from_fn(|i| p[i] * transform.wb[i] * gain));
if raw.format == RawFormat::Cr3 {
@@ -617,7 +638,15 @@ fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usiz
(sensor, clipped) = align_cr3_framing(sensor, clipped, &reference);
}
let mut sensor = fit(&sensor, size, size, Filter::Box);
let reference = reference_at(&reference, &sensor);
let mut reference_small = reference_at(&reference, &sensor);
if let Some((warped, warped_clipped)) = warped {
let candidate = fit(&warped, size, size, Filter::Box);
let candidate_reference = reference_at(&reference, &candidate);
if alignment(&candidate, &candidate_reference) > alignment(&sensor, &reference_small) {
(sensor, reference_small, clipped) = (candidate, candidate_reference, warped_clipped);
}
}
let reference = reference_small;
// A proxy pixel is clipped when any sample it averages is (outside the CR3 framing the mask is
// NaN: those pixels are left out of the pairs anyway). Resized to exactly the proxy's size, like
// the reference, so the mask pairs pixel for pixel.
@@ -779,12 +808,12 @@ fn align_cr3_framing(sensor: Rgb32f, clipped: Rgb32f, reference: &Rgb32f) -> (Rg
/// Colour training pairs of one raw for a camera profile: white-balanced camera RGB (with the
/// baseline exposure) → its camera JPEG (linear Rec.2020). `None` for formats with their own
/// colour matrices, other files or unusable previews.
pub(crate) fn profile_pairs(raw: &RawImage, bytes: &[u8]) -> Option<Vec<([f64; 3], [f64; 3])>> {
pub(crate) fn profile_pairs(raw: &RawImage, bytes: &[u8], lens: Option<&LensGeometry>) -> Option<Vec<([f64; 3], [f64; 3])>> {
if !file_local_look(raw.format) || lightcraft_raw::color::has_matrix(&raw.color) {
return None;
}
let transform = lightcraft_raw::color::camera_transform(raw, lightcraft_raw::color::as_shot_white_xy(raw));
let (sensor, reference, clipped) = proxies(raw, bytes, &transform, PROFILE_PROXY, 0.0)?;
let (sensor, reference, clipped) = proxies(raw, bytes, &transform, PROFILE_PROXY, 0.0, lens)?;
let to_camera = transform.matrix.inverse()?;
// clipped sensor pixels don't record the scene's colour (see `fit_look`)
let sensor = without_clipped(&sensor, &clipped).unwrap_or(sensor);
@@ -799,6 +828,102 @@ pub(crate) fn fit_profile(pairs: &[([f64; 3], [f64; 3])]) -> Option<(Mat3, Optio
Some((matrix, fit_hue_sat(pairs, &matrix)))
}
/// A raw's lens-correction warp, as its camera applied it to the embedded JPEG: the first `WarpRectilinear` of
/// `OpcodeList3` (its green plane; the in-file corrections LightCraft reads have no tangential terms). `None` without
/// one, or for a DNG 1.6 `WarpRectilinear2` chain (phone DNGs, which carry colour matrices and are not fitted).
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct LensGeometry {
/// `kr0..kr3`: source radius = r * (kr0 + kr1 r^2 + kr2 r^4 + kr3 r^6), r over the farthest corner distance.
k: [f64; 4],
/// The optical centre, relative to the active area (as the opcode has it).
center: [f64; 2],
}
pub(crate) fn lens_geometry(list3: &[lightcraft_raw::Opcode]) -> Option<LensGeometry> {
use lightcraft_raw::Opcode;
let ops: Vec<&Opcode> = lightcraft_raw::opcodes::applied_ops(list3).collect();
if ops.iter().any(|op| matches!(op, Opcode::WarpRectilinear2 { .. })) {
return None;
}
ops.into_iter().find_map(|op| match op {
Opcode::WarpRectilinear { planes, center } => {
let p = planes.get(1).or(planes.first())?;
let k = [p[0], p[1], p[2], p[3]];
(k.iter().chain(center).all(|v| v.is_finite()) && (0.0..=1.0).contains(&center[0]) && (0.0..=1.0).contains(&center[1]))
.then_some(LensGeometry { k, center: *center })
}
_ => None,
})
}
/// How well two same-size proxies line up: the correlation of their high-passed log luminance over the pixels both
/// have (a tone curve or colour change leaves it alone, a geometric offset lowers it). `-1` without enough pixels.
fn alignment(a: &Rgb32f, b: &Rgb32f) -> f64 {
let (w, h) = (a.width, a.height);
if (b.width, b.height) != (w, h) || w < 3 || h < 3 {
return -1.0;
}
let lum = |img: &Rgb32f| img.data.iter().map(|p| (luminance_2020(*p).max(1e-4) as f64).ln()).collect::<Vec<f64>>();
let high = |l: &[f64]| -> Vec<f64> {
(0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
if x == 0 || y == 0 || x + 1 == w || y + 1 == h {
return f64::NAN;
}
let around = [i - w - 1, i - w, i - w + 1, i - 1, i + 1, i + w - 1, i + w, i + w + 1];
l[i] - around.iter().map(|&j| l[j]).sum::<f64>() / 8.0
})
.collect()
};
let (ha, hb) = (high(&lum(a)), high(&lum(b)));
let pairs: Vec<(f64, f64)> = ha.into_iter().zip(hb).filter(|(x, y)| x.is_finite() && y.is_finite()).collect();
if pairs.len() < 64 {
return -1.0;
}
let n = pairs.len() as f64;
let (ma, mb) = (pairs.iter().map(|p| p.0).sum::<f64>() / n, pairs.iter().map(|p| p.1).sum::<f64>() / n);
let cov: f64 = pairs.iter().map(|p| (p.0 - ma) * (p.1 - mb)).sum();
let (va, vb): (f64, f64) = (pairs.iter().map(|p| (p.0 - ma).powi(2)).sum(), pairs.iter().map(|p| (p.1 - mb).powi(2)).sum());
let r = cov / (va * vb).sqrt().max(1e-12);
if r.is_finite() { r } else { -1.0 }
}
/// A `k`-binned sensor proxy of `raw` ([`sensor_proxy`]: the default crop, or the active area when the crop is
/// smaller than a block, at `1/k`) moved into the camera JPEG's geometry: each pixel samples the proxy where `lens`
/// maps it, and is NaN where that falls outside the proxy (the pairs leave it out). `None` for a proxy of another
/// size (the demosaiced fallback of linear raws).
fn into_lens_geometry(img: &Rgb32f, raw: &RawImage, k: usize, lens: &LensGeometry) -> Option<Rgb32f> {
let a = raw.active_area;
let c = raw.crop.clipped(a.width, a.height);
let c = if c.width < k || c.height < k { lightcraft_raw::Rect::new(0, 0, a.width, a.height) } else { c };
let (w, h) = (img.width, img.height);
if k == 0 || w != c.width / k || h != c.height / k || w == 0 || h == 0 || a.width < 2 || a.height < 2 {
return None;
}
let (aw, ah) = (a.width as f64 - 1.0, a.height as f64 - 1.0);
let (cx, cy) = (lens.center[0] * aw, lens.center[1] * ah);
let m = [(0.0, 0.0), (aw, 0.0), (0.0, ah), (aw, ah)].iter().map(|&(x, y)| (x - cx).hypot(y - cy)).fold(0.0, f64::max).max(1e-9);
let [k0, k1, k2, k3] = lens.k;
let kf = k as f64;
let (x0, y0) = (c.x as f64, c.y as f64);
let mut out = Rgb32f::new(w, h);
for (i, px) in out.data.iter_mut().enumerate() {
// the proxy pixel's centre in active-area pixel-index coordinates, mapped to the sensor
let (ox, oy) = (x0 + ((i % w) as f64 + 0.5) * kf - 0.5, y0 + ((i / w) as f64 + 0.5) * kf - 0.5);
let (dx, dy) = ((ox - cx) / m, (oy - cy) / m);
let r2 = dx * dx + dy * dy;
let f = k0 + r2 * (k1 + r2 * (k2 + r2 * k3));
let (sx, sy) = ((cx + m * dx * f + 0.5 - x0) / kf, (cy + m * dy * f + 0.5 - y0) / kf);
*px = if sx >= 0.5 && sy >= 0.5 && sx <= w as f64 - 0.5 && sy <= h as f64 - 0.5 {
img.sample_bilinear(sx as f32, sy as f32)
} else {
[f32::NAN; 3]
};
}
Some(out)
}
fn sensor_proxy(raw: &RawImage, k: usize, edge: usize) -> Option<Rgb32f> {
Some(match raw.develop_binned(k, SENSOR_CLIP).ok()? {
Some(sensor) => sensor,
@@ -1296,11 +1421,11 @@ fn monochrome(reference: &Rgb32f) -> bool {
/// `transform` is the colour the photo gets otherwise (the spectral matrices or the fallback's); the look keeps it and
/// adds the tone curve alone ([`fit_tone_only`]), as [`fit_look`] tries the photo's own colour after a profile's. The
/// chroma curve stays neutral: the JPEG has no colourfulness to learn.
fn monochrome_look(raw: &RawImage, bytes: &[u8], transform: &CameraTransform) -> Option<CameraLook> {
fn monochrome_look(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, lens: Option<&LensGeometry>) -> Option<CameraLook> {
if !file_local_look(raw.format) {
return None;
}
let (sensor, reference, clipped) = proxies(raw, bytes, transform, PROXY, 0.0)?;
let (sensor, reference, clipped) = proxies(raw, bytes, transform, PROXY, 0.0, lens)?;
if !monochrome(&reference) {
return None;
}
@@ -2334,6 +2459,93 @@ mod tests {
}
}
/// The lens warp the colour fit follows is the first `WarpRectilinear`'s green plane; no warp, a DNG 1.6 chain
/// (phone DNGs, which carry colour matrices), a centre outside the area or a non-finite term give none.
#[test]
fn lens_geometry_reads_the_first_rectilinear_warp() {
use lightcraft_raw::Opcode;
let warp = |k0: f64, c: [f64; 2]| Opcode::WarpRectilinear {
planes: vec![[9.0, 0.0, 0.0, 0.0, 0.0, 0.0], [k0, 0.1, -0.02, 0.003, 0.0, 0.0], [9.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
center: c,
};
assert_eq!(lens_geometry(&[warp(0.97, [0.5, 0.4])]), Some(LensGeometry { k: [0.97, 0.1, -0.02, 0.003], center: [0.5, 0.4] }));
let single = Opcode::WarpRectilinear { planes: vec![[1.02, 0.0, 0.0, 0.0, 0.0, 0.0]], center: [0.5, 0.5] };
assert_eq!(lens_geometry(std::slice::from_ref(&single)).map(|g| g.k[0]), Some(1.02));
assert!(lens_geometry(&[]).is_none());
assert!(lens_geometry(&[warp(1.0, [1.5, 0.5])]).is_none());
assert!(lens_geometry(&[warp(f64::NAN, [0.5, 0.5])]).is_none());
let mut p = [0.0; lightcraft_raw::opcodes::WR2_LEN];
p[0] = 1.0;
p[18] = 1e6;
let chain = Opcode::WarpRectilinear2 { planes: vec![p], center: [0.5, 0.5], reciprocal: false, optional: false };
assert!(lens_geometry(&[chain, single]).is_none());
}
/// A binned sensor proxy moved into the camera JPEG's geometry: the identity warp leaves it as it is, a warp
/// whose source radius is 0.8 of the output's samples nearer the centre, output pixels whose source falls outside
/// the proxy are NaN (left out of the pairs), and a proxy of another size is left alone.
#[test]
fn the_sensor_proxy_follows_the_cameras_lens_warp() {
let mut raw = raw_of(RawFormat::Nef, "NIKON CORPORATION", "NIKON Z 6_2", Default::default());
raw.active_area = lightcraft_raw::Rect::new(0, 0, 64, 48);
raw.crop = raw.active_area;
// a 1/4 proxy whose pixels hold their own coordinates
let mut img = Rgb32f::new(16, 12);
for (i, p) in img.data.iter_mut().enumerate() {
*p = [(i % 16) as f32, (i / 16) as f32, 1.0];
}
let geometry = |k0: f64| LensGeometry { k: [k0, 0.0, 0.0, 0.0], center: [0.5, 0.5] };
let same = into_lens_geometry(&img, &raw, 4, &geometry(1.0)).unwrap();
assert!(same.data.iter().zip(&img.data).all(|(a, b)| (0..3).all(|c| (a[c] - b[c]).abs() < 1e-4)));
// the corner pixel's centre (1.5, 1.5) maps to (31.5 - 0.8 * 30, 23.5 - 0.8 * 22) = (7.5, 5.9) on the sensor
let zoomed = into_lens_geometry(&img, &raw, 4, &geometry(0.8)).unwrap();
assert!((zoomed.data[0][0] - 1.5).abs() < 1e-4 && (zoomed.data[0][1] - 1.1).abs() < 1e-4, "{:?}", zoomed.data[0]);
let outside = into_lens_geometry(&img, &raw, 4, &geometry(1.3)).unwrap();
assert!(outside.data[0][0].is_nan() && outside.data[6 * 16 + 8][0].is_finite());
assert!(into_lens_geometry(&Rgb32f::new(15, 12), &raw, 4, &geometry(1.0)).is_none());
}
/// A raw with an in-file lens correction (Fujifilm, Panasonic, the A7R IVA, Nikon) may have a lens-corrected
/// camera JPEG: the proxies the fit pairs line up with it at least as well as the uncorrected sensor, on every
/// corpus file the colour fit uses that carries such a warp, and better on those whose JPEG is corrected.
#[test]
fn corpus_lens_corrected_jpegs_pair_with_the_warped_sensor() {
let dir = std::env::var_os("LIGHTCRAFT_CORPUS")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
.join("raw");
let Ok(entries) = std::fs::read_dir(&dir) else {
eprintln!("skip: no corpus at {}", dir.display());
return;
};
let mut paths: Vec<_> = entries.flatten().map(|e| e.path()).collect();
paths.sort();
let (mut checked, mut gain) = (0, 0.0);
for path in paths {
let Ok(bytes) = std::fs::read(&path) else { continue };
let Ok(mut raw) = lightcraft_raw::decode(&bytes) else { continue };
let Some(lens) = lens_geometry(&raw.opcodes.list3) else { continue };
if !file_local_look(raw.format) || lightcraft_raw::color::has_matrix(&raw.color) {
continue;
}
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let t = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw));
let (Some((plain, reference, _)), Some((chosen, chosen_reference, _))) =
(proxies(&raw, &bytes, &t, PROXY, 0.0, None), proxies(&raw, &bytes, &t, PROXY, 0.0, Some(&lens)))
else {
continue;
};
let (before, after) = (alignment(&plain, &reference), alignment(&chosen, &chosen_reference));
eprintln!("{}: alignment {before:.3} -> {after:.3}", path.display());
// (chosen on camera RGB, measured after the colour transform: a near tie may flip by a little)
assert!(after >= before - 0.01, "{}: {before:.3} -> {after:.3}", path.display());
gain += after - before;
checked += 1;
}
eprintln!("{checked} corpus raws with an in-file lens warp, mean gain {:.3}", gain / checked.max(1) as f64);
assert!(checked == 0 || gain > 0.0);
}
/// Step 1: a file's own colour matrices win over everything, including a spectral row for its model; no JPEG fit
/// is tried.
#[test]
@@ -2381,7 +2593,7 @@ mod tests {
assert!(crate::camera_profiles::get("ILCE-7M4").is_some());
let camera = lightcraft_raw::spectral::find(Some("SONY"), "ILCE-7M4").unwrap();
let mut raw = raw_of(RawFormat::Arw, "SONY", "ILCE-7M4", Default::default());
let (_, t, look) = starting_colour(&mut raw, &[], None);
let (_, t, look) = starting_colour(&mut raw, &[], None, None);
assert!(look.is_none());
assert_eq!(raw.color.color_matrix, camera.color_matrix.map(Some));
assert!(!t.matrix_is_fallback);
@@ -2440,13 +2652,13 @@ mod tests {
#[test]
fn a_camera_outside_the_table_keeps_the_fallback() {
let mut raw = raw_of(RawFormat::Cr2, "Canon", "Canon EOS 7D", Default::default());
let (_, t, look) = starting_colour(&mut raw, &[], None);
let (_, t, look) = starting_colour(&mut raw, &[], None, None);
assert!(look.is_none() && t.matrix_is_fallback);
assert_eq!(raw.color, lightcraft_raw::ColorData::default());
// without a model there is no table row
let mut raw = raw_of(RawFormat::Cr2, "Canon", "", Default::default());
raw.metadata.model = None;
assert!(starting_colour(&mut raw, &[], None).1.matrix_is_fallback);
assert!(starting_colour(&mut raw, &[], None, None).1.matrix_is_fallback);
}
/// The colour precedence on public raws (each skipped without its corpus file): a photo whose JPEG fit is
@@ -2465,13 +2677,14 @@ mod tests {
continue;
};
let mut raw = lightcraft_raw::decode(&bytes).unwrap();
let lens = lens_geometry(&raw.opcodes.list3);
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let own = lightcraft_raw::color::has_matrix(&raw.color);
let (make, model) = (raw.metadata.make.as_deref(), raw.metadata.model.as_deref().unwrap_or_default());
let measured = lightcraft_raw::spectral::find(make, model).is_some();
let covered = measured || lightcraft_raw::kelvin_presets::covers(make, model);
let before = raw.color.clone();
let (_, t, fitted) = starting_colour(&mut raw, &bytes, None);
let (_, t, fitted) = starting_colour(&mut raw, &bytes, lens.as_ref(), None);
let spectral = !own && fitted.is_none() && measured;
eprintln!("{name}: own matrices {own}, JPEG fit {}, spectral {spectral}", fitted.is_some());
assert_eq!(raw.color != before, spectral, "{name}");
@@ -2579,7 +2792,7 @@ mod tests {
assert_eq!(raw.format, RawFormat::Cr3);
assert_eq!(raw.info().developed_size(), (6000, 4000));
let transform = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw));
let look = fit_preview(&raw, &bytes, &transform).0;
let look = fit_preview(&raw, &bytes, &transform, None).0;
eprintln!("R100 guarded colour fit accepted: {}", look.is_some());
assert!(look.is_some(), "this R100 corpus photo has enough matching colour after camera-framing alignment");
let (_, info) = crate::files::load_bytes(&bytes, 400).unwrap();
@@ -2612,7 +2825,7 @@ mod tests {
let raw = lightcraft_raw::decode(&bytes).unwrap();
assert_eq!(raw.metadata.model.as_deref(), Some(model));
let transform = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw));
let look = fit_preview(&raw, &bytes, &transform).0.expect("fit accepted");
let look = fit_preview(&raw, &bytes, &transform, None).0.expect("fit accepted");
let expected = bundled.matrix().mul(&transform.matrix.inverse().unwrap());
for (row, want) in look.matrix.0.iter().zip(expected.0) {
for (got, want) in row.iter().zip(want) {
@@ -2632,6 +2845,18 @@ mod tests {
corpus_uses_the_bundled_profile("arw-sony-a99m2.arw", "ILCA-99M2");
}
/// The public Z 6II NEF wasn't one of the photos its bundled profile was fitted to.
#[test]
fn corpus_z6ii_uses_the_bundled_profile() {
corpus_uses_the_bundled_profile("nef-nikon-z6ii-lossless.nef", "NIKON Z 6_2");
}
/// The public Z 8 NEF (Z 14-24 f/2.8 S) wasn't one of the photos its bundled profile was fitted to.
#[test]
fn corpus_z8_uses_the_bundled_profile() {
corpus_uses_the_bundled_profile("nef-nikon-z8-lossless.nef", "NIKON Z 8");
}
/// A public D7500 NEF (skipped without the corpus): its look is fitted to its own JPEG and white
/// balance is relative to the as-shot look.
#[test]
@@ -2981,9 +3206,9 @@ mod tests {
let mut raw = lightcraft_raw::decode(&bytes).unwrap();
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let transform = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw));
let look = fit_preview(&raw, &bytes, &transform).0.expect("a camera look");
let look = fit_preview(&raw, &bytes, &transform, None).0.expect("a camera look");
// the look fitted to the JPEG, as before this change
let (sensor, reference, clipped) = proxies(&raw, &bytes, &transform, PROXY, 0.0).unwrap();
let (sensor, reference, clipped) = proxies(&raw, &bytes, &transform, PROXY, 0.0, None).unwrap();
let profile = raw.metadata.model.as_deref().and_then(crate::camera_profiles::get);
let colour = profile.as_ref().and_then(|p| Some((p.matrix().mul(&transform.matrix.inverse()?), p.hue_sat.clone())));
let fitted = fit_look(&sensor, &reference, &clipped, colour).unwrap();
@@ -3573,12 +3798,12 @@ mod tests {
tiff
};
let t = lightcraft_raw::color::camera_transform(&raw, lightcraft_raw::color::as_shot_white_xy(&raw));
let look = monochrome_look(&raw, &file([1.0; 3]), &t).expect("a tone curve from the black-and-white JPEG");
let look = monochrome_look(&raw, &file([1.0; 3]), &t, None).expect("a tone curve from the black-and-white JPEG");
assert!(look.matrix == Mat3::IDENTITY && look.hue_sat.is_none(), "on the transform's colour");
let curve = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0);
let mid = curve.apply(0.25);
assert!((mid - (1.0 - (-0.75f32).exp())).abs() < 0.05, "the camera's curve at 0.25: {mid}");
assert!(monochrome_look(&raw, &file([1.0, 0.9, 0.8]), &t).is_none(), "a tinted JPEG is not black and white");
assert!(monochrome_look(&raw, &file([1.0, 0.9, 0.8]), &t, None).is_none(), "a tinted JPEG is not black and white");
}
/// The black-and-white fit has the partial look's checks: a JPEG of another picture, or one the default curve
+7 -2
View File
@@ -28,6 +28,9 @@ pub const BUNDLED: &[(&str, &str)] = &[
("ILCA-99M2", include_str!("../../../assets/camera-profiles/ILCA-99M2.json")),
("ILCE-7CR", include_str!("../../../assets/camera-profiles/ILCE-7CR.json")),
("ILCE-7M4", include_str!("../../../assets/camera-profiles/ILCE-7M4.json")),
("NIKON D5300", include_str!("../../../assets/camera-profiles/NIKON_D5300.json")),
("NIKON Z 6_2", include_str!("../../../assets/camera-profiles/NIKON_Z_6_2.json")),
("NIKON Z 8", include_str!("../../../assets/camera-profiles/NIKON_Z_8.json")),
("X-H2S", include_str!("../../../assets/camera-profiles/X-H2S.json")),
("X-T4", include_str!("../../../assets/camera-profiles/X-T4.json")),
];
@@ -195,10 +198,12 @@ impl Pool {
/// NEF or RAF without colour matrices, no usable camera JPEG, too little colour).
pub fn add(&mut self, bytes: &[u8]) -> Result<Option<String>, String> {
let mut raw = lightcraft_raw::decode(bytes).map_err(|e| e.to_string())?;
// colour only: geometric lens corrections would stop the sensor proxy from binning
// colour only: geometric lens corrections would stop the sensor proxy from binning (the pairs still follow
// the warp the camera applied to its JPEG)
let lens = crate::camera_preview::lens_geometry(&raw.opcodes.list3);
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
let Some(model) = raw.metadata.model.as_deref().map(str::trim).filter(|m| !m.is_empty()) else { return Ok(None) };
let Some(pairs) = crate::camera_preview::profile_pairs(&raw, bytes) else { return Ok(None) };
let Some(pairs) = crate::camera_preview::profile_pairs(&raw, bytes, lens.as_ref()) else { return Ok(None) };
let step = pairs.len().div_ceil(PAIRS_PER_FILE).max(1);
let entry = self.models.entry(model.to_owned()).or_default();
entry.0 += 1;
+25
View File
@@ -240,6 +240,10 @@ fn fill_missing_meta(id: lightcraft_catalog::PhotoId, ph: &lightcraft_catalog::P
m.aperture = m.aperture.or(src.aperture);
m.iso = m.iso.or(src.iso);
m.gps = m.gps.or(src.gps);
// the camera's focus point: photos imported before it was read get it here
if m.focus.is_none() {
m.focus = src.focus.clone();
}
let mut ops = Vec::new();
if m != before {
ops.push(Op::SetMeta { id, meta: Box::new(m) });
@@ -441,6 +445,27 @@ mod tests {
assert!(fill_missing_meta(PhotoId(1), &ph, &info).is_empty());
}
/// Reload from Disk gives a photo imported before focus points were read its camera focus point,
/// and never replaces one it has.
#[test]
fn reload_fills_a_missing_focus_point() {
let fp = |x: f64| lightcraft_meta::FocusPoint {
rect: lightcraft_meta::Rect { x0: x, y0: 0.4, x1: x + 0.02, y1: 0.42 },
mode: Some("AF-C".into()),
area: None,
tracking: None,
};
let mut ph = Photo::new(PhotoId(1), Source::File { path: "/x.arw".into() }, "x.arw", "ARW", 8640, 5760, "2026-10-06T00:00:00");
ph.captured = Some("2026-10-04T08:16:11".into());
let mut info = crate::media::ProbeInfo { captured: ph.captured.clone(), ..Default::default() };
info.meta.focus = Some(fp(0.3));
let ops = fill_missing_meta(PhotoId(1), &ph, &info);
let Some(Op::SetMeta { meta, .. }) = ops.first() else { panic!("{ops:?}") };
assert_eq!(meta.focus, Some(fp(0.3)));
ph.meta.focus = Some(fp(0.1));
assert!(fill_missing_meta(PhotoId(1), &ph, &info).is_empty(), "a known focus point is kept");
}
fn lens() -> lightcraft_develop::EmbeddedLens {
lightcraft_develop::EmbeddedLens {
warp: Some(lightcraft_develop::EmbeddedWarp { planes: [[1.0, -0.1, 0.03, 0.0, 0.0, 0.0]; 3], ..Default::default() }),
+1 -1
View File
@@ -16,7 +16,7 @@ const SAME: f32 = 0.93;
fn secs(iso: &str) -> Option<i64> {
let d = lightcraft_meta::DateTime::parse_iso(iso)?;
Some(((d.year as i64 * 372 + d.month as i64 * 31 + d.day as i64) * 24 + d.hour as i64) * 3600 + d.minute as i64 * 60 + d.second as i64)
Some(d.unix_seconds())
}
fn analyze(s: &mut Session, p: &Value) -> Result<Value> {
+147 -11
View File
@@ -17,7 +17,7 @@ use lightcraft_faces::yunet::{Detector, Face, Options};
use lightcraft_meta::{Region, RegionKind};
use serde_json::{Value, json};
use super::{CommandSpec, bad, cmd, has_selection};
use super::{CommandSpec, always, bad, cmd, has_selection};
use crate::{AiModel, EngineError, Result, Session};
const C: &str = "faces.detect";
@@ -54,10 +54,12 @@ impl NoDetector {
}
}
/// The YuNet loaded last, and from where.
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
/// The installed YuNet, loaded once and kept (a new one is loaded when the model folder changes).
/// (also used by the background scan, which aligns faces by the detector's landmarks and finds faces in photos that have none)
pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, NoDetector> {
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
let home = s
.face_models_dir
.as_deref()
@@ -82,10 +84,133 @@ pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, NoDete
let d = Arc::new(
Detector::new(&bytes).map_err(|e| NoDetector::Unusable(format!("{e}: remove the detector in Settings > Faces and download it again")))?,
);
#[cfg(not(target_arch = "wasm32"))]
{
let net = d.clone();
offer_gpu(s, "detection", move |gpu| net.accelerate(gpu, true));
}
*loaded = Some((path, d.clone()));
Ok(d)
}
/// Where a face model runs, for Settings ▸ Faces and `faces.gpu`.
#[derive(Clone, Debug, Default, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GpuState {
/// The background check is still running.
pub checking: bool,
/// The GPU it runs on; `None` when it runs on the CPU.
pub device: Option<String>,
/// Why it runs on the CPU.
pub why: Option<String>,
}
/// Each model's [`GpuState`], by what it does ("detection", "recognition").
static GPU_STATE: Mutex<Vec<(&'static str, GpuState)>> = Mutex::new(Vec::new());
pub(crate) fn gpu_state(what: &str) -> Option<GpuState> {
GPU_STATE.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(w, _)| *w == what).map(|(_, st)| st.clone())
}
fn set_gpu_state(what: &'static str, st: GpuState) {
let mut all = GPU_STATE.lock().unwrap_or_else(PoisonError::into_inner);
match all.iter_mut().find(|(w, _)| *w == what) {
Some((_, old)) => *old = st,
None => all.push((what, st)),
}
}
/// The user chose the CPU for face models (Settings ▸ Faces).
pub(crate) fn cpu_only(s: &Session) -> bool {
s.face_models_dir.as_deref().is_some_and(|d| super::face_models::read_settings(d).cpu_only)
}
/// Check in the background whether the GPU runs a face model faster than the CPU and gives its results; the model
/// runs on the CPU meanwhile, and stays there when the GPU is off, missing, wrong or slower, or the user chose the CPU.
/// The check shows in the activity stack while it runs.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn offer_gpu(
s: &Session,
what: &'static str,
attach: impl FnOnce(Arc<dyn lightcraft_faces::net::Accelerator>) -> std::result::Result<(), String> + Send + 'static,
) {
if cpu_only(s) {
set_gpu_state(what, GpuState { why: Some("the CPU is chosen in Settings".into()), ..GpuState::default() });
return;
}
let task = s.activity.start("facesGpu", "Checking the GPU for faces", crate::activity::Cancel::No);
task.detail(what);
set_gpu_state(what, GpuState { checking: true, ..GpuState::default() });
let spawned = std::thread::Builder::new().name(format!("faces-gpu-{what}")).spawn(move || {
let _task = task;
let gpu = match lightcraft_gpu::faces::accelerator() {
Ok(gpu) => gpu,
Err(why) => {
log::info!("faces: {what} runs on the CPU ({why})");
return set_gpu_state(what, GpuState { why: Some(why), ..GpuState::default() });
}
};
let device = lightcraft_faces::net::Accelerator::device(&*gpu);
match attach(gpu) {
Ok(()) => {
log::info!("faces: {what} runs on {device}");
set_gpu_state(what, GpuState { device: Some(device), ..GpuState::default() });
}
Err(why) => {
log::info!("faces: {what} stays on the CPU: {why}");
set_gpu_state(what, GpuState { why: Some(why), ..GpuState::default() });
}
}
});
if let Err(e) = spawned {
log::warn!("faces: could not check the GPU for {what}: {e}");
set_gpu_state(what, GpuState { why: Some(format!("could not check the GPU: {e}")), ..GpuState::default() });
}
}
/// `faces.gpu`: where the face models run, and the switch that keeps them on the CPU.
fn gpu(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.gpu";
if let Some(cpu) = p.get("cpuOnly") {
let cpu = cpu.as_bool().ok_or_else(|| bad(C, "`cpuOnly` is true or false"))?;
let dir = s.face_models_dir.clone().ok_or_else(|| bad(C, "this build has nowhere to keep face models (the desktop app does)"))?;
super::face_models::set_cpu_only(&dir, cpu)?;
let detector = LOADED.lock().unwrap_or_else(PoisonError::into_inner).as_ref().map(|(_, d)| d.clone());
#[cfg(not(target_arch = "wasm32"))]
let embedder = s.faces.embedder.as_ref().map(|(_, e)| e.clone());
#[cfg(target_arch = "wasm32")]
let embedder: Option<Arc<lightcraft_faces::runtime::Embedder>> = None;
if cpu {
for (what, net) in [("detection", detector.as_ref().map(|d| d.net())), ("recognition", embedder.as_ref().map(|e| e.net()))] {
if let Some(net) = net {
net.detach();
}
set_gpu_state(what, GpuState { why: Some("the CPU is chosen in Settings".into()), ..GpuState::default() });
}
} else {
#[cfg(not(target_arch = "wasm32"))]
{
if let Some(d) = detector {
offer_gpu(s, "detection", move |gpu| d.accelerate(gpu, true));
}
if let Some(e) = embedder {
offer_gpu(s, "recognition", move |gpu| e.accelerate(gpu, true).map_err(|e| e.to_string()));
}
}
}
}
Ok(gpu_json(s))
}
/// What `faces.gpu` and `faces.models.list` say about the GPU.
pub(crate) fn gpu_json(s: &Session) -> Value {
json!({
"cpuOnly": cpu_only(s),
"detection": gpu_state("detection"),
"recognition": gpu_state("recognition"),
})
}
fn face_json(f: &Face) -> Value {
json!({
"rect": {"x0": f.x0, "y0": f.y0, "x1": f.x1, "y1": f.y1},
@@ -173,15 +298,26 @@ fn detect(s: &mut Session, p: &Value) -> Result<Value> {
}
pub fn specs() -> Vec<CommandSpec> {
vec![cmd!(
"faces.detect",
"Detect Faces",
["Photo"],
None,
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
has_selection,
detect
)]
vec![
cmd!(
"faces.detect",
"Detect Faces",
["Photo"],
None,
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
has_selection,
detect
),
cmd!(
query "faces.gpu",
"Face Models on the GPU",
[],
None,
"{cpuOnly?: bool} → {cpuOnly, detection, recognition}: where the face models run, each as {checking, device, why} (null until it is loaded). The GPU is used only after a background check shows it gives the CPU's results faster; `cpuOnly: true` keeps every face model on the CPU (Settings ▸ Faces), `false` checks the GPU again",
always,
gpu
),
]
}
#[cfg(test)]
+12
View File
@@ -27,6 +27,9 @@ pub(super) struct FaceSettings {
pub(super) enabled: bool,
/// The installed recogniser in use.
pub(super) embedder: Option<String>,
/// Keep every face model on the CPU, even when the GPU would be faster.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub(super) cpu_only: bool,
}
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
@@ -57,6 +60,13 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
})
}
/// Keep the face models on the CPU (or let them use the GPU again).
pub(super) fn set_cpu_only(dir: &Path, on: bool) -> Result<()> {
let mut st = read_settings(dir);
st.cpu_only = on;
write_settings(dir, &st)
}
fn write_settings(dir: &Path, st: &FaceSettings) -> Result<()> {
std::fs::create_dir_all(dir).map_err(|e| fail("could not create the face models folder", e))?;
write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| fail("settings", e))?)
@@ -195,6 +205,8 @@ fn list(s: &mut Session, _: &Value) -> Result<Value> {
"dir": dir.as_ref().map(|d| d.display().to_string()),
"enabled": settings.enabled,
"embedder": settings.embedder,
// where the models run: {cpuOnly, detection, recognition} as `faces.gpu` gives it
"gpu": super::face_detect::gpu_json(s),
// Native builds always have the pure-Rust runner; model files/background scanning remain native-only.
"runtime": cfg!(not(target_arch = "wasm32")),
"models": models,
+5
View File
@@ -321,6 +321,8 @@ mod imp {
return Ok(e.clone());
}
let embedder = Arc::new(Embedder::load(&dir.join(&id).join("model.onnx"), &installed.manifest).map_err(|e| bad(cmd, e.to_string()))?);
let net = embedder.clone();
super::super::face_detect::offer_gpu(s, "recognition", move |gpu| net.accelerate(gpu, true).map_err(|e| e.to_string()));
let dim = match installed.manifest.output {
lightcraft_faces::OutputSpec::Embedding { dim } => dim as usize,
_ => return Err(bad(cmd, "the chosen model is not a recognition model")),
@@ -672,6 +674,9 @@ mod imp {
if let Some(w) = &s.faces.worker {
w.allow(target, pace);
}
// the GPU is held back the same way: a light pace leaves it mostly free for the window's own renders
let (runs, share) = crate::faces_worker::gpu_pace(pace);
lightcraft_gpu::faces::set_pace(runs, share);
while target > 0 && s.faces.in_flight.len() < target + PREFETCH {
let Some(id) = s.faces.queue.pop() else { break };
let Some(job) = prepare(s, id, true) else { continue };
+50
View File
@@ -0,0 +1,50 @@
//! Focus points for photos already in the library: `photo.readFocusPoints` (Photo ▸ Read Focus Points) and the
//! per-frame `focus.pump` that keeps the background job going. The job itself is `crate::focus_scan`.
use serde_json::Value;
use super::{CommandSpec, cmd};
use crate::{Result, Session};
#[cfg(not(target_arch = "wasm32"))]
fn read_points(s: &mut Session, p: &Value) -> Result<Value> {
crate::focus_scan::read_points(s, p)
}
#[cfg(target_arch = "wasm32")]
fn read_points(_: &mut Session, _: &Value) -> Result<Value> {
Err(super::bad("photo.readFocusPoints", "reading focus points needs the photos' files on disk"))
}
#[cfg(not(target_arch = "wasm32"))]
fn pump(s: &mut Session, p: &Value) -> Result<Value> {
crate::focus_scan::pump(s, p)
}
#[cfg(target_arch = "wasm32")]
fn pump(_: &mut Session, _: &Value) -> Result<Value> {
Ok(serde_json::json!({"active": false}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(
"photo.readFocusPoints",
"Read Focus Points",
["Photo"],
None,
"{ids?, wait?: false} → {queued, running, last?: {total, done, found, none, unavailable, bytesRead, seconds, cancelled}} — read the camera's focus point for photos imported before it was read (Sony α1 and A7S III raws and JPEGs), from the few KB of each file that hold it, on a background job with a row in the activity stack. Without `ids`, every photo in the library not looked at yet; with `ids`, those photos, looked at again. Photos that have a point keep it. No undo step and no sidecar write. `wait: true` returns when all are read (scripts)",
super::always,
read_points
),
cmd!(
query "focus.pump",
"Read Focus Points in the Background",
[],
None,
"{pace?: pause | light | normal} → {active, running?: {done, total, found}, last?, lookedAt} — cheap to call every frame (the app does): takes in what the background focus-point reader has read and commits it in batches; reads nothing new while paused and rests between files when light. Starts the reader by itself once per library opening when photos are left to look at",
super::always,
pump
),
]
}
+1 -1
View File
@@ -1488,7 +1488,7 @@ pub fn specs() -> Vec<CommandSpec> {
always, |_, p| {
let path = str_param(p,"path").ok_or_else(|| bad("library.inspectLightroom","missing path"))?;
let data = crate::lightroom_catalog::read(std::path::Path::new(path)).map_err(|e| bad("library.inspectLightroom",e))?;
Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p|&p.path).collect::<Vec<_>>(),"warnings":data.warnings}))
Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !p.original_is_file()).map(|p|&p.path).collect::<Vec<_>>(),"warnings":data.warnings}))
}),
cmd!(
"library.importLightroom",
@@ -77,6 +77,7 @@ fn relative_ratings_and_history_use_auto_write() {
std::fs::create_dir_all(&dir).unwrap();
// Metadata-only fixtures: no import, image decoding or model/service dependency.
for id in 1..=4 {
std::fs::write(dir.join(format!("{id}.jpg")), b"original").unwrap();
s.catalog
.apply(Op::SetFile {
id: PhotoId(id),
+7 -2
View File
@@ -9,8 +9,13 @@ use crate::Result;
fn rename_args(s: &crate::Session, p: &Value, c: &str) -> Result<(Vec<lightcraft_catalog::PhotoId>, String, usize)> {
let template = str_param(p, "template").ok_or_else(|| bad(c, "missing `template`"))?.to_string();
let start = p.get("start").and_then(Value::as_u64).unwrap_or(1) as usize;
Ok((s.targets(p)?, template, start))
let start = p.get("start").and_then(Value::as_u64).unwrap_or(1);
let ids = s.targets(p)?;
// every number of the sequence `start..start + n` must fit
let last = start.checked_add((ids.len() as u64).saturating_sub(1));
let start = last.and_then(|l| usize::try_from(l).ok()).and_then(|_| usize::try_from(start).ok());
let start = start.ok_or_else(|| bad(c, "start out of range"))?;
Ok((ids, template, start))
}
/// A named set of colour-label names (red, yellow, green, blue, purple; empty = the colour's name).
+80 -4
View File
@@ -150,6 +150,55 @@ fn masks_edit(s: &mut Session, c: &str, label: &str, f: impl FnOnce(&mut Vec<Mas
Ok(json!({"activeMask": active}))
}
/// Most pieces a tapered ⇧-click segment is cut into.
const MAX_TAPER_STEPS: usize = 64;
/// The strokes a ⇧-click adds: `to` drawn on from point `from`, the last point of stroke `last` (a straight segment
/// to `to`'s first point, then the rest of `to`). When the brush's size, feather or flow changed since `last`, the
/// segment is cut into pieces whose values step from `last`'s to `to`'s, so the line tapers; strokes combine by their
/// maximum, so the pieces join without building up.
fn connected_strokes(from: Point, last: &BrushStroke, to: BrushStroke) -> Vec<BrushStroke> {
let Some(&first) = to.points.first() else { return vec![to] };
let same = |a: f64, b: f64| (a - b).abs() <= 1e-9;
// pen pressure along the line: the last dab's at its start, the click's at its end (none when neither has any)
let pressured = !last.pressure.is_empty() || !to.pressure.is_empty();
let (p_from, p_to) = (last.pressure_at(0), to.pressure_at(0));
if same(last.size, to.size) && same(last.feather, to.feather) && same(last.flow, to.flow) {
let mut points = Vec::with_capacity(to.points.len() + 1);
points.push(from);
points.extend(to.points.iter().copied());
let pressure = if pressured { std::iter::once(p_from).chain((0..to.points.len()).map(|i| to.pressure_at(i))).collect() } else { Vec::new() };
return vec![BrushStroke { points, pressure, ..to }];
}
// about 5% of size, or 2 points of feather or flow, per piece
let ratio = if last.size > 0.0 && to.size > 0.0 { (to.size / last.size).ln().abs() / 1.05f64.ln() } else { 0.0 };
let needed = [ratio, (to.feather - last.feather).abs() / 2.0, (to.flow - last.flow).abs() / 2.0]
.into_iter()
.filter(|v| v.is_finite())
.fold(1.0f64, f64::max);
let steps = (needed.ceil() as usize).clamp(1, MAX_TAPER_STEPS);
let lerp = |a: f64, b: f64, t: f64| a + (b - a) * t;
let at = |t: f64| Point::new(lerp(from.x, first.x, t), lerp(from.y, first.y, t));
let mut out: Vec<BrushStroke> = (0..steps)
.map(|i| {
let (t0, t1) = (i as f64 / steps as f64, (i + 1) as f64 / steps as f64);
let p = |t: f64| (f64::from(p_from) + (f64::from(p_to) - f64::from(p_from)) * t) as f32;
BrushStroke {
points: vec![at(t0), at(t1)],
size: lerp(last.size, to.size, t1),
feather: lerp(last.feather, to.feather, t1),
flow: lerp(last.flow, to.flow, t1),
pressure: if pressured { vec![p(t0), p(t1)] } else { Vec::new() },
..to.clone()
}
})
.collect();
if to.points.len() > 1 {
out.push(to);
}
out
}
fn mask_id(p: &Value, active: Option<u32>, c: &str) -> Result<u32> {
p.get("id").and_then(Value::as_u64).map(|v| v as u32).or(active).ok_or_else(|| bad(c, "no mask selected (give `id`)"))
}
@@ -233,7 +282,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Brush Stroke",
[],
None,
"{id?: maskId, points: [[x,y],…] normalized, size?: fraction of long edge (0.03), feather?, flow?, density?, erase?: bool, autoMask?: bool, pressure?: [p,…] pen pressure 0..1 per point (scales each dab's size and flow; omitted: full)}",
"{id?: maskId, points: [[x,y],…] normalized, size?: fraction of long edge (0.03), feather?, flow?, density?, erase?: bool, autoMask?: bool, pressure?: [p,…] pen pressure 0..1 per point (scales each dab's size and flow; omitted: full), connect?: bool — start from the last point of the mask's last stroke (⇧-click), size / feather / flow tapering from that stroke's to these} → {activeMask, connected}",
has_active,
|s, p| {
let pts: Vec<Point> = p
@@ -255,10 +304,12 @@ pub fn specs() -> Vec<CommandSpec> {
auto_mask: bool_or(p, "autoMask", false),
pressure: pressure_list(p.get("pressure"), pts_len)?,
};
let connect = bool_or(p, "connect", false);
let mut connected = false;
let next = s.active().and_then(|id| s.develop_of(id)).map(|d| d.next_mask_id()).unwrap_or(1);
let want = p.get("id").and_then(Value::as_u64).map(|v| v as u32).or(s.active_mask);
let adjust = super::mask_effects::new_mask_adjust(s, "brush");
masks_edit(s, "mask.brushStroke", "Brush", |masks, active| {
let mut out = masks_edit(s, "mask.brushStroke", "Brush", |masks, active| {
let i = match want.and_then(|m| masks.iter().position(|x| x.id == m)) {
Some(i) => i,
None => {
@@ -271,7 +322,28 @@ pub fn specs() -> Vec<CommandSpec> {
if let Some(MaskComponent { shape: MaskShape::Brush { strokes }, .. }) =
m.components.iter_mut().rev().find(|c| matches!(c.shape, MaskShape::Brush { .. }))
{
strokes.push(stroke);
// the previous stroke's end and its brush values (not its points)
let anchor = strokes.last().filter(|_| connect).and_then(|l| {
Some((
*l.points.last()?,
BrushStroke {
points: Vec::new(),
size: l.size,
feather: l.feather,
flow: l.flow,
// the pressure the last dab was painted with, when the stroke recorded any
pressure: if l.pressure.is_empty() { Vec::new() } else { vec![l.pressure_at(l.points.len().saturating_sub(1))] },
..BrushStroke::default()
},
))
});
match anchor {
Some((from, last)) => {
strokes.extend(connected_strokes(from, &last, stroke));
connected = true;
}
None => strokes.push(stroke),
}
} else {
m.components.push(MaskComponent {
name: None,
@@ -281,7 +353,11 @@ pub fn specs() -> Vec<CommandSpec> {
});
}
Ok(())
})
})?;
if let Some(o) = out.as_object_mut() {
o.insert("connected".into(), json!(connected));
}
Ok(out)
}
),
cmd!("mask.update", "Update Mask Shape", [], None, "{id?, component?: index (0), shape: MaskShape JSON}", has_active, |s, p| {
+2
View File
@@ -19,6 +19,7 @@ pub(crate) mod face_detect;
mod face_models;
mod face_recognize;
pub mod filters;
mod focus;
pub mod geocode;
pub mod keywords;
pub mod library;
@@ -174,6 +175,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(face_models::specs());
v.extend(face_detect::specs());
v.extend(face_recognize::specs());
v.extend(focus::specs());
v.extend(activity::specs());
v
})
+1 -1
View File
@@ -324,7 +324,7 @@ pub fn specs() -> Vec<CommandSpec> {
"{ids?} → {settings: {layout, image, page, guides, job}, papers, pageWidth, pageHeight, printable, pageCount, cellsPerPage, gridCells?, overlaps}; lengths are inches",
always, get),
cmd!(query "print.set", "Set Print Settings", [], None,
"{layout?: {style: singleImage|contactSheet|picturePackage|customPackage, page: {paper, customWidth, customHeight, orientation: portrait|landscape, margins: {top,right,bottom,left}}, grid: {columns, rows, spacingX, spacingY, keepSquare}, spacing}, image?: {zoomToFill, rotateToFit, borderEnabled, borderWidth, borderColor: [r,g,b], strokeEnabled, strokeWidthPt, strokeColor}, page?: {background: [r,g,b]}, guides?: {showRulers, units: in|cm|mm|pt, snap: off|cells|grid, gridSize, showMargins, showBleed, showPageGrid, showCells, showDimensions}, job?: {ppi: 72..720, color?: {profile: {kind: managed, space: srgb|adobeRgb} | {kind: icc, path}, intent: perceptual|relative, blackPointCompensation, brightness: -100..100, contrast: -100..100, sharpen: off|low|standard|high, media: matte|glossy, sixteenBit}}} — merges into the current print; cells and pages change with print.cell.* / print.layout.*; → print.get",
"{layout?: {style: singleImage|contactSheet|picturePackage|customPackage, page: {paper, customWidth, customHeight, orientation: portrait|landscape, margins: {top,right,bottom,left}}, grid: {columns, rows, spacingX, spacingY, keepSquare}, spacing}, image?: {zoomToFill, rotateToFit, borderEnabled, borderWidth, borderColor: [r,g,b], strokeEnabled, strokeWidthPt, strokeColor}, page?: {background: [r,g,b], pageNumbers, pageInfo, cropMarks} (Page Options: single image and contact sheet only), guides?: {showRulers, units: in|cm|mm|pt, snap: off|cells|grid, gridSize, showMargins, showBleed, showPageGrid, showCells, showDimensions}, job?: {ppi: 72..720, color?: {profile: {kind: managed, space: srgb|adobeRgb} | {kind: icc, path}, intent: perceptual|relative, blackPointCompensation, brightness: -100..100, contrast: -100..100, sharpen: off|low|standard|high, media: matte|glossy, sixteenBit}}} — merges into the current print; cells and pages change with print.cell.* / print.layout.*; → print.get",
always, set),
cmd!(query "print.reset", "Reset Print Settings", [], None, "{} → print.get", always, reset),
cmd!(query "print.cell.add", "Add Print Cell", [], None,
+43 -1
View File
@@ -99,6 +99,21 @@ fn adjust(c: &XmpValue) -> Value {
Value::Object(a)
}
/// The correction's local sliders we have no counterpart for (e.g. `LocalGrain`), when set:
/// reported, never dropped silently.
fn unmapped_local(c: &XmpValue) -> Vec<String> {
let XmpValue::Struct(m) = c else { return Vec::new() };
m.keys()
.filter_map(|k| {
let field = k.strip_prefix("crs:").filter(|f| f.starts_with("Local"))?;
// 100 = "curve saturation unchanged", written with every correction
let neutral = if field == "LocalCurveRefineSaturation" { 100.0 } else { 0.0 };
let set = num(c, k).is_some_and(|v| v != neutral);
(set && !LOCAL.iter().any(|(crs, ..)| *crs == field)).then(|| field.to_string())
})
.collect()
}
/// Perceptual lightness 0..1 (how range masks store luminance) → our range scale: log
/// luminance on −8..+4 EV mapped to 0..1.
fn lightness_to_range(l: f64) -> f64 {
@@ -214,8 +229,12 @@ fn component(m: &XmpValue, container: &str, aspect: f64) -> Result<Value, String
Some("2") => {
let q: Vec<f64> = text(r, "crs:LumRange").unwrap_or("0 0 1 1").split_whitespace().filter_map(|v| v.parse().ok()).collect();
let [a, b, c, d] = q[..] else { return Err("luminance range".into()) };
// full strength at white (1, the top of their scale) takes in everything brighter: ours
// goes on above scene white, so that end stays open instead of cutting off at 0 EV
let open_top = c >= 1.0;
let (a, b, c, d) = (lightness_to_range(a), lightness_to_range(b), lightness_to_range(c), lightness_to_range(d));
json!({"kind": "luminanceRange", "lo": b, "hi": c, "lo_feather": (b - a).max(0.0), "hi_feather": (d - c).max(0.0)})
let (hi, hi_feather) = if open_top { (1.0, 0.0) } else { (c, (d - c).max(0.0)) };
json!({"kind": "luminanceRange", "lo": b, "hi": hi, "lo_feather": (b - a).max(0.0), "hi_feather": hi_feather})
}
Some("3") => {
let q: Vec<f64> = text(r, "crs:DepthRange").unwrap_or("0 0 1 1").split_whitespace().filter_map(|v| v.parse().ok()).collect();
@@ -252,6 +271,7 @@ pub fn masks(values: &Values, aspect: f64) -> (Vec<Value>, Vec<String>) {
if parts.is_empty() {
continue;
}
skipped.extend(unmapped_local(c));
// brush dabs of one correction are one brush component
let mut merged: Vec<Value> = Vec::new();
for p in parts {
@@ -506,6 +526,28 @@ mod tests {
assert_ne!(render(&DevelopSettings { light: s.light, ..Default::default() }), render(&s), "the masks change the picture");
}
#[test]
fn range_to_white_stays_open_and_unmapped_local_sliders_are_reported() {
// written for this test: a highlights range (full from 0.85 up to white) carrying a local slider we lack
let x = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<rdf:Description rdf:about="" xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/">
<crs:MaskGroupBasedCorrections><rdf:Seq>
<rdf:li><rdf:Description crs:What="Correction" crs:LocalClarity2012="-0.3" crs:LocalGrain="0.4" crs:LocalCurveRefineSaturation="100">
<crs:CorrectionMasks><rdf:Seq>
<rdf:li crs:What="Mask/RangeMask"><crs:CorrectionRangeMask crs:Type="2" crs:LumRange="0.6 0.85 1 1"/></rdf:li>
</rdf:Seq></crs:CorrectionMasks>
</rdf:Description></rdf:li>
</rdf:Seq></crs:MaskGroupBasedCorrections>
</rdf:Description></rdf:RDF></x:xmpmeta>"#;
let d = lightcraft_meta::parse_xmp(x).unwrap();
let (partial, unmapped) = crate::crs::to_partial_report(&d.properties, Some(&d.values), crate::crs::Target::Any, 1.5);
let s = DevelopSettings::default().merged(&partial).expect("valid settings");
let MaskShape::LuminanceRange { hi, hi_feather, .. } = &s.masks[0].components[0].shape else { panic!() };
assert_eq!((*hi, *hi_feather), (1.0, 0.0), "brighter than scene white is still in the range");
assert_eq!(s.masks[0].adjust.clarity, -30.0);
assert_eq!(unmapped, ["Mask: LocalGrain"], "a refine saturation of 100 is neutral, not reported");
}
#[test]
fn lrtemplate_corrections() {
let t = r#"s = { title = "Grad", value = { settings = { Exposure2012 = 0.1, GradientBasedCorrections = {
+10
View File
@@ -30,6 +30,16 @@ pub fn load(s: &mut Session) {
creator: "LightCraft Demo".into(),
..Default::default()
};
// one demo photo carries a camera focus point (as a Sony raw would), on the bloom's centre, so
// View ▸ Show Focus Point and Zoom to Focus Point can be tried without a camera file
if sc.name == "Cosmos bloom" {
p.meta.focus = Some(lightcraft_meta::FocusPoint {
rect: lightcraft_geom::Rect { x0: 0.42, y0: 0.433, x1: 0.50, y1: 0.487 },
mode: Some("AF-S".into()),
area: Some("Flexible Spot".into()),
tracking: None,
});
}
p.rating = [0, 3, 4, 5, 2, 0, 4, 3][sc.id as usize % 8];
p.flag = match sc.id % 6 {
1 => Flag::Pick,
+63
View File
@@ -0,0 +1,63 @@
//! Paths stored in a file LightCraft reads but the user did not choose path by path, such as the
//! originals a Lightroom catalog lists.
//!
//! Such a path reaches the file system only through [`resolve`]: a relative path is taken from the
//! document's folder, and only local files, or files on the network share the document itself is
//! on, are looked at (see `craft_docpath`). A path that is not followed is handled like a missing
//! file.
use std::fmt;
use std::path::{Path, PathBuf};
pub use craft_docpath::Refused;
/// Why a stored path is not followed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NotFollowed {
/// The path is on another computer (other than the document's share), names a device, is a
/// URL, or is not a usable path.
Refused(Refused),
/// The path is not an absolute path on this computer, such as a Windows drive path read on
/// macOS or Linux.
NotHere,
}
impl fmt::Display for NotFollowed {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Refused(r) => r.fmt(f),
Self::NotHere => f.write_str("the path is not a path on this computer"),
}
}
}
impl std::error::Error for NotFollowed {}
/// The file to look at for `stored`, a path or `file:` URL stored in the document at `document`,
/// or why it is not followed. A relative `document` is taken from the working directory, where it
/// was opened from.
pub fn resolve(stored: &str, document: Option<&Path>) -> Result<PathBuf, NotFollowed> {
let document = document.map(|d| if d.is_relative() { std::path::absolute(d).unwrap_or_else(|_| d.to_path_buf()) } else { d.to_path_buf() });
let path = craft_docpath::resolve(stored, document.as_deref()).map_err(NotFollowed::Refused)?;
// A path that is still relative would be taken from the working directory, which the
// document does not choose.
if path.is_relative() {
return Err(NotFollowed::NotHere);
}
Ok(path)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn relative_paths_are_taken_from_the_document_folder_and_other_computers_are_refused() {
let doc = std::env::temp_dir().join("lrcat").join("catalog.lrcat");
assert_eq!(resolve("Photos/a.jpg", Some(&doc)), Ok(std::env::temp_dir().join("lrcat").join("Photos").join("a.jpg")));
assert_eq!(resolve("a.jpg", None), Err(NotFollowed::NotHere));
assert_eq!(resolve(r"\\192.0.2.1\share\x.jpg", Some(&doc)), Err(NotFollowed::Refused(Refused::Network)));
assert_eq!(resolve("file://192.0.2.1/s/x.jpg", Some(&doc)), Err(NotFollowed::Refused(Refused::Network)));
assert_eq!(resolve(r"\\.\COM1", Some(&doc)), Err(NotFollowed::Refused(Refused::Device)));
}
}
+46 -1
View File
@@ -1310,7 +1310,15 @@ pub fn encode_with_metadata(img: &Rgba8, o: &ExportOptions, meta: Option<&Metada
let exif = meta.map(lightcraft_meta::write_exif);
let xmp = meta.map(|m| lightcraft_meta::write_xmp(m, None));
let meta = EncodeMeta { icc: Some(&profile), exif: exif.as_deref(), xmp: xmp.as_deref(), ppi: Some(o.ppi) };
let e = EncodeImage::rgba8(&img);
// Opaque photo exports need no alpha channel (#1670). Keep real transparency for
// callers of this encoder, and leave unrelated formats on their existing path.
let rgb: Option<Vec<u8>> = (matches!(o.format, ExportFormat::Png | ExportFormat::Tiff | ExportFormat::Webp)
&& img.data.iter().all(|p| p.last() == Some(&255)))
.then(|| img.data.iter().flat_map(|p| p.iter().take(3).copied()).collect());
let e = match &rgb {
Some(data) => EncodeImage::new(img.width as u32, img.height as u32, 3, Samples::U8(data)),
None => EncodeImage::rgba8(&img),
};
let r = match o.format {
ExportFormat::Jpeg => {
// 4:4:4 normally; 4:2:0 when targeting a file size (much smaller at equal visual quality).
@@ -2281,6 +2289,43 @@ mod tests {
img
}
/// Issue #1670: removing a redundant alpha channel preserves samples and metadata.
#[test]
fn opaque_eight_bit_exports_are_rgb() {
let img = test_image();
let metadata = Metadata { copyright: Some("Synthetic export test".into()), ..Default::default() };
for format in [ExportFormat::Png, ExportFormat::Tiff, ExportFormat::Webp] {
for compression in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate, TiffCompression::PackBits] {
let options = ExportOptions { format, tiff_compression: compression, ..Default::default() };
let bytes = encode_with_metadata(&img, &options, Some(&metadata)).unwrap();
let decoded = lightcraft_codecs::decode(&bytes, Default::default()).unwrap();
assert!(!decoded.has_alpha, "{format:?}/{compression:?} must be RGB");
assert_eq!((decoded.width, decoded.height, decoded.bit_depth), (96, 64, 8));
assert_eq!(decoded.to_srgb8().data, img.data, "{format:?}/{compression:?} pixels");
assert_eq!(decoded.icc.as_deref(), Some(icc::write_named(NamedSpace::Srgb).as_slice()));
assert!(decoded.xmp.as_deref().unwrap_or_default().contains("Synthetic export test"));
if format == ExportFormat::Png {
assert_eq!(bytes[25], 2, "PNG IHDR declares RGB");
}
if format != ExportFormat::Tiff {
break;
}
}
}
}
#[test]
fn eight_bit_exports_preserve_real_alpha() {
let mut img = test_image();
img.data[0][3] = 128;
for format in [ExportFormat::Png, ExportFormat::Tiff, ExportFormat::Webp] {
let bytes = encode_image(&img, &ExportOptions { format, ..Default::default() }).unwrap();
let decoded = lightcraft_codecs::decode(&bytes, Default::default()).unwrap();
assert!(decoded.has_alpha, "{format:?} must retain real alpha");
assert_eq!(decoded.to_srgb8().data, img.data, "{format:?} samples");
}
}
#[test]
fn formats_have_magic() {
let img = test_image();
+93 -13
View File
@@ -167,6 +167,9 @@ impl Index {
self.entries.insert((photo, key), (emb.into(), view));
}
}
// a half-written last record (a crash while appending): written fresh at the next save, so later records are
// not appended after it and shifted
self.rewrite = !rest.is_empty();
}
fn header(&self) -> Vec<u8> {
@@ -201,12 +204,11 @@ impl Index {
return Ok(());
}
if self.file_matches && !self.rewrite {
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
let mut buf = Vec::new();
for (k, e) in &self.unsaved {
buf.extend(self.record(k, e));
}
f.write_all(&buf)?;
append(&path, &buf)?;
} else {
let mut buf = self.header();
for (k, e) in &self.entries {
@@ -229,6 +231,22 @@ impl Drop for Index {
}
}
/// Append `buf` to the file at `path`. When the write fails (a full disk), the file is cut back to its old length so
/// the part that got in does not shift what is appended later.
fn append(path: &Path, buf: &[u8]) -> std::io::Result<()> {
let mut f = std::fs::OpenOptions::new().append(true).open(path)?;
let old_len = f.metadata()?.len();
let written = f.write_all(buf);
if written.is_err() {
drop(f);
// an append-only handle may not truncate (Windows): cut through a writable one
if let Ok(w) = std::fs::OpenOptions::new().write(true).open(path) {
let _ = w.set_len(old_len);
}
}
written
}
const SCAN_MAGIC: &[u8; 4] = b"LCFS";
const SCAN_VERSION: u8 = 1;
/// The detector a search was made with: a record made by another one is ignored (a better detector may find more).
@@ -238,7 +256,7 @@ const SCAN_MAX_FILE: u64 = 64 << 20;
/// The photos the detector has already searched for faces, so each is searched once, however many times LightCraft
/// starts. Kept beside the library (`face-scanned.bin`: a header naming the detector, then photo ids, appended to).
#[derive(Default)]
/// The focus-point backfill keeps its own record the same way ([`Scanned::new`] with its magic and decoder tag).
pub(crate) struct Scanned {
set: std::collections::HashSet<u64>,
unsaved: Vec<u64>,
@@ -246,9 +264,23 @@ pub(crate) struct Scanned {
file_matches: bool,
/// A photo was taken out ([`Scanned::remove`]): the record is written fresh at the next save.
rewrite: bool,
magic: [u8; 4],
/// What made the record: a file made by anything else is started afresh.
tag: String,
}
impl Default for Scanned {
fn default() -> Self {
Scanned::new(*SCAN_MAGIC, SCAN_TAG)
}
}
impl Scanned {
/// An empty record (in memory until [`Scanned::reset`] gives it a path) whose file starts with `magic` and `tag`.
pub fn new(magic: [u8; 4], tag: &str) -> Scanned {
Scanned { set: Default::default(), unsaved: Vec::new(), path: None, file_matches: false, rewrite: false, magic, tag: tag.to_string() }
}
pub fn path(&self) -> Option<&Path> {
self.path.as_deref()
}
@@ -292,12 +324,13 @@ impl Scanned {
}
}
fn header() -> Vec<u8> {
fn header(&self) -> Vec<u8> {
let mut h = Vec::new();
h.extend_from_slice(SCAN_MAGIC);
h.extend_from_slice(&self.magic);
h.push(SCAN_VERSION);
h.extend_from_slice(&(SCAN_TAG.len() as u16).to_le_bytes());
h.extend_from_slice(SCAN_TAG.as_bytes());
let tag = self.tag.as_bytes().get(..usize::from(u16::MAX)).unwrap_or(self.tag.as_bytes());
h.extend_from_slice(&(tag.len() as u16).to_le_bytes());
h.extend_from_slice(tag);
h
}
@@ -307,11 +340,14 @@ impl Scanned {
return;
}
let Ok(bytes) = std::fs::read(path) else { return };
let head = Self::header();
let head = self.header();
let Some(rest) = bytes.strip_prefix(head.as_slice()) else { return };
self.file_matches = true;
// a half-written last id (a crash while appending) is ignored
self.set.extend(rest.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c)));
// a half-written last id (a crash while appending) is ignored, and the record is written fresh at the next save
// so later ids are not appended after it and shifted
let (ids, torn) = rest.as_chunks::<8>();
self.set.extend(ids.iter().map(|c| u64::from_le_bytes(*c)));
self.rewrite = !torn.is_empty();
}
/// Write what was added since the last save: appended when the file is this detector's, else the whole record
@@ -326,11 +362,10 @@ impl Scanned {
return Ok(());
}
if self.file_matches && !self.rewrite {
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
let buf: Vec<u8> = self.unsaved.iter().flat_map(|id| id.to_le_bytes()).collect();
f.write_all(&buf)?;
append(&path, &buf)?;
} else {
let mut buf = Self::header();
let mut buf = self.header();
buf.extend(self.set.iter().flat_map(|id| id.to_le_bytes()));
let part = path.with_extension("part");
std::fs::write(&part, &buf)?;
@@ -472,6 +507,51 @@ mod tests {
assert!(!t.contains(1) && t.contains(2) && t.contains(3));
let _ = std::fs::remove_dir_all(&dir);
}
/// A torn last record (a crash or a full disk while appending) does not shift the records appended after it
/// (issue #2129).
#[test]
fn records_appended_after_a_torn_tail_stay_aligned() {
let dir = std::env::temp_dir().join(format!("lc-faces-torn-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let (emb, scan) = (dir.join("face-embeddings.bin"), dir.join("face-scanned.bin"));
let junk = |p: &Path| {
let mut f = std::fs::OpenOptions::new().append(true).open(p).unwrap();
f.write_all(&[0x3f; 5]).unwrap();
};
let mut a = Index::default();
a.reset("m@abc", 3, Some(emb.clone()));
a.insert(7, &rect(0.1), vec![1.0, 0.0, 0.5]);
a.save().unwrap();
drop(a);
junk(&emb);
let mut b = Index::default();
b.reset("m@abc", 3, Some(emb.clone()));
b.insert(9, &rect(0.5), vec![0.5, 0.25, 0.125]);
b.save().unwrap();
drop(b);
let mut c = Index::default();
c.reset("m@abc", 3, Some(emb));
assert_eq!(c.len(), 2);
assert_eq!(c.get(7, &rect(0.1)).map(|e| e.to_vec()), Some(vec![1.0, 0.0, 0.5]));
assert_eq!(c.get(9, &rect(0.5)).map(|e| e.to_vec()), Some(vec![0.5, 0.25, 0.125]));
let mut s = Scanned::default();
s.reset(Some(scan.clone()));
s.insert(42);
s.save().unwrap();
drop(s);
junk(&scan);
let mut t = Scanned::default();
t.reset(Some(scan.clone()));
t.insert(43);
t.save().unwrap();
drop(t);
let mut u = Scanned::default();
u.reset(Some(scan));
assert!(u.contains(42) && u.contains(43) && u.len() == 2);
let _ = std::fs::remove_dir_all(&dir);
}
}
/// What the recognition commands keep between calls: the loaded model and the embeddings made with it.
+21 -7
View File
@@ -127,6 +127,17 @@ fn pool_threads_for(cores: usize, pace: Pace, cap: Option<usize>) -> usize {
cap.map_or(n, |c| n.min(c.max(1)))
}
/// How much of the GPU face models may take at `pace`: face model runs at once, and the share of the time it may be
/// busy. Paused or light: one run at a time, busy a quarter of the time (the user is working, and the window renders
/// on the same GPU); normal: two at once, busy most of the time; full: as fast as it goes.
pub(crate) fn gpu_pace(pace: Pace) -> (usize, f32) {
match pace {
Pace::Pause | Pace::Light => (1, 0.25),
Pace::Normal => (2, 0.75),
Pace::Full => (4, 1.0),
}
}
/// One photo's faces to embed (and, if `detect`, to find first).
pub(crate) struct Prepared {
/// `<model id>@<sha256>` the embeddings are for.
@@ -201,7 +212,8 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
let img = Rgb { data: &rgb, width: w, height: h };
let (iw, ih) = embedder.input_size();
let scaled = |f: &Face| f.landmarks.map(|(x, y)| (x * w as f32, y * h as f32));
let mut results: Vec<(Rect, Option<Vec<f32>>, Rect)> = todo
// every face's crop first, then all of them through the model at once (one GPU dispatch for a group photo)
let mut faces: Vec<(Rect, Option<Vec<u8>>, Rect)> = todo
.into_iter()
.map(|rect| {
let best = found.iter().map(|f| (overlap(&rect, f), f)).filter(|(o, _)| *o >= MATCH_IOU).max_by(|a, b| a.0.total_cmp(&b.0));
@@ -218,20 +230,18 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
ih,
),
};
let v = crop.and_then(|c| embedder.embed(&c).ok());
// shown by the detector's box where it found this face: every face then has the same tightness
let view = best.and_then(|(_, f)| face_rect(f)).unwrap_or(rect);
(rect, v, view)
(rect, crop, view)
})
.collect();
// a photo with no face regions: what the detector is sure of becomes its faces, embedded in the same pass
let new_faces = if detect {
detected.map(|faces| {
detected.map(|found| {
let mut rects = Vec::new();
for f in faces.iter().filter(|f| f.score >= REGION_SCORE) {
for f in found.iter().filter(|f| f.score >= REGION_SCORE) {
let Some(rect) = face_rect(f) else { continue };
let v = align_to_template(&img, &scaled(f), iw, ih).and_then(|c| embedder.embed(&c).ok());
results.push((rect, v, rect));
faces.push((rect, align_to_template(&img, &scaled(f), iw, ih), rect));
rects.push(rect);
}
rects
@@ -239,6 +249,10 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
} else {
None
};
let crops: Vec<&[u8]> = faces.iter().filter_map(|(_, c, _)| c.as_deref()).collect();
let mut embedded = embedder.embed_batch(&crops).into_iter();
let results: Vec<(Rect, Option<Vec<f32>>, Rect)> =
faces.into_iter().map(|(rect, crop, view)| (rect, crop.and_then(|_| embedded.next().and_then(Result::ok)), view)).collect();
(results, new_faces)
}));
let (results, found) = run.unwrap_or_else(|_| (fallback(rects), None));
+10 -4
View File
@@ -41,6 +41,7 @@ fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {
// paths from `lr:hierarchicalSubject`, names from `dc:subject` that aren't their levels
keywords: lightcraft_catalog::keywords::from_file(&m.keywords, &m.hierarchical_keywords),
regions: m.regions.clone(),
focus: m.focus.clone(),
};
(meta, m.capture_time.as_ref().map(|d| d.to_iso()))
}
@@ -423,13 +424,17 @@ fn load_bytes_now(
// (removed before the camera look's binned sensor proxy too, which needs an empty `OpcodeList3`).
let info = raw.info();
let lens = crate::lens_profiles::for_raw(&info, || lightcraft_meta::extract(&bytes));
// (the camera look still pairs the sensor with the camera JPEG through the warp the camera applied to it)
let lens_geometry = crate::camera_preview::lens_geometry(&raw.opcodes.list3);
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
// Which scale white balance is edited on, and the as-shot white on it, from the file's own colour tags or
// the camera's measured matrices (`Photo::relative_wb` decides the same from the catalog's camera)
let scale = WbScaleOf::new(raw.format, &raw.color, raw.wb_multipliers, &raw.metadata);
// the source's segmentation mattes (DNG semantic masks), read while the starting colour is fitted
let ((xy, t, camera_look), mattes) =
rayon::join(|| crate::camera_preview::starting_colour(&mut raw, &bytes, lens.as_ref()), || dng_mattes(&bytes, &info));
let ((xy, t, camera_look), mattes) = rayon::join(
|| crate::camera_preview::starting_colour(&mut raw, &bytes, lens_geometry.as_ref(), lens.as_ref()),
|| dng_mattes(&bytes, &info),
);
drop(bytes);
// Previews and thumbnails bin the mosaic straight to (about) the size they need; only
// larger levels (exports, 1:1) demosaic the whole sensor.
@@ -702,8 +707,9 @@ pub(crate) fn decode_raw_preview(
covering: Option<usize>,
) -> Option<lightcraft_codecs::Decoded> {
let jpeg = stand_in_image(bytes, covering)?;
// the colour space is a property of the JPEG; a TIFF stand-in is read by the TIFF reader
let is_jpeg = matches!(jpeg, std::borrow::Cow::Owned(_));
// the colour space is a property of the JPEG; a TIFF stand-in (the file itself, or a Hasselblad RGB preview
// wrapped as a TIFF) and a JPEG XL preview are read by their own decoders
let is_jpeg = jpeg.starts_with(&[0xff, 0xd8]);
match lightcraft_raw::embedded_preview_color_space(bytes).filter(|_| is_jpeg) {
Some(space) => {
let space = match space {
+432
View File
@@ -0,0 +1,432 @@
//! Reading the camera's focus point for photos already in the library (issue #1480).
//!
//! Import reads a photo's focus point ([`lightcraft_meta::focus`]); photos imported before that have none. This job
//! fills them in, cheaply enough to run over a whole library on a spinning disk: it opens only the photos from a body
//! whose focus data is decoded (by the camera the catalog stores, so nothing else is opened), and reads only the few
//! kilobytes of each file that hold the focus data ([`lightcraft_meta::focus::read_header`]: about 32 KB of a 75 MB α1
//! ARW), one file at a time on one worker thread.
//!
//! - **Commits** go in batches of [`COMMIT_EVERY`] photos (or every [`COMMIT_AFTER`]), as LightCraft's own bookkeeping
//! ([`Session::apply_system`]): no undo step, no sidecar write (the focus point is never written to XMP). Only
//! `meta.focus` changes, and only on a photo that still has none, still in the library at the same path.
//! - **"Looked, found nothing"** (manual focus, a centre-only Multi report) is kept beside the library in
//! `focus-scanned.bin` (photo ids, appended to, headed by the decoder's tag): it is cheap to append, never touches the
//! catalog, and a decoder that learns more bodies has another tag, so its record starts afresh and those photos are
//! looked at again. A photo whose file can't be read (offline drive, file gone or replaced mid-read) is not recorded:
//! it is tried again the next time the library opens.
//! - **Runs** off the UI thread with a row in the activity stack ("Reading focus points", n of N, ✕ stops it). The UI's
//! per-frame `focus.pump` gives it the pace: nothing new while the user drags, types or scrolls, a short rest between
//! files while they are around.
//! - **Starts** by itself once per library opening (from `focus.pump`) when photos are left to look at, and on demand
//! with `photo.readFocusPoints`.
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::mpsc;
use std::time::{Duration, Instant};
use lightcraft_catalog::{Op, PhotoId, Source};
use lightcraft_meta::FocusPoint;
use lightcraft_meta::focus::HeaderRead;
use serde_json::{Value, json};
use crate::activity::{Cancel, TaskGuard};
use crate::faces_index::Scanned;
use crate::{Result, Session};
/// The record of photos looked at with nothing found, beside the library.
pub(crate) const RECORD_FILE: &str = "focus-scanned.bin";
const RECORD_MAGIC: [u8; 4] = *b"LCFP";
/// Photos committed at once.
pub(crate) const COMMIT_EVERY: usize = 200;
/// What has been read is committed at least this often, so points appear while a long run goes on.
const COMMIT_AFTER: Duration = Duration::from_secs(3);
/// The rest between files while the user is around (`light` pace): their own reads of the same disk go first.
const LIGHT_REST: Duration = Duration::from_millis(40);
/// How often a paused worker looks at its pace again.
const PAUSE_POLL: Duration = Duration::from_millis(50);
/// Results taken in per pump (the rest wait for the next frame).
const MAX_DRAIN: usize = 5000;
/// The file extensions read (Sony bodies also shoot HEIF, which carries no maker note this decoder reads).
const EXTENSIONS: [&str; 3] = ["arw", "jpg", "jpeg"];
/// How hard the worker may work, from what the user is doing (`focus.pump`'s `pace`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum Pace {
Pause = 0,
Light = 1,
Normal = 2,
}
impl Pace {
pub fn parse(name: Option<&str>) -> Pace {
match name {
Some("pause") => Pace::Pause,
Some("light") => Pace::Light,
_ => Pace::Normal,
}
}
}
/// The focus point in the file at `path`, read from its header only.
pub fn read_focus(path: &str) -> std::result::Result<HeaderRead, String> {
use std::io::{Read, Seek, SeekFrom};
let mut file = std::fs::File::open(path).map_err(|e| format!("{path}: {e}"))?;
let len = file.metadata().map_err(|e| format!("{path}: {e}"))?.len();
lightcraft_meta::focus::read_header(len, &mut |off, buf| {
file.seek(SeekFrom::Start(off))?;
file.read_exact(buf)
})
.map_err(|e| format!("{path}: {e}"))
}
/// What the worker says about one photo.
enum Outcome {
/// The header was read: the point it holds, or none.
Read { id: PhotoId, focus: Option<FocusPoint>, bytes: u64 },
/// The file couldn't be read (offline, gone, changed under the read): tried again another time.
Unavailable,
}
/// What a run did (the last one is kept for `focus.status` and `photo.readFocusPoints {wait}`).
#[derive(Clone, Debug, Default, PartialEq)]
pub(crate) struct Summary {
pub total: u64,
pub done: u64,
pub found: u64,
pub none: u64,
pub unavailable: u64,
pub bytes: u64,
pub seconds: f64,
pub cancelled: bool,
}
impl Summary {
fn json(&self) -> Value {
json!({
"total": self.total,
"done": self.done,
"found": self.found,
"none": self.none,
"unavailable": self.unavailable,
"bytesRead": self.bytes,
"seconds": self.seconds,
"cancelled": self.cancelled,
})
}
}
struct Job {
identity: Arc<()>,
cancel: Arc<AtomicBool>,
pace: Arc<AtomicU8>,
rx: mpsc::Receiver<(Outcome, String)>,
/// Its row in the activity stack (dropping the job removes it).
guard: TaskGuard,
/// The path each photo was read at: a photo moved or relinked since is left for another run.
paths: HashMap<PhotoId, String>,
pending: Vec<(PhotoId, Option<FocusPoint>)>,
committed_at: Instant,
started: Instant,
summary: Summary,
}
impl Drop for Job {
fn drop(&mut self) {
// the worker stops before its next file (and when it can't hand a result over)
self.cancel.store(true, Ordering::Relaxed);
}
}
/// The backfill's state in the session.
pub(crate) struct FocusScan {
record: Scanned,
/// The library opening the record and the automatic start belong to.
library: Option<Arc<()>>,
auto_checked: bool,
job: Option<Job>,
last: Option<Summary>,
/// The pace the UI last gave (a job started between two pumps starts at it).
pace: Pace,
}
impl Default for FocusScan {
fn default() -> Self {
FocusScan {
record: Scanned::new(RECORD_MAGIC, &lightcraft_meta::focus::decoder_tag()),
library: None,
auto_checked: false,
job: None,
last: None,
pace: Pace::Normal,
}
}
}
/// Follow the session's library: another one (or none) drops a running job (its results belong to the old library and
/// are read again there next time) and loads that library's record.
fn follow_library(s: &mut Session) {
if s.focus_scan.library.as_ref().is_some_and(|l| Arc::ptr_eq(l, &s.library_identity)) {
return;
}
s.focus_scan.job = None;
s.focus_scan.auto_checked = false;
let path = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(RECORD_FILE));
s.focus_scan.record.reset(path);
s.focus_scan.library = Some(Arc::clone(&s.library_identity));
}
/// The photos to read: in the library, no focus point yet, from a body whose focus data is decoded, a raw or JPEG file;
/// with `ids`, those of them; without, every one not already looked at. In path order (neighbours on disk together).
fn candidates(s: &Session, ids: Option<&[PhotoId]>) -> Vec<(PhotoId, String)> {
let wanted = |ph: &lightcraft_catalog::Photo| -> Option<(PhotoId, String)> {
if !ph.in_library() || ph.meta.focus.is_some() || !lightcraft_meta::focus::camera_supported(&ph.meta.camera) {
return None;
}
let Source::File { path } = &ph.source else { return None };
let ext = std::path::Path::new(path).extension()?.to_str()?.to_ascii_lowercase();
EXTENSIONS.contains(&ext.as_str()).then(|| (ph.id, path.clone()))
};
let mut out: Vec<(PhotoId, String)> = match ids {
Some(ids) => ids.iter().filter_map(|id| s.catalog.photo(*id)).filter_map(|ph| wanted(ph)).collect(),
None => s.catalog.photos().filter(|ph| !s.focus_scan.record.contains(ph.id.0)).filter_map(|ph| wanted(ph)).collect(),
};
out.sort_by(|a, b| a.1.cmp(&b.1));
out.dedup_by_key(|c| c.0);
out
}
/// Start reading `list` on a worker thread.
fn start(s: &mut Session, list: Vec<(PhotoId, String)>, pace: Pace) -> std::result::Result<(), String> {
let cancel = Arc::new(AtomicBool::new(false));
let pace = Arc::new(AtomicU8::new(pace as u8));
let (tx, rx) = mpsc::channel();
let paths: HashMap<PhotoId, String> = list.iter().cloned().collect();
let total = list.len() as u64;
let (worker_cancel, worker_pace) = (cancel.clone(), pace.clone());
std::thread::Builder::new()
.name("focus-points".into())
.spawn(move || {
for (id, path) in list {
loop {
if worker_cancel.load(Ordering::Relaxed) {
return;
}
if worker_pace.load(Ordering::Relaxed) != Pace::Pause as u8 {
break;
}
std::thread::sleep(PAUSE_POLL);
}
if worker_pace.load(Ordering::Relaxed) == Pace::Light as u8 {
std::thread::sleep(LIGHT_REST);
}
// a decoder bug in one file is that file's problem, not the job's end
let outcome = match crate::guard::catch("reading a focus point", || read_focus(&path)) {
Ok(Ok(r)) => Outcome::Read { id, focus: r.focus, bytes: r.bytes_read },
Ok(Err(e)) => {
log::debug!("focus point: {e}");
Outcome::Unavailable
}
Err(_) => Outcome::Unavailable,
};
let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
if tx.send((outcome, name)).is_err() {
return;
}
}
})
.map_err(|e| format!("could not start reading focus points: {e}"))?;
let guard = s.activity.start("focus", "Reading focus points", Cancel::Flag(cancel.clone()));
guard.progress(0, total);
let now = Instant::now();
s.focus_scan.job = Some(Job {
identity: Arc::clone(&s.library_identity),
cancel,
pace,
rx,
guard,
paths,
pending: Vec::new(),
committed_at: now,
started: now,
summary: Summary { total, ..Default::default() },
});
Ok(())
}
/// Commit what has been read: the points found (one system op, no undo step, no sidecar write) and the photos with none
/// (to the record).
fn commit(s: &mut Session) {
let Some(job) = s.focus_scan.job.as_mut() else { return };
let pending = std::mem::take(&mut job.pending);
job.committed_at = Instant::now();
if !Arc::ptr_eq(&job.identity, &s.library_identity) {
return;
}
let mut ops = Vec::new();
let mut none = Vec::new();
for (id, focus) in pending {
let Some(ph) = s.catalog.photo(id) else { continue };
let same_file = matches!((&ph.source, job.paths.get(&id)), (Source::File { path }, Some(read)) if path == read);
// deleted, moved, relinked or given a point meanwhile: leave it
if !same_file || !ph.in_library() || ph.meta.focus.is_some() {
continue;
}
match focus {
Some(f) => {
let mut meta = ph.meta.clone();
meta.focus = Some(f);
ops.push(Op::SetMeta { id, meta: Box::new(meta) });
}
None => none.push(id.0),
}
}
for id in none {
s.focus_scan.record.insert(id);
}
if let Err(e) = s.focus_scan.record.save() {
log::warn!("could not save the list of photos read for focus points: {e}");
}
if ops.is_empty() {
return;
}
match s.apply_system(Op::Batch { ops }) {
// the focus point is never written to XMP: no sidecar is rewritten for it
Ok(()) => s.skip_auto_write = true,
Err(e) => log::warn!("could not store focus points: {e}"),
}
}
/// Take in what the worker has read, commit when a batch is due, and end the job when it is done or cancelled.
/// Returns whether a job is still running.
fn step(s: &mut Session, pace: Pace) -> bool {
let Some(job) = s.focus_scan.job.as_mut() else { return false };
job.pace.store(pace as u8, Ordering::Relaxed);
let mut finished = job.cancel.load(Ordering::Relaxed);
let mut last_name = None;
for _ in 0..MAX_DRAIN {
match job.rx.try_recv() {
Ok((outcome, name)) => {
job.summary.done += 1;
last_name = Some(name);
match outcome {
Outcome::Read { id, focus, bytes } => {
job.summary.bytes = job.summary.bytes.saturating_add(bytes);
if focus.is_some() {
job.summary.found += 1;
} else {
job.summary.none += 1;
}
job.pending.push((id, focus));
}
Outcome::Unavailable => job.summary.unavailable += 1,
}
}
Err(mpsc::TryRecvError::Empty) => break,
// the worker ended (all read, cancelled, or it could not go on)
Err(mpsc::TryRecvError::Disconnected) => {
finished = true;
break;
}
}
}
finished |= job.summary.done >= job.summary.total;
job.guard.progress(job.summary.done, job.summary.total);
if let Some(name) = last_name {
job.guard.detail(&name);
}
if finished || job.pending.len() >= COMMIT_EVERY || (!job.pending.is_empty() && job.committed_at.elapsed() >= COMMIT_AFTER) {
commit(s);
}
if finished && let Some(job) = s.focus_scan.job.take() {
let mut summary = job.summary.clone();
summary.cancelled = job.cancel.load(Ordering::Relaxed) && summary.done < summary.total;
summary.seconds = job.started.elapsed().as_secs_f64();
log::info!(
"focus points: read {} of {} photos ({} found, {} without, {} unavailable, {} KB) in {:.1} s",
summary.done,
summary.total,
summary.found,
summary.none,
summary.unavailable,
summary.bytes / 1024,
summary.seconds
);
s.focus_scan.last = Some(summary);
return false;
}
true
}
fn status(s: &Session) -> Value {
let running = s.focus_scan.job.as_ref().map(|j| json!({"done": j.summary.done, "total": j.summary.total, "found": j.summary.found}));
json!({"active": running.is_some(), "running": running, "last": s.focus_scan.last.as_ref().map(Summary::json), "lookedAt": s.focus_scan.record.len()})
}
/// `focus.pump {pace?}`: cheap to call every frame.
pub(crate) fn pump(s: &mut Session, p: &Value) -> Result<Value> {
follow_library(s);
let pace = Pace::parse(p.get("pace").and_then(Value::as_str));
s.focus_scan.pace = pace;
if !step(s, pace) && !s.focus_scan.auto_checked {
// once per library opening: start by itself when photos are left to look at (a few KB each)
s.focus_scan.auto_checked = true;
let list = candidates(s, None);
if !list.is_empty()
&& let Err(e) = start(s, list, pace)
{
log::warn!("{e}");
}
}
Ok(status(s))
}
/// `photo.readFocusPoints {ids?, wait?}`.
pub(crate) fn read_points(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "photo.readFocusPoints";
follow_library(s);
if s.focus_scan.job.is_some() {
return Ok(json!({"queued": 0, "running": true, "status": status(s)}));
}
let ids: Option<Vec<PhotoId>> = match p.get("ids") {
None | Some(Value::Null) => None,
Some(v) => Some(
v.as_array()
.ok_or_else(|| crate::cmd::bad(C, "`ids` must be a list of photo ids"))?
.iter()
.map(|v| v.as_u64().map(PhotoId).ok_or_else(|| crate::cmd::bad(C, "`ids` must be a list of photo ids")))
.collect::<Result<Vec<_>>>()?,
),
};
let list = candidates(s, ids.as_deref());
let queued = list.len();
if queued == 0 {
return Ok(json!({"queued": 0, "running": false}));
}
let wait = p.get("wait").and_then(Value::as_bool) == Some(true);
start(s, list, if wait { Pace::Normal } else { s.focus_scan.pace }).map_err(|e| crate::cmd::bad(C, e))?;
// a run asked for counts as this library opening's automatic one
s.focus_scan.auto_checked = true;
if !wait {
return Ok(json!({"queued": queued, "running": true}));
}
// a script's call: read them all now (the batches still commit as they come)
while step(s, Pace::Normal) {
std::thread::sleep(Duration::from_millis(5));
}
let last = s.focus_scan.last.as_ref().map(Summary::json).unwrap_or(Value::Null);
Ok(json!({"queued": queued, "running": false, "last": last}))
}
#[cfg(test)]
pub(crate) fn record_len(s: &Session) -> usize {
s.focus_scan.record.len()
}
#[cfg(test)]
pub(crate) fn record_contains(s: &Session, id: PhotoId) -> bool {
s.focus_scan.record.contains(id.0)
}
+27 -3
View File
@@ -382,6 +382,17 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
/// [`expand`] with the walk bounded by `limits` → (files, whether a walk stopped at a bound).
pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk::Limits) -> (Vec<String>, bool) {
expand_noting(paths, skip, limits, None)
}
/// [`expand_within`], handing the listing of every folder that is walked to `note`
/// ([`crate::walk::files_in_noting`]).
pub(crate) fn expand_noting(
paths: &[String],
skip: Option<&Path>,
limits: crate::walk::Limits,
mut note: Option<crate::walk::NoteFolder>,
) -> (Vec<String>, bool) {
let mut out = Vec::new();
let mut truncated = false;
for p in paths {
@@ -390,7 +401,10 @@ pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk:
continue;
}
if p.is_dir() {
let w = crate::walk::files_in(p, skip, limits, is_supported);
let w = match note.as_mut() {
Some(note) => crate::walk::files_in_noting(p, skip, limits, is_supported, &mut **note),
None => crate::walk::files_in(p, skip, limits, is_supported),
};
if w.truncated {
log::warn!(
"import: stopped looking in {} after {} entries (limits: {} entries, {} folder levels)",
@@ -638,6 +652,11 @@ impl ScanInput {
(input, paths.to_vec())
}
/// The folder a scan never enters (the library's own).
pub(crate) fn skip(&self) -> Option<&Path> {
self.skip.as_deref()
}
/// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one).
pub(crate) fn library(&self) -> std::sync::Arc<lightcraft_catalog::Catalog> {
std::sync::Arc::clone(&self.library)
@@ -656,9 +675,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it
/// has) when `progress.cancel` is set.
pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
use std::sync::atomic::Ordering::Relaxed;
pub fn scan_with(input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
let files = expand(paths, input.skip.as_deref());
scan_files(input, files, progress)
}
/// [`scan_with`] for files that were found already (`paths` expanded, [`expand`]).
pub(crate) fn scan_files(mut input: ScanInput, files: Vec<String>, progress: &ScanProgress) -> ScanOutput {
use std::sync::atomic::Ordering::Relaxed;
let by_path = library_files(&input.library, &files);
let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect();
progress.total.store(todo.len(), Relaxed);
+18
View File
@@ -29,6 +29,7 @@ pub mod demo;
pub mod denoise;
pub mod devices;
pub mod display;
pub mod doc_path;
mod errors;
pub mod export;
pub mod face_download;
@@ -38,6 +39,8 @@ mod faces_index;
#[cfg(not(target_arch = "wasm32"))]
mod faces_worker;
pub mod files;
#[cfg(not(target_arch = "wasm32"))]
mod focus_scan;
pub mod fonts;
pub mod guard;
pub mod import;
@@ -54,6 +57,7 @@ pub mod media;
pub mod memory;
pub mod merge;
mod model_download;
pub mod open_path;
pub mod originals;
mod pdf;
pub mod preset_import;
@@ -252,6 +256,9 @@ pub struct Session {
/// The loaded recognition model and the face embeddings made with it.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) faces: faces_index::FacesState,
/// Reading focus points for photos imported before they were read (`photo.readFocusPoints`, `focus.pump`).
#[cfg(not(target_arch = "wasm32"))]
pub(crate) focus_scan: focus_scan::FocusScan,
/// Copied develop settings (partial JSON) for Paste.
pub clipboard: Option<Value>,
/// The folder on disk the [`LibrarySource::Folder`] view browses.
@@ -293,6 +300,10 @@ pub struct Session {
pub xmp: sidecar::XmpPrefs,
/// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json.
pub last_export: Option<serde_json::Value>,
/// The options of the last Photo Merge started, per command (`merge.hdr`, `merge.panorama`,
/// `merge.hdrPanorama`): the merge dialog opens with them and "… with Last Settings" uses
/// them. Persisted in prefs.json.
pub last_merge: std::collections::BTreeMap<String, serde_json::Value>,
/// The user's export presets (built-ins: [`export::builtin_presets`]), persisted in prefs.json.
pub export_presets: Vec<export::ExportPreset>,
/// The Print module's current layout, image settings, page, guides and job (`print.*`). Lives
@@ -431,6 +442,8 @@ impl Session {
face_catalog: Default::default(),
#[cfg(not(target_arch = "wasm32"))]
faces: Default::default(),
#[cfg(not(target_arch = "wasm32"))]
focus_scan: Default::default(),
clipboard: None,
meta_clipboard: None,
browse: None,
@@ -452,6 +465,7 @@ impl Session {
library_identity: Arc::new(()),
xmp: sidecar::XmpPrefs::default(),
last_export: None,
last_merge: Default::default(),
export_presets: Vec::new(),
print: Default::default(),
print_templates: Vec::new(),
@@ -1250,6 +1264,8 @@ mod tests_face_recognize;
mod tests_face_tiles;
#[cfg(test)]
mod tests_filesystem_history;
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests_focus_scan;
#[cfg(test)]
mod tests_folders;
#[cfg(test)]
@@ -1275,6 +1291,8 @@ mod tests_mask_effects;
#[cfg(test)]
mod tests_merge;
#[cfg(test)]
mod tests_move_to_share;
#[cfg(test)]
mod tests_organize;
#[cfg(test)]
mod tests_persist;
+4
View File
@@ -207,6 +207,8 @@ struct PrefsFile {
/// Parameters of the last export (for Export with Previous).
#[serde(alias = "last_export")]
last_export: Option<serde_json::Value>,
/// Options of the last Photo Merge, per command.
last_merge: std::collections::BTreeMap<String, serde_json::Value>,
/// The user's export presets.
export_presets: Vec<crate::export::ExportPreset>,
/// Print templates.
@@ -492,6 +494,7 @@ impl Session {
let prefs = settings.read::<PrefsFile>(files.as_mut(), "prefs.json").unwrap_or_default();
self.xmp = prefs.xmp;
self.last_export = prefs.last_export;
self.last_merge = prefs.last_merge;
self.export_presets = prefs.export_presets;
self.print_templates = prefs.print_templates;
self.metadata_presets = prefs.metadata_presets;
@@ -746,6 +749,7 @@ impl Session {
let v = serde_json::to_vec_pretty(&PrefsFile {
xmp: self.xmp,
last_export: self.last_export.clone(),
last_merge: self.last_merge.clone(),
export_presets: self.export_presets.clone(),
print_templates: self.print_templates.clone(),
metadata_presets: self.metadata_presets.clone(),
+2
View File
@@ -538,6 +538,7 @@ mod tests {
keywords: vec!["Trips|Goa".into()],
history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])],
snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])],
original: crate::lightroom_catalog::Original::Local,
};
CatalogImport {
source: "source.lrcat".into(),
@@ -621,6 +622,7 @@ mod tests {
keywords: Vec::new(),
history: vec![BTreeMap::from([("text".into(), serde_json::json!(format!("history-{index}-{}", "x".repeat(8 * 1024))))])],
snapshots: Vec::new(),
original: crate::lightroom_catalog::Original::Local,
})
.collect();
assert_eq!(store_best_effort(&path, &input).unwrap(), None);
+213 -23
View File
@@ -2,7 +2,7 @@
//! Lightroom, Python and a C SQLite runtime are not needed. Original paths stay in place.
//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Component, Path, PathBuf};
use std::path::{Path, PathBuf};
use std::sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
@@ -146,6 +146,31 @@ pub struct CatalogPhoto {
pub keywords: Vec<String>,
pub history: Vec<Row>,
pub snapshots: Vec<Row>,
/// Whether `path` may be looked at (derived when the catalog is read; not archived).
#[serde(skip)]
pub original: Original,
}
/// Whether the original a catalog lists may be looked at. Its path is resolved with the catalog
/// as the document ([`crate::doc_path::resolve`]).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum Original {
/// `path` is a local file path, or one on the network share the catalog is on.
Local,
/// `path` is not a path on this computer (a catalog made on another system): the photo is
/// kept as an offline record for later relinking, and the path is not looked at.
#[default]
NotHere,
/// `path` is refused (why): it is on another computer or a device, or its folder or file name
/// is an absolute path. The photo is reported missing and not added to the library.
Refused(String),
}
impl CatalogPhoto {
/// Whether the original is a file here; only a [`Original::Local`] path is looked at.
pub fn original_is_file(&self) -> bool {
self.original == Original::Local && Path::new(&self.path).is_file()
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -343,25 +368,36 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {
/// Where a root folder is when its `absolutePath` isn't a folder here (the catalog came from
/// another computer): `relativePathFromCatalog`, which Lightroom records for roots on the
/// catalog's volume, resolved against the catalog's folder, if that is a folder.
/// catalog's volume, resolved against the catalog's folder, if that is a folder. Both paths are
/// looked at only when [`crate::doc_path::resolve`] follows them.
fn relocated_root(absolute: &str, relative: &str, catalog: &Path) -> Option<PathBuf> {
if relative.is_empty() || !Path::new(relative).is_relative() || Path::new(absolute).is_dir() {
if relative.is_empty() || !Path::new(relative).is_relative() || crate::doc_path::resolve(absolute, Some(catalog)).is_ok_and(|p| p.is_dir()) {
return None;
}
let dir = std::path::absolute(catalog).ok()?.parent()?.join(relative);
let mut out = PathBuf::new();
for part in dir.components() {
match part {
Component::ParentDir => {
if !out.pop() {
return None;
}
}
Component::CurDir => {}
part => out.push(part),
}
crate::doc_path::resolve(relative, Some(catalog)).ok().filter(|dir| dir.is_dir())
}
/// Whether a folder path or file name from the catalog is absolute: `Path::join` would put it in
/// place of the root folder.
fn is_absolute_part(part: &str) -> bool {
let bytes = part.as_bytes();
part.starts_with(['/', '\\'])
|| (bytes.first().is_some_and(u8::is_ascii_alphabetic) && bytes.get(1) == Some(&b':'))
|| Path::new(part).is_absolute()
}
/// The path of an original under `root` and whether it may be looked at, resolved with the
/// catalog at `catalog` as the document.
fn original_path(root: &str, folder: &str, name: &str, catalog: &Path) -> (String, Original) {
let stored = Path::new(root).join(folder).join(name).to_string_lossy().to_string();
if is_absolute_part(folder) || is_absolute_part(name) {
return (stored, Original::Refused("its folder or file name is an absolute path".into()));
}
match crate::doc_path::resolve(&stored, Some(catalog)) {
Ok(found) => (found.to_string_lossy().to_string(), Original::Local),
Err(crate::doc_path::NotFollowed::NotHere) => (stored, Original::NotHere),
Err(crate::doc_path::NotFollowed::Refused(why)) => (stored, Original::Refused(why.to_string())),
}
out.is_dir().then_some(out)
}
/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.
@@ -457,6 +493,7 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
out
};
let mut root_paths = HashMap::new();
let mut refused_roots = HashSet::new();
let mut root_ids: Vec<_> = roots.keys().copied().collect();
root_ids.sort_unstable();
for (id, root) in root_ids.iter().filter_map(|id| Some((*id, roots.get(id)?))) {
@@ -469,6 +506,11 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
}
None => absolute.to_string(),
};
if let Err(crate::doc_path::NotFollowed::Refused(why)) = crate::doc_path::resolve(&resolved, Some(path)) {
warnings
.push(format!("root folder {}: {resolved} is not opened: {why}; its photos are listed as missing and not added", text(root, "name")));
refused_roots.insert(id);
}
root_paths.insert(id, resolved);
}
let mut history = group(optional_table("Adobe_libraryImageDevelopHistoryStep", &mut warnings)?);
@@ -487,13 +529,19 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?;
let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?;
let root = roots.get(&number(folder, "rootFolder")).ok_or_else(|| format!("image {id}: missing root record"))?;
let root_path = root_paths.get(&number(root, "id_local")).map_or("", String::as_str);
let root_id = number(root, "id_local");
let root_path = root_paths.get(&root_id).map_or("", String::as_str);
let folder_path = text(folder, "pathFromRoot");
let name = text(file, "idx_filename");
if root_path.is_empty() || name.is_empty() {
return Err(format!("image {id}: incomplete original path"));
}
let path = Path::new(root_path).join(folder_path).join(name).to_string_lossy().to_string();
let (photo_path, original) = original_path(root_path, folder_path, name, path);
if let Original::Refused(why) = &original
&& !refused_roots.contains(&root_id)
{
out.warnings.push(format!("image {id}: {photo_path} is not opened: {why}; listed as missing and not added"));
}
let packet = match xmp(metadata.get(&id).and_then(|r| r.get("xmp"))) {
Ok(s) => s,
Err(e) => {
@@ -504,19 +552,64 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
out.photos.push(CatalogPhoto {
source_id: id,
uuid: text(&image, "id_global").into(),
path,
path: photo_path,
image,
settings: develops.get(&id).cloned().unwrap_or_default(),
xmp: packet,
keywords: keywords.remove(&id).unwrap_or_default(),
history: history.remove(&id).unwrap_or_default(),
snapshots: snapshots.remove(&id).unwrap_or_default(),
original,
});
}
refuse_non_media(&mut out);
ensure_read_active(cancel)?;
Ok(out)
}
/// Drop the catalog's images whose original is not a file type LightCraft reads, saying so in the
/// import's warnings.
///
/// The catalog decides these paths, and a record it leaves behind is one click from the system
/// opener (Photo ▸ Edit In ▸ Open Original in Other Editor). A record for `payload.exe`,
/// `start.lnk` or `Evil.app` would make a `.lrcat` someone was sent a way to have a program run
/// (issue #2135), so such an image never becomes a photo: not imported, not relinkable, not
/// openable. A virtual copy of a refused original goes with it — it has no original of its own.
///
/// Called where the catalog is read and again where its photos become records, so neither route
/// can keep one; the second pass finds nothing left to say.
pub(crate) fn refuse_non_media(data: &mut CatalogImport) {
// masters first: a virtual copy may come before its master in the table
let refused: HashSet<i64> = data
.photos
.iter()
.filter(|p| number(&p.image, "masterImage") == 0 && !crate::open_path::is_media(Path::new(&p.path)))
.map(|p| p.source_id)
.collect();
if refused.is_empty() {
return;
}
let mut warnings = Vec::new();
data.photos.retain(|p| {
let master = number(&p.image, "masterImage");
// `masterImage` is 0 for a master, so a refused id of 0 (a malformed catalog) must not
// take every master of the catalog with it
if master != 0 {
if !refused.contains(&master) {
return true;
}
warnings.push(format!("image {}: not imported; its original was not imported", p.source_id));
return false;
}
if !refused.contains(&p.source_id) {
return true;
}
warnings.push(format!("image {}: not imported; {} isn't one of the file types LightCraft reads", p.source_id, p.path));
false
});
data.warnings.extend(warnings);
}
fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
}
@@ -665,11 +758,11 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
let paths: Vec<_> = data
.photos
.iter()
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file())
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && p.original_is_file())
.map(|p| p.path.clone())
.collect();
let missing_sources: HashSet<String> =
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect();
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !p.original_is_file()).map(|p| key(&p.path)).collect();
let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;
let applied =
apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;
@@ -692,8 +785,10 @@ pub(crate) struct ApplyPrepared<'a> {
pub(crate) undo0: usize,
}
pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
pub(crate) fn apply_prepared(s: &mut crate::Session, mut data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;
// nothing but a file type LightCraft reads becomes a record, whichever route got here (#2135)
refuse_non_media(&mut data);
validate(&data).map_err(error)?;
let mut warnings = data.warnings.clone();
let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect();
@@ -713,12 +808,18 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
let mut photos: Vec<_> = data.photos.iter().collect();
photos.sort_by_key(|p| number(&p.image, "masterImage") != 0);
let copy_masters: HashSet<_> = data.photos.iter().map(|p| number(&p.image, "masterImage")).filter(|id| *id != 0).collect();
let refused: HashSet<_> = data.photos.iter().filter(|p| matches!(p.original, Original::Refused(_))).map(|p| p.source_id).collect();
let mut planned: HashMap<PhotoId, Photo> = HashMap::new();
for src in photos {
let master = number(&src.image, "masterImage");
if master == 0 && missing_sources.contains(&key(&src.path)) {
missing.push(src.path.clone());
}
// A refused path must not become a library record the app would look at later; its
// virtual copies go with it.
if refused.contains(&src.source_id) || refused.contains(&master) {
continue;
}
let indexed = index
.photos
.get(&identity(&data, src))
@@ -940,6 +1041,7 @@ mod tests {
keywords: vec!["Travel|Goa".into()],
history: vec![row(json!({"text":"s = { Exposure2012 = 0.5 }"}))],
snapshots: Vec::new(),
original: Original::Local,
};
let mut copy = photo.clone();
copy.source_id = 2;
@@ -962,6 +1064,48 @@ mod tests {
warnings: Vec::new(),
}
}
/// A crafted `.lrcat` that names a program as a photo's original gets no record for it, so
/// Photo ▸ Edit In ▸ Open Original has nothing to hand the system opener (#2135). Its virtual
/// copies go with it; the catalog's real photos still import.
#[test]
fn a_catalog_original_that_is_not_a_photo_file_is_not_imported() {
let mut data = sample();
let mut exe = data.photos[1].clone();
exe.source_id = 3;
exe.uuid = "payload".into();
exe.path = "/Users/sky/Downloads/payload.exe".into();
exe.image = row(json!({"id_local":3,"fileFormat":"ARW","fileWidth":20,"fileHeight":10}));
let mut exe_copy = exe.clone();
exe_copy.source_id = 4;
exe_copy.uuid = "payload-copy".into();
exe_copy.path = "/Users/sky/Downloads/payload.exe".into();
exe_copy.image = row(json!({"id_local":4,"masterImage":3,"copyName":"Mono"}));
// the copy comes before its master, as a catalog's table order may have it
data.photos.insert(0, exe_copy);
data.photos.push(exe);
for bad in [r"C:\Users\sky\start.lnk", "/Applications/Evil.app", "/Users/sky/go.command", "/Users/sky/run.bat"] {
let mut one = data.clone();
one.photos.retain(|p| p.source_id < 3);
let mut planted = one.photos[1].clone();
planted.source_id = 5;
planted.uuid = "planted".into();
planted.path = bad.into();
planted.image = row(json!({"id_local":5,"fileFormat":"ARW"}));
one.photos.push(planted);
let mut s = crate::Session::new();
let r = apply(&mut s, one, false).unwrap();
assert_eq!(r["photos"], 2, "{bad} is not catalogued: {r}");
assert!(s.catalog.photos().all(|p| !matches!(&p.source, Source::File { path } if path == bad)), "{bad} became a library record");
}
let mut s = crate::Session::new();
let r = apply(&mut s, data, false).unwrap();
assert_eq!(r["photos"], 2, "only the real master and its copy: {r}");
assert!(s.catalog.photos().all(|p| !matches!(&p.source, Source::File { path } if path.ends_with(".exe"))), "the program was catalogued");
let warnings = r["warnings"].to_string();
assert!(warnings.contains("payload.exe") && warnings.contains("file types LightCraft reads"), "{warnings}");
assert!(warnings.contains("image 4: not imported"), "the virtual copy of the refused original goes with it: {warnings}");
}
#[test]
fn offline_local_master_and_copy_join_library_with_one_undo_step() {
let mut s = crate::Session::new();
@@ -1508,11 +1652,57 @@ mod tests {
// a relative path that leads nowhere keeps the recorded one
std::fs::write(&path, native_catalog(&photo_tables(("C:/Users/me/Project/RAW/", "../Elsewhere/")))).unwrap();
let catalog = read(&path).unwrap();
assert!(catalog.photos[0].path.starts_with("C:/Users/me/Project/RAW/"), "{}", catalog.photos[0].path);
assert!(Path::new(&catalog.photos[0].path).starts_with("C:/Users/me/Project/RAW/"), "{}", catalog.photos[0].path);
assert!(catalog.warnings.is_empty(), "{:?}", catalog.warnings);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn originals_on_other_computers_are_never_looked_at_or_added() {
let dir = std::env::temp_dir().join(format!("lightcraft-lrcat-remote-roots-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("Photos")).unwrap();
std::fs::write(dir.join("Photos/test.ARW"), b"original").unwrap();
let source = dir.join("source.lrcat");
let mut roots = vec![r"\\192.0.2.1\share\".to_string(), "file://192.0.2.1/s/".to_string()];
if cfg!(unix) {
// `//<local folder>/` is a local folder to Linux, but a network path to the resolver.
roots.push(format!("/{}/", dir.display()));
}
for root in &roots {
std::fs::write(&source, native_catalog(&photo_tables((root, "")))).unwrap();
let catalog = read(&source).unwrap();
assert!(matches!(catalog.photos[0].original, Original::Refused(_)), "{root}: {:?}", catalog.photos[0]);
assert!(catalog.warnings.iter().any(|w| w.contains("not opened")), "{root}: {:?}", catalog.warnings);
let mut s = crate::Session::new();
let probes = Arc::new(AtomicUsize::new(0));
let calls = probes.clone();
s.media.file_probe = Some(Arc::new(move |_| {
calls.fetch_add(1, Ordering::Relaxed);
Err("a refused original must not be probed".into())
}));
let report = import(&mut s, &source, false).unwrap();
assert_eq!(probes.load(Ordering::Relaxed), 0, "{root}");
assert_eq!(report["missing"].as_array().map(Vec::len), Some(1), "{root}: {report}");
assert!(s.catalog.is_empty(), "{root}");
}
// An absolute folder path would replace the root.
let mut tables = photo_tables((dir.to_str().unwrap(), ""));
tables[1].2[2] = SqlValue::Text(format!("{}/Photos/", dir.display()));
std::fs::write(&source, native_catalog(&tables)).unwrap();
assert!(matches!(read(&source).unwrap().photos[0].original, Original::Refused(_)));
// A root beside the catalog is followed.
std::fs::write(&source, native_catalog(&photo_tables((dir.to_str().unwrap(), "")))).unwrap();
let catalog = read(&source).unwrap();
assert_eq!(catalog.photos[0].original, Original::Local);
assert!(catalog.photos[0].original_is_file());
// A root that moved is found through its path relative to the catalog.
std::fs::write(&source, native_catalog(&photo_tables((r"\\192.0.2.1\share\", "./")))).unwrap();
let catalog = read(&source).unwrap();
assert!(catalog.photos[0].original_is_file(), "{:?}", catalog.photos[0]);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn oversized_optional_table_warns_and_is_skipped() {
let mut main = vec![0u8; 2 * 512];
+3 -1
View File
@@ -329,7 +329,7 @@ fn original_paths(data: &CatalogImport, cancel: &AtomicBool) -> crate::Result<(V
if !seen.insert(key.clone()) {
continue;
}
if Path::new(&photo.path).is_file() {
if photo.original_is_file() {
paths.push(photo.path.clone());
} else {
missing.insert(key);
@@ -393,6 +393,7 @@ mod tests {
keywords: Vec::new(),
history: Vec::new(),
snapshots: Vec::new(),
original: lightroom_catalog::Original::Local,
}],
collections: Vec::new(),
members: Vec::new(),
@@ -531,6 +532,7 @@ mod tests {
serde_json::json!(format!("history-{source_id}-{}", "x".repeat(8 * 1024))),
)])],
snapshots: Vec::new(),
original: lightroom_catalog::Original::Local,
}
})
.collect();
+34
View File
@@ -2189,6 +2189,40 @@ mod tests {
assert!(mean(&a) > mean(&b) + 10.0, "two stops brighter after: {} vs {}", mean(&a), mean(&b));
}
// LR-VIEW-ZOOM: what the loupe shows at 1:1 is what an export at full size writes. A window of
// the zoomed frame (the photo's own pixels) is pixel-for-pixel the export, at the centre and
// at a corner, with tone, colour and detail edits on a camera-sized photo
#[test]
fn one_to_one_window_pixels_are_export_pixels() {
use lightcraft_pipeline::{OutputDepth, OutputSpace, PixelWindow};
let mut s = crate::Session::with_demo();
let id = s.active().unwrap();
let p = s.catalog.photo(id).unwrap().clone();
let (w, h) = (p.width as usize, p.height as usize);
assert!(w.max(h) > 2560, "larger than the preview cap: {w}×{h}");
let mut d = (*s.develop_of(id).unwrap()).clone();
d.light.exposure = 0.7;
d.light.contrast = 25.0;
d.color.saturation = 20.0;
s.set_develop(id, d, "Edits").unwrap();
let export = s.render_export(id, w, h, OutputSpace::Srgb, OutputDepth::U8).unwrap().image;
assert_eq!((export.width, export.height), (w, h), "a full-size export");
for win in [PixelWindow { x: w / 2 - 256, y: h / 2 - 256, w: 512, h: 384 }, PixelWindow { x: 0, y: h - 300, w: 400, h: 300 }] {
let tile = s.region_job(id, w, h, win, true).unwrap().run().rendered.unwrap().image;
assert_eq!((tile.width, tile.height), (win.w, win.h));
let mut worst = 0u8;
for y in 0..win.h {
for x in 0..win.w {
let (a, b) = (tile.data[y * win.w + x], export.data[(win.y + y) * w + win.x + x]);
for k in 0..3 {
worst = worst.max(a[k].abs_diff(b[k]));
}
}
}
assert!(worst <= 1, "window {win:?}: the 1:1 view differs from the export by up to {worst} levels");
}
}
/// The thumbnail of a photo whose preview-level source is decoded is cut from it: the file (here
/// one nothing can decode) is not read again.
#[test]
+117
View File
@@ -0,0 +1,117 @@
//! Whether a path the library holds may be handed to the operating system.
//!
//! A photo's `Source::File` path is not always something a user typed: it can come from an
//! imported Lightroom catalog, an `.lrcat` someone was sent, a relink or a synchronised folder.
//! Photo ▸ Edit In ▸ *Open Original in Other Editor* and the system default opener run whatever
//! the path names, so a record pointing at `payload.exe`, `thing.lnk`, `run.bat`, `page.hta`
//! (Windows) or `Evil.app`, `go.command` (macOS) would be executed on one click (issue #2135).
//!
//! [`openable`] is the one gate every such hand-over goes through: the path must be a regular
//! file whose extension is one LightCraft itself reads ([`crate::import::EXTENSIONS`], which
//! already covers the Edit In copy formats `tif`/`tiff`/`psd`), and what a link resolves to must
//! be one too, so a `photo.jpg` pointing at `/bin/sh` is refused as well. What comes back is the
//! absolute path to hand over; what comes back on refusal is the sentence to show the user.
//!
//! Nothing here opens, stats-for-content or reads the file: it only decides whether the path may
//! leave LightCraft. Catalog import applies the same test ([`crate::lightroom_catalog`]), so a
//! refused original never becomes a library record in the first place.
use std::path::Path;
/// The absolute path to hand to an opener or external editor, or the reason it is refused.
///
/// `label` names the thing in the error (a file name, a photo's name); an empty `label` uses
/// the path's own file name.
pub fn openable(path: &str, label: &str) -> Result<String, String> {
let p = Path::new(path);
let name = match label.trim() {
"" => p.file_name().map_or_else(|| path.to_string(), |n| n.to_string_lossy().into_owned()),
l => l.to_string(),
};
if !is_media(p) {
return Err(format!("{name} isn't one of the file types LightCraft reads, so it isn't handed to another application"));
}
match std::fs::metadata(p) {
Ok(m) if m.is_file() => {}
Ok(_) => return Err(format!("{name} isn't a file, so it isn't handed to another application")),
Err(e) => return Err(format!("the original is missing: {path} ({e})")),
}
// a link may name a photo and point at a program; what runs is what it points at
if let Ok(target) = p.canonicalize()
&& !is_media(&target)
{
return Err(format!("{name} points at {}, which isn't a file type LightCraft reads", target.display()));
}
let absolute = std::path::absolute(p).map_err(|e| format!("{name}: {e}"))?;
Ok(absolute.to_string_lossy().into_owned())
}
/// Whether `path`'s extension is one LightCraft reads (the Edit In copy formats included).
pub fn is_media(path: &Path) -> bool {
crate::import::is_supported(path)
}
#[cfg(test)]
mod tests {
use super::*;
fn write(dir: &Path, name: &str) -> String {
let p = dir.join(name);
std::fs::write(&p, b"x").unwrap();
p.to_string_lossy().into_owned()
}
#[test]
fn only_file_types_lightcraft_reads_are_handed_over() {
let dir = std::env::temp_dir().join(format!("lc-openable-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
for good in ["DSC_1.NEF", "a.jpg", "copy-Edit.tif", "copy-Edit.psd"] {
let path = write(&dir, good);
assert_eq!(openable(&path, "").unwrap(), path, "{good} is a photo LightCraft reads");
}
for bad in ["payload.exe", "shortcut.lnk", "run.bat", "page.hta", "go.command", "no-extension"] {
let path = write(&dir, bad);
let e = openable(&path, "").unwrap_err();
assert!(e.contains(bad) && e.contains("file types LightCraft reads"), "{bad}: {e}");
}
// a bundle (macOS `.app`) and any other folder are not files
let bundle = dir.join("Evil.app");
std::fs::create_dir_all(&bundle).unwrap();
let e = openable(&bundle.to_string_lossy(), "").unwrap_err();
assert!(e.contains("isn't one of the file types") || e.contains("isn't a file"), "{e}");
let e = openable(&dir.join("gone.jpg").to_string_lossy(), "").unwrap_err();
assert!(e.contains("missing"), "{e}");
let _ = std::fs::remove_dir_all(&dir);
}
/// A relative path is made absolute: the opener runs with its own working directory.
#[test]
fn the_path_handed_over_is_absolute() {
let dir = std::env::temp_dir().join(format!("lc-openable-abs-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = write(&dir, "a.jpg");
let dotted = Path::new(&path).parent().unwrap().join(".").join("a.jpg");
let out = openable(&dotted.to_string_lossy(), "").unwrap();
assert!(Path::new(&out).is_absolute(), "{out}");
assert!(out.ends_with("a.jpg"), "{out}");
let _ = std::fs::remove_dir_all(&dir);
}
/// A link named like a photo that points at a program is refused: what the opener runs is
/// the target, not the name.
#[cfg(unix)]
#[test]
fn a_link_to_a_program_is_refused() {
let dir = std::env::temp_dir().join(format!("lc-openable-link-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let target = write(&dir, "payload.command");
let link = dir.join("holiday.jpg");
std::os::unix::fs::symlink(&target, &link).unwrap();
let e = openable(&link.to_string_lossy(), "").unwrap_err();
assert!(e.contains("points at") && e.contains("payload.command"), "{e}");
let _ = std::fs::remove_dir_all(&dir);
}
}
+246 -6
View File
@@ -1,5 +1,6 @@
//! Composes print pages: places each cell's photo (fit or Zoom to Fill, optionally Rotate to Fit),
//! draws Photo Border and Inner Stroke, fills the page background and returns [`PageRaster`]s.
//! draws Photo Border and Inner Stroke, fills the page background, adds the Page Options (crop
//! marks, page number, page info) and returns [`PageRaster`]s.
//!
//! Work is split like an export: [`prepare`] reads the session (which photos go in which cell, one
//! render job per cell) and [`Prepared::run`] does the heavy part without it, so it can run on a
@@ -28,6 +29,13 @@ pub const MAX_BORDER_IN: f64 = 2.0;
pub const MAX_STROKE_PT: f64 = 36.0;
/// Renders kept for cells that print the same photo at the same size.
const RENDER_CACHE: usize = 4;
/// A crop mark's length and its gap from the photo's corner, inches.
pub const CROP_MARK_IN: f64 = 0.25;
pub const CROP_MARK_GAP_IN: f64 = 0.0625;
/// A crop mark's weight, points.
pub const CROP_MARK_PT: f64 = 0.5;
/// Size of the page number and page info text, points.
pub const PAGE_TEXT_PT: f64 = 9.0;
/// Lightroom's Image Settings panel.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -85,11 +93,18 @@ impl ImageSettings {
pub struct PageStyle {
/// Page background colour, sRGB.
pub background: [u8; 3],
/// Page Options (Single Image and Contact Sheet, as in Lightroom; the packages ignore them):
/// the page number at the bottom right of each page.
pub page_numbers: bool,
/// The print settings (sharpening, profile, resolution) at the bottom left of each page.
pub page_info: bool,
/// Marks at the corners of each photo to cut along.
pub crop_marks: bool,
}
impl Default for PageStyle {
fn default() -> Self {
Self { background: [255; 3] }
Self { background: [255; 3], page_numbers: false, page_info: false, crop_marks: false }
}
}
@@ -100,6 +115,41 @@ pub fn rec2020_from_srgb8(c: [u8; 3]) -> [u16; 3] {
lin.map(|v| (trc.encode(v) * 65535.0 + 0.5) as u16)
}
/// Whether a layout style prints the Page Options (page numbers, page info, crop marks):
/// Lightroom offers them for Single Image and Contact Sheet; its packages have Cut Guides instead.
pub fn has_page_options(style: Style) -> bool {
!style.is_package()
}
/// The colour of crop marks and page text on `background`: black, or white on a dark page.
pub fn mark_ink(background: [u8; 3]) -> [u8; 3] {
let [r, g, b] = background.map(f64::from);
if 0.2126 * r + 0.7152 * g + 0.0722 * b < 128.0 { [255; 3] } else { [0; 3] }
}
/// The Page Info line: print sharpening, profile and resolution (Lightroom names the printer
/// where the resolution is; this print goes to a PDF).
pub fn page_info_text(settings: &PrintSettings) -> String {
use crate::print_color::{ManagedSpace, PrintMedia, PrintProfile, PrintSharpen};
let c = &settings.job.color;
let media = match c.media {
PrintMedia::Matte => "Matte",
PrintMedia::Glossy => "Glossy",
};
let sharpen = match c.sharpen {
PrintSharpen::Off => "Off".to_string(),
PrintSharpen::Low => format!("Low ({media})"),
PrintSharpen::Standard => format!("Standard ({media})"),
PrintSharpen::High => format!("High ({media})"),
};
let profile = match &c.profile {
PrintProfile::Managed { space: ManagedSpace::Srgb } => "Managed by Printer (sRGB)".to_string(),
PrintProfile::Managed { space: ManagedSpace::AdobeRgb } => "Managed by Printer (Adobe RGB)".to_string(),
PrintProfile::Icc { path } => path.file_stem().map_or_else(|| path.display().to_string(), |n| n.to_string_lossy().into_owned()),
};
format!("Print Sharpening: {sharpen} · Profile: {profile} · {:.0} ppi", settings.job.effective_ppi())
}
// ---- placement ------------------------------------------------------------------------------
/// How a photo of `iw` x `ih` goes into an `aw` x `ah` area.
@@ -355,8 +405,8 @@ impl CellStyle {
}
/// Draws `photo` into `cell` of `page`: border frame, the photo fitted or filled (and turned when
/// asked), then the inner stroke on top of it.
pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &CellStyle) -> Result<(), String> {
/// asked), then the inner stroke on top of it. Returns where the photo landed.
pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &CellStyle) -> Result<PxRect, String> {
// The border may not eat the cell: at least one pixel of photo area stays.
let border = style.border_px.min(cell.w.min(cell.h).saturating_sub(1) / 2);
let (ax, ay) = (cell.x + border, cell.y + border);
@@ -382,7 +432,105 @@ pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &Cell
page.fill_rect(dx, dy + s, s, h - 2 * s, style.stroke);
page.fill_rect(dx + w - s, dy + s, s, h - 2 * s, style.stroke);
}
Ok(())
Ok(PxRect { x: dx, y: dy, w, h })
}
/// What the Page Options draw on every page, in page pixels.
struct Marks {
/// Crop mark length, gap from the corner and weight.
crop: Option<(usize, usize, usize)>,
numbers: bool,
info: Option<String>,
ink: [u16; 3],
text_px: f32,
/// The printable area: the text sits in the margin below it, aligned with its sides.
area: Option<PxRect>,
}
impl Marks {
fn of(settings: &PrintSettings, ppi: f64, pw: usize, ph: usize) -> Self {
let on = has_page_options(settings.layout.style);
let page = &settings.page;
let px = |inches: f64| (inches * ppi).round() as usize;
Marks {
crop: (on && page.crop_marks).then(|| (px(CROP_MARK_IN), px(CROP_MARK_GAP_IN), px(CROP_MARK_PT / 72.0).max(1))),
numbers: on && page.page_numbers,
info: (on && page.page_info).then(|| page_info_text(settings)),
ink: rec2020_from_srgb8(mark_ink(page.background)),
text_px: (PAGE_TEXT_PT / 72.0 * ppi) as f32,
area: settings.layout.printable().ok().and_then(|a| px_rect(&a, ppi, pw, ph)),
}
}
/// Draws the marks of page `number` (one-based) around `photos`, the photos on it.
fn draw(&self, page: &mut Image16, photos: &[PxRect], number: usize) {
if let Some((len, gap, weight)) = self.crop {
for r in photos {
crop_marks(page, r, len, gap, weight, photos, self.ink);
}
}
let area = self.area.unwrap_or(PxRect { x: 0, y: 0, w: page.w, h: page.h });
let (area_bottom, page_h) = ((area.y + area.h) as f32, page.h as f32);
// centred in the bottom margin, and on the paper when the margin is thinner than the text
let top = |h: f32| (area_bottom + (page_h - area_bottom - h) / 2.0).clamp(0.0, (page_h - h).max(0.0));
if self.numbers
&& let Some(t) = crate::text::TextLayout::new(&number.to_string(), self.text_px, false, crate::fonts::CRAFT_FONTS)
{
let x = ((area.x + area.w) as f32 - t.width).max(0.0);
draw_text(page, &t, x, top(t.height), self.ink);
}
if let Some(info) = &self.info
&& let Some(t) = crate::text::TextLayout::new(info, self.text_px, false, crate::fonts::CRAFT_FONTS)
{
draw_text(page, &t, area.x as f32, top(t.height), self.ink);
}
}
}
/// Crop marks for the photo at `r`: at each corner, a line along each of its edges that starts
/// `gap` past the corner and runs `len` outward, `weight` thick. Marks never cover a photo.
fn crop_marks(page: &mut Image16, r: &PxRect, len: usize, gap: usize, weight: usize, photos: &[PxRect], ink: [u16; 3]) {
let (x0, y0, x1, y1) = (r.x as i64, r.y as i64, (r.x + r.w) as i64, (r.y + r.h) as i64);
let (l, g, t) = (len as i64, gap as i64, weight.min(r.w).min(r.h) as i64);
let segments = [
(x0 - g - l, y0, l, t),
(x1 + g, y0, l, t),
(x0 - g - l, y1 - t, l, t),
(x1 + g, y1 - t, l, t),
(x0, y0 - g - l, t, l),
(x0, y1 + g, t, l),
(x1 - t, y0 - g - l, t, l),
(x1 - t, y1 + g, t, l),
];
let on_photo = |x: usize, y: usize| photos.iter().any(|p| x >= p.x && x < p.x + p.w && y >= p.y && y < p.y + p.h);
for (sx, sy, sw, sh) in segments {
for y in sy.max(0)..(sy + sh).min(page.h as i64) {
for x in sx.max(0)..(sx + sw).min(page.w as i64) {
let (x, y) = (x as usize, y as usize);
if !on_photo(x, y) {
page.fill_rect(x, y, 1, 1, ink);
}
}
}
}
}
/// Blends text in `ink` into `page` with its top-left corner at (`x`, `y`).
fn draw_text(page: &mut Image16, text: &crate::text::TextLayout, x: f32, y: f32, ink: [u16; 3]) {
let (w, h) = (page.w, page.h);
text.draw_in(x, y, w as f32, h as f32, |gx, gy, cov| {
let (Ok(gx), Ok(gy)) = (usize::try_from(gx), usize::try_from(gy)) else { return };
if gx >= w || gy >= h {
return;
}
let i = (gy * w + gx) * 3;
if let Some(px) = page.rgb.get_mut(i..i + 3) {
let k = cov.clamp(0.0, 1.0);
for (p, c) in px.iter_mut().zip(ink) {
*p = (f32::from(*p) + (f32::from(c) - f32::from(*p)) * k).round().clamp(0.0, 65535.0) as u16;
}
}
});
}
// ---- the job ---------------------------------------------------------------------------------
@@ -402,6 +550,7 @@ pub struct Prepared {
ppi: f32,
background: [u16; 3],
style: CellStyle,
marks: Marks,
/// Print sharpening, applied to each photo once it's rendered at its print size.
color: crate::print_color::PrintColorSettings,
}
@@ -430,6 +579,7 @@ impl Prepared {
let mut out = Vec::with_capacity(self.pages.len());
for (index, &(w, h)) in self.pages.iter().enumerate() {
let mut page = Image16::filled(w, h, self.background)?;
let mut placed = Vec::new();
while let Some(item) = items.next_if(|i| i.page == index) {
if !progress(done, &item.name) {
return Err("Print cancelled".into());
@@ -448,8 +598,9 @@ impl Prepared {
}
};
let photo = &cache.get(at).ok_or("render cache lost an entry")?.1;
draw_cell(&mut page, photo, item.cell, &self.style).map_err(|e| format!("{}: {e}", item.name))?;
placed.push(draw_cell(&mut page, photo, item.cell, &self.style).map_err(|e| format!("{}: {e}", item.name))?);
}
self.marks.draw(&mut page, &placed, index + 1);
out.push(PageRaster { width: w as u32, height: h as u32, ppi: self.ppi, rgb: page.rgb });
}
if !progress(done, "") {
@@ -508,6 +659,7 @@ pub fn prepare(session: &mut Session, settings: &PrintSettings, photos: &[PhotoI
ppi: ppi as f32,
background: rec2020_from_srgb8(settings.page.background),
style,
marks: Marks::of(&settings, ppi, pw, ph),
color: settings.job.color.clone(),
})
}
@@ -825,6 +977,94 @@ mod tests {
assert!(green_px > 1000, "the border frames the photo inside its cell: {green_px} border pixels");
}
/// Pages for two photos, one per 4x6 page at 100 ppi, each filling its 0.5 in margins (so the
/// margins are bare background unless something draws there).
fn page_options_pages(s: &mut Session, style: Style, page: PageStyle) -> Vec<PageRaster> {
let ids = first_ids(s, 2);
let mut set = settings(style);
if style.is_package() {
set.layout.add_cell(3.0, 5.0).unwrap();
set.layout.new_page().unwrap();
let id = set.layout.add_cell(3.0, 5.0).unwrap();
set.layout.set_cell_page(id, 1).unwrap();
}
set.image.zoom_to_fill = true;
set.page = page;
prepare(s, &set, &ids).unwrap().run(&mut |_, _| true).unwrap()
}
fn rgb(p: &PageRaster, x: usize, y: usize) -> [u16; 3] {
let i = (y * p.width as usize + x) * 3;
[p.rgb[i], p.rgb[i + 1], p.rgb[i + 2]]
}
/// The pixels of `x0..x1` x `y0..y1` that are not the white background.
fn inked(p: &PageRaster, x0: usize, x1: usize, y0: usize, y1: usize) -> usize {
(y0..y1).flat_map(|y| (x0..x1).map(move |x| (x, y))).filter(|&(x, y)| rgb(p, x, y) != WHITE).count()
}
#[test]
fn crop_marks_sit_outside_each_photo_corner() {
let mut s = Session::with_demo();
let plain = page_options_pages(&mut s, Style::SingleImage, PageStyle::default());
assert_eq!(inked(&plain[0], 0, 400, 0, 50), 0, "no marks unless asked");
let pages = page_options_pages(&mut s, Style::SingleImage, PageStyle { crop_marks: true, ..Default::default() });
// The photo fills 50..350 x 50..550; a mark is a line 0.25 in long, 1/16 in from the corner,
// along each edge of the photo.
for p in &pages {
for (x, y) in [(30, 50), (50, 30), (370, 50), (349, 30), (30, 549), (50, 570), (370, 549), (349, 570)] {
assert_eq!(rgb(p, x, y), [0; 3], "a mark at ({x}, {y})");
}
for (x, y) in [(47, 50), (50, 47), (30, 60), (60, 30), (10, 50), (50, 10)] {
assert_eq!(rgb(p, x, y), WHITE, "the gap, or past the mark, at ({x}, {y})");
}
}
// a dark page gets white marks
let dark = page_options_pages(&mut s, Style::SingleImage, PageStyle { background: [20; 3], crop_marks: true, ..Default::default() });
assert_eq!(rgb(&dark[0], 30, 50), WHITE);
}
#[test]
fn page_numbers_and_info_print_in_the_bottom_margin() {
let mut s = Session::with_demo();
let numbered = page_options_pages(&mut s, Style::SingleImage, PageStyle { page_numbers: true, ..Default::default() });
// bottom right: the number, different on each page; nothing at the bottom left
let (right, left) = ((200, 400), (0, 200));
for p in &numbered {
assert!(inked(p, right.0, right.1, 550, 600) > 10, "a page number");
assert_eq!(inked(p, left.0, left.1, 550, 600), 0);
assert_eq!(inked(p, 0, 400, 0, 50), 0, "nothing at the top");
}
let region = |p: &PageRaster| (550..600).flat_map(|y| (right.0..right.1).map(move |x| (x, y))).map(|(x, y)| rgb(p, x, y)).collect::<Vec<_>>();
assert_ne!(region(&numbered[0]), region(&numbered[1]), "page 1 and page 2 are numbered differently");
let info = page_options_pages(&mut s, Style::SingleImage, PageStyle { page_info: true, ..Default::default() });
for p in &info {
assert!(inked(p, left.0, left.1, 550, 600) > 10, "the print settings line");
}
// the packages have no Page Options in Lightroom: nothing is drawn
let all = PageStyle { page_numbers: true, page_info: true, crop_marks: true, ..Default::default() };
let package = page_options_pages(&mut s, Style::PicturePackage, all);
assert_eq!(package.len(), 2);
for p in &package {
assert_eq!(inked(p, 0, 400, 550, 600) + inked(p, 0, 400, 0, 50), 0);
}
}
#[test]
fn page_info_names_the_sharpening_profile_and_resolution() {
use crate::print_color::{ManagedSpace, PrintMedia, PrintProfile, PrintSharpen};
let mut set = PrintSettings::default();
let line = page_info_text(&set);
assert_eq!(line, "Print Sharpening: Standard (Glossy) · Profile: Managed by Printer (sRGB) · 240 ppi");
set.job.color.sharpen = PrintSharpen::Off;
set.job.color.media = PrintMedia::Matte;
set.job.color.profile = PrintProfile::Icc { path: "/profiles/Lustre Paper.icc".into() };
set.job.ppi = 300.0;
assert_eq!(page_info_text(&set), "Print Sharpening: Off · Profile: Lustre Paper · 300 ppi");
set.job.color.profile = PrintProfile::Managed { space: ManagedSpace::AdobeRgb };
assert!(page_info_text(&set).contains("Managed by Printer (Adobe RGB)"));
}
#[test]
fn caps_and_hostile_settings_are_refused() {
let mut s = Session::with_demo();
+27 -7
View File
@@ -433,12 +433,31 @@ impl MoveFs for RealFs {
}
fn copy_no_replace(&self, a: &Path, b: &Path) -> std::io::Result<()> {
use std::io::Write;
let bytes = std::fs::read(a)?;
let mut f = std::fs::OpenOptions::new().write(true).create_new(true).open(b)?;
f.write_all(&bytes).and_then(|_| f.sync_all()).inspect_err(|_| {
let _ = std::fs::remove_file(b);
})
if self.exists(b) {
return Err(std::io::Error::new(std::io::ErrorKind::AlreadyExists, format!("{} already exists", b.display())));
}
// Streamed (a large raw or video is never read into memory whole), into a hidden name
// beside `b` that only takes `b`'s name once complete: a share that drops mid-copy, so
// the partial file can't be removed either, leaves no truncated file under the photo's
// name for a retry to take for the photo.
let name = b.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
// a free name: one left by a copy the share cut off (it couldn't be removed then) doesn't
// block the next
let partial = (0..1000u32)
.map(|k| b.with_file_name(format!(".{name}.lc-partial-{}-{k}", std::process::id())))
.find(|p| !self.exists(p))
.ok_or_else(|| std::io::Error::other(format!("no free temporary name beside {}", b.display())))?;
let copied = (|| {
let mut from = std::fs::File::open(a)?;
let mut f = std::fs::OpenOptions::new().write(true).create_new(true).open(&partial)?;
std::io::copy(&mut from, &mut f)?;
f.sync_all()
})();
let placed = copied.and_then(|()| self.rename_no_replace(&partial, b));
if placed.is_err() {
let _ = std::fs::remove_file(&partial);
}
placed
}
fn remove_file(&self, p: &Path) -> std::io::Result<()> {
@@ -897,7 +916,8 @@ fn plan_rename_core(
) -> Vec<RenamePlan> {
let mut taken: HashSet<String> = HashSet::new(); // lower-case target paths/names claimed in this batch
let mut out = Vec::new();
for (seq, p) in (start..).zip(photos) {
for (i, p) in photos.iter().enumerate() {
let seq = start.saturating_add(i);
let want = expand(template, p, seq);
let (stem, ext) = split_ext(&want);
let (stem, ext) = (stem.to_string(), ext.to_string());
+38 -2
View File
@@ -4,6 +4,8 @@
//! change when [`XmpPrefs::auto_write`] is on): rating, colour label, title, caption, copyright,
//! creator, keywords, capture time and GPS in the standard namespaces, plus our complete develop
//! settings (`lc:settings`, JSON), the pick/reject flag (`lc:flag`) and location (`lc:location`).
//! A sidecar is written only beside an original that is an existing file at an absolute path;
//! otherwise the save fails.
//!
//! An existing sidecar is never replaced wholesale (issue #92): LightCraft's properties
//! ([`OWNED`], [`OWNED_IF_STATED`]) are merged into it ([`lightcraft_meta::merge_xmp`]) and
@@ -66,6 +68,13 @@ pub fn sidecar_path(original: &str, naming: SidecarNaming) -> PathBuf {
/// accepting `.XMP` and `.Xmp`, and a full name whose photo extension differs in case, such as
/// `IMG_1.cr3.xmp` beside `IMG_1.CR3`, which a case-sensitive file system would otherwise miss).
pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
sidecar_candidates(original, naming).find(|p| p.is_file())
}
/// The names `original`'s sidecar can have, in the order they are looked for: under `naming`,
/// then under the other naming (the full name also with the photo extension in the other case),
/// each as `.xmp`, `.XMP` and `.Xmp`.
pub(crate) fn sidecar_candidates(original: &str, naming: SidecarNaming) -> impl Iterator<Item = PathBuf> {
let other = if naming == SidecarNaming::Stem { SidecarNaming::Full } else { SidecarNaming::Stem };
let ext = Path::new(original).extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
let base = Path::new(original).with_extension("");
@@ -73,7 +82,7 @@ pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
let full_variants: Vec<String> = [recased(ext.to_lowercase()), recased(ext.to_uppercase())].into_iter().flatten().collect();
[naming, other]
.into_iter()
.flat_map(|n| {
.flat_map(move |n| {
let mut v = vec![sidecar_path(original, n)];
if n == SidecarNaming::Full {
v.extend(full_variants.iter().map(|o| sidecar_path(o, n)));
@@ -81,7 +90,6 @@ pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
v
})
.flat_map(|p| [p.clone(), p.with_extension("XMP"), p.with_extension("Xmp")])
.find(|p| p.is_file())
}
/// What a sidecar (or embedded XMP packet) says about a photo. `None` = not stated.
@@ -626,6 +634,10 @@ impl Session {
return Err(EngineError::Other(format!("{} is a virtual copy: its settings live only in the library", p.file_name)));
}
let orig = file_path(p).ok_or_else(|| EngineError::Other(format!("{} is not a file on disk", p.file_name)))?;
// A relative path would be taken from the working directory.
if Path::new(orig).is_relative() || !Path::new(orig).is_file() {
return Err(EngineError::Other(format!("{}: the original {orig} is missing; no XMP sidecar is written", p.file_name)));
}
let naming = owners.naming(p, self.xmp.naming);
let preferred = sidecar_path(orig, naming);
let other = if naming == SidecarNaming::Stem { SidecarNaming::Full } else { SidecarNaming::Stem };
@@ -780,6 +792,30 @@ impl Session {
#[cfg(test)]
mod tests {
#[test]
fn sidecars_are_written_only_beside_an_existing_original() {
let dir = std::env::temp_dir().join(format!("lightcraft-sidecar-local-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("IMG_1.jpg"), b"original").unwrap();
let originals = [
(r"\\192.0.2.1\share\IMG_1.jpg".to_string(), false),
(dir.join("IMG_2.jpg").to_string_lossy().to_string(), false),
(dir.join("IMG_1.jpg").to_string_lossy().to_string(), true),
];
for (original, written) in originals {
let _ = std::fs::remove_file(dir.join("IMG_1.xmp"));
let mut s = Session::new();
let p = Photo::new(PhotoId(1), Source::File { path: original.clone() }, "IMG_1.jpg", "JPEG", 4, 3, "2026-01-01T00:00:00");
s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
assert_eq!(s.save_sidecar(PhotoId(1)).is_ok(), written, "{original}");
assert_eq!(dir.join("IMG_1.xmp").is_file(), written, "{original}");
assert!(!dir.join("IMG_2.xmp").exists(), "{original}");
assert!(!Path::new(r"\\192.0.2.1\share\IMG_1.xmp").exists(), "{original}");
}
std::fs::remove_dir_all(dir).unwrap();
}
/// A sidecar written by Lightroom Classic (names flat, paths in `lr:hierarchicalSubject`)
/// gives the photo its keywords as paths.
#[test]
+1 -1
View File
@@ -903,7 +903,7 @@ fn shadow(img: &mut Rgba8, tl: &crate::text::TextLayout, x: f64, y: f64, sigma:
let my0 = (y - pad).floor().max(-pad);
let mx1 = (x + tl.width as f64 + pad).ceil().min(iw + pad);
let my1 = (y + tl.height as f64 + pad).ceil().min(ih + pad);
if !(mx1 > mx0 && my1 > my0) || !mx0.is_finite() || !my0.is_finite() {
if !(mx1 > mx0 && my1 > my0 && mx0.is_finite() && my0.is_finite()) {
return;
}
let (w, h) = ((mx1 - mx0) as usize, (my1 - my0) as usize);
+3 -1
View File
@@ -115,7 +115,7 @@ fn color(v: &Value) -> std::result::Result<Option<[u8; 3]>, String> {
Value::String(s) if s.is_empty() || s.eq_ignore_ascii_case("none") => Ok(None),
Value::String(s) => {
let h = s.strip_prefix('#').unwrap_or(s);
if h.len() != 6 || !h.is_ascii() {
if h.len() != 6 || !h.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(bad());
}
let byte = |i: usize| h.get(i..i + 2).and_then(|x| u8::from_str_radix(x, 16).ok()).ok_or_else(bad);
@@ -747,6 +747,8 @@ mod tests {
json!({"repeat": 1}),
json!({"fadeColor": "#12345"}),
json!({"fadeColor": "#ééé"}),
json!({"fadeColor": "#+f+f+f"}),
json!({"fadeColor": "-12345"}),
json!({"fadeColor": [300, 0, 0]}),
json!({"fadeColor": [1, 2]}),
json!({"speed": 2}),
+167 -12
View File
@@ -38,7 +38,7 @@
//! not cancelled midway.
use std::collections::HashMap;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use lightcraft_catalog::{Catalog, Filter, Op, Photo, PhotoId, Sort, Source};
@@ -203,18 +203,37 @@ pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges {
let namings = crate::sidecar::namings_in(&labels, naming, &ids);
let photos: Vec<(Arc<Photo>, SidecarNaming)> =
ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
// The folder is walked once, and its listing answers for the library's photos in it: on a
// network share that is one request for a folder's names where it was several for each
// photo. Then both halves wait on the disk at once: the new files are probed (on their own
// threads) beside the checks of the photos, which ask the disk only what the listing can't say.
// (how many photos there are to check is known before the walk: the count shows while it runs)
progress.to_check.store(photos.len(), std::sync::atomic::Ordering::Relaxed);
let mut listing = Listing::default();
let (files, _) = crate::import::expand_noting(
std::slice::from_ref(&folder),
scan.skip(),
crate::walk::Limits::default(),
Some(&mut |dir, names, complete| listing.note(dir, names, complete)),
);
let listing = &listing;
// (no threads on the web: one after the other there)
#[cfg(target_arch = "wasm32")]
let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress));
let (out, checks) = (crate::import::scan_files(scan, files, &progress.files), check_photos(&photos, &labels, listing, progress));
#[cfg(not(target_arch = "wasm32"))]
let (out, checks) = std::thread::scope(|sc| {
let photos = &photos;
let labels = &labels;
let checks = sc.spawn(move || check_photos(photos, labels, progress));
let out = crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files);
let checks = sc.spawn(move || check_photos(photos, labels, listing, progress));
let out = crate::import::scan_files(scan, files, &progress.files);
// (a check that panicked counts as nothing found; the scan still reports the rest)
(out, checks.join().unwrap_or_default())
});
log::debug!(
"synchronize: {} folders listed; the disk was asked {} times about the library's photos",
listing.folders.len(),
listing.asked.load(std::sync::atomic::Ordering::Relaxed)
);
let mut changes = FolderChanges { path: folder, seen, ..Default::default() };
for ((p, _), check) in photos.iter().zip(checks) {
let Some(file) = crate::cmd::missing::checked_path(p) else { continue };
@@ -335,11 +354,136 @@ enum Check {
News,
}
fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
/// The listings of the folders a scan walked: what is in each, by name. It answers "is this file
/// there" for the library's photos and their sidecars without asking the disk, which on a
/// network share is a round trip for every name (and a photo without a sidecar took five).
///
/// It answers only what holds on any file system. A name is there when the folder lists exactly
/// that name ([`asks_the_disk`] aside). It is not there when the folder was listed to its end,
/// every name in the folder is ASCII, the name asked about is plain ASCII ([`plain_name`]), and
/// no name in the folder is that name in another case. Anything else is not answered, and the
/// disk is asked as before: a name that differs in case only, a link, a folder that was not
/// walked or not to its end, and every name that is not listed in a folder that holds a name
/// outside ASCII. (File systems hold names outside ASCII equal in more ways than can be listed
/// here: by case, where the capital sharp s is `SS`; by how accents are written; and some pass
/// over marks altogether.)
///
/// Where the listing and a look-up would differ, the listing is the folder as the scan saw it:
/// a file it lists is there though it could not be opened by its path (a folder that may be
/// listed but not searched, a path longer than the system takes), and a share that drops after
/// the walk does not turn every photo the listing answers for into a missing one.
#[derive(Default)]
pub(crate) struct Listing {
folders: HashMap<PathBuf, Listed>,
/// How often the disk was asked after all (for the tests, and the log).
pub(crate) asked: std::sync::atomic::AtomicUsize,
}
/// One folder's listing.
struct Listed {
/// name → what it is.
names: HashMap<String, crate::walk::EntryKind>,
/// The names, lower-cased (only kept for a folder whose names are all ASCII): what a plain
/// name asked about could be in another case.
lower: std::collections::HashSet<String>,
/// Every entry was read, and every name is ASCII: a plain name that is not listed, in any
/// case, is not there.
sure: bool,
}
/// Whether the disk is asked about `name` whatever the listing holds: a name Windows takes for
/// a device (`NUL`, `aux.jpg`: there in every folder, or not openable where it is listed).
fn asks_the_disk(name: &str) -> bool {
let stem = name.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase();
// (`COM` and `LPT` with any one digit, the superscript ones Windows also takes among them)
let numbered = ["com", "lpt"].iter().any(|port| {
let mut rest = stem.strip_prefix(port).unwrap_or_default().chars();
matches!((rest.next(), rest.next()), (Some(d), None) if d.is_ascii_digit() || ['\u{b9}', '\u{b2}', '\u{b3}'].contains(&d))
});
numbered || ["con", "prn", "aux", "nul", "conin$", "conout$", "clock$"].contains(&stem.as_str())
}
/// Whether `name` can only be found under itself in some case: ASCII, and nothing in it that a
/// file system resolves another way (a short name with `~`, a stream after `:`, dots or spaces
/// at its end, which Windows drops).
fn plain_name(name: &str) -> bool {
name.is_ascii() && !name.contains(['~', ':']) && !name.ends_with(['.', ' ']) && !name.is_empty()
}
impl Listing {
/// Take the listing of `folder` from the walk ([`crate::walk::files_in_noting`]).
pub(crate) fn note(&mut self, folder: &Path, entries: Vec<(std::ffi::OsString, crate::walk::EntryKind)>, complete: bool) {
let mut sure = complete;
let mut names = HashMap::with_capacity(entries.len());
let mut lower = std::collections::HashSet::new();
for (name, kind) in entries {
match name.into_string() {
Ok(name) => {
sure &= name.is_ascii();
lower.insert(name.to_ascii_lowercase());
names.insert(name, kind);
}
// (a name that is no text is not one a photo's path can be compared with)
Err(_) => sure = false,
}
}
if !sure {
lower.clear();
}
self.folders.insert(folder.to_path_buf(), Listed { names, lower, sure });
}
/// What the listing of `path`'s folder says of it: its kind when it is listed under exactly
/// this name, `Some(None)` when it is surely not there, `None` when the listing can't say.
fn kind(&self, path: &Path) -> Option<Option<crate::walk::EntryKind>> {
let name = path.file_name()?.to_str()?;
// (the path ends in that name: no `/.` or separator after it, which the disk would resolve)
if !path.to_str()?.ends_with(name) {
return None;
}
if asks_the_disk(name) {
return None;
}
let folder = self.folders.get(path.parent()?)?;
if let Some(kind) = folder.names.get(name) {
return Some(Some(*kind));
}
(folder.sure && plain_name(name) && !folder.lower.contains(&name.to_ascii_lowercase())).then_some(None)
}
/// `Path::exists`, where the listing can say.
pub(crate) fn exists(&self, path: &Path) -> Option<bool> {
match self.kind(path)? {
Some(crate::walk::EntryKind::Other) => None,
Some(_) => Some(true),
None => Some(false),
}
}
/// `Path::is_file`, where the listing can say.
pub(crate) fn is_file(&self, path: &Path) -> Option<bool> {
match self.kind(path)? {
Some(crate::walk::EntryKind::Other) => None,
Some(kind) => Some(kind == crate::walk::EntryKind::File),
None => Some(false),
}
}
/// The listing's answer, or the disk's where it has none.
fn or_ask(&self, answer: Option<bool>, ask: impl FnOnce() -> bool) -> bool {
answer.unwrap_or_else(|| {
self.asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
ask()
})
}
}
fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog, listing: &Listing) -> Check {
let Some(file) = crate::cmd::missing::checked_path(p) else { return Check::Nothing };
if !Path::new(file).exists() {
let file = Path::new(file);
if !listing.or_ask(listing.exists(file), || file.exists()) {
Check::Missing
} else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming) {
} else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming, listing) {
Check::News
} else {
Check::Nothing
@@ -348,12 +492,12 @@ fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
/// [`check_photo`] for each photo, in order, on several threads (each file is a wait on the
/// disk, not work for the CPU). Photos not reached before a cancel count as nothing found.
fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, progress: &SyncProgress) -> Vec<Check> {
fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, listing: &Listing, progress: &SyncProgress) -> Vec<Check> {
use std::sync::atomic::Ordering::Relaxed;
progress.to_check.store(photos.len(), Relaxed);
let cancel = &progress.files.cancel;
let check = |p: &Photo, naming: SidecarNaming| {
let c = check_photo(p, naming, labels);
let c = check_photo(p, naming, labels, listing);
progress.checked.fetch_add(1, Relaxed);
c
};
@@ -421,10 +565,17 @@ impl Session {
/// Whether `p`'s XMP sidecar file was saved after the library last had its say about the photo
/// (imported, or edited here) and says something the library doesn't.
fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming) -> bool {
fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming, listing: &Listing) -> bool {
let Source::File { path } = &p.source else { return false };
let Some(file) = crate::sidecar::find_sidecar(path, naming) else { return false };
let Ok(meta) = std::fs::metadata(&file) else { return false };
// ([`crate::sidecar::find_sidecar`], each name looked up in the listing before the disk)
// (a name the listing holds is read for its date next, and one that can't be read is passed
// over for the next name, as a name that is no file was)
let found = crate::sidecar::sidecar_candidates(path, naming).find_map(|c| {
let there = listing.or_ask(listing.is_file(&c), || c.is_file());
let meta = there.then(|| std::fs::metadata(&c).ok()).flatten().filter(std::fs::Metadata::is_file)?;
Some((c, meta))
});
let Some((file, meta)) = found else { return false };
if meta.len() > MAX_SIDECAR_BYTES {
return false;
}
@@ -708,3 +859,7 @@ impl SyncCommit {
self.report
}
}
#[cfg(test)]
#[path = "sync/tests_listing.rs"]
mod tests_listing;
+192
View File
@@ -0,0 +1,192 @@
//! Synchronize Folder answers "is the photo's file there" and "has it a sidecar" from the
//! listing of the folder its scan has just made, instead of asking the disk about every photo.
//!
//! On a network share each question is a round trip (about 5 ms a file on a hard disk over SMB,
//! measured), and a photo without a sidecar took five of them; the listing has every name.
//! The listing answers only what it can be sure of on any file system: where a name could match
//! another way (case, letters outside ASCII, a link), the disk is asked as before.
use super::*;
use crate::walk::EntryKind::{File, Folder, Other};
fn listing(folders: &[(&str, &[(&str, crate::walk::EntryKind)], bool)]) -> Listing {
let mut l = Listing::default();
for (dir, names, complete) in folders {
l.note(Path::new(dir), names.iter().map(|(n, k)| (std::ffi::OsString::from(n), *k)).collect(), *complete);
}
l
}
/// Given the listing of a folder, when it is asked about a name, then it says "there" for a
/// name it lists, "not there" for one nothing in the folder could be taken for, and nothing
/// (ask the disk) wherever a file system could see it otherwise.
#[test]
fn the_listing_answers_only_what_it_is_sure_of() {
let l = listing(&[
(
"/pics/trip",
&[("IMG_1.CR3", File), ("IMG_1.xmp", File), ("img_2.jpg", File), ("sub", Folder), ("link.jpg", Other), (".hidden.xmp", File)],
true,
),
("/pics/cut", &[("IMG_1.CR3", File)], false),
("/pics/names", &[("IMG_1.CR3", File), ("Größe.JPG", File)], true),
// (the capital sharp s, which a file system that folds case takes for `SS`)
("/pics/sharp", &[("IMG_1.CR3", File), ("STRA\u{1e9e}E.jpg", File)], true),
("/pics/devices", &[("aux.jpg", File), ("COM\u{b9}.jpg", File), ("IMG_1.CR3", File)], true),
]);
let (there, gone, ask) = (Some(true), Some(false), None);
for (path, exists, is_file) in [
("/pics/trip/IMG_1.CR3", there, there),
("/pics/trip/IMG_1.xmp", there, there),
("/pics/trip/.hidden.xmp", there, there),
("/pics/trip/sub", there, gone),
("/pics/trip/IMG_9.CR3", gone, gone),
("/pics/trip/IMG_1.XMP", ask, ask),
("/pics/trip/IMG_2.JPG", ask, ask),
("/pics/trip/link.jpg", ask, ask),
("/pics/trip/SUB", ask, ask),
// a folder the walk did not list to its end, or not at all
("/pics/cut/IMG_1.CR3", there, there),
("/pics/cut/IMG_2.CR3", ask, ask),
("/pics/other/IMG_1.CR3", ask, ask),
// a folder that holds a name outside ASCII: what it lists is there, and nothing is surely not there (file
// systems hold such names equal in more ways than one)
("/pics/names/IMG_1.CR3", there, there),
("/pics/names/Größe.JPG", there, there),
("/pics/names/IMG_2.CR3", ask, ask),
("/pics/names/größe.JPG", ask, ask),
("/pics/trip/Größe.JPG", ask, ask),
("/pics/sharp/strasse.jpg", ask, ask),
("/pics/sharp/STRASSE.xmp", ask, ask),
("/pics/sharp/IMG_2.CR3", ask, ask),
// names Windows resolves another way: a short name, a stream, dots or spaces at the end, a device
("/pics/trip/IMG_10~1.CR3", ask, ask),
("/pics/trip/IMG_1.CR3::$DATA", ask, ask),
("/pics/trip/IMG_1.CR3.", ask, ask),
("/pics/trip/IMG_1.CR3 ", ask, ask),
("/pics/devices/aux.jpg", ask, ask),
("/pics/devices/NUL", ask, ask),
("/pics/devices/nul.xmp", ask, ask),
("/pics/devices/COM1 .jpg", ask, ask),
("/pics/devices/COM\u{b9}.jpg", ask, ask),
("/pics/trip/con", ask, ask),
("/pics/trip/PRN.txt", ask, ask),
("/pics/trip/com2.jpg", ask, ask),
("/pics/trip/LPT9.xmp", ask, ask),
("/pics/trip/lpt0", ask, ask),
("/pics/trip/nul.tar.gz", ask, ask),
("/pics/trip/CONIN$", ask, ask),
("/pics/trip/com10.jpg", gone, gone),
("/pics/trip/computer.jpg", gone, gone),
("/pics/devices/IMG_1.CR3", there, there),
("/pics/devices/auxiliary.jpg", ask, ask),
("/pics/trip/auxiliary.jpg", gone, gone),
// a path that does not end in its file's name is the disk's to resolve
("/pics/trip/IMG_1.CR3/", ask, ask),
("/pics/trip/IMG_1.CR3/.", ask, ask),
("/pics/trip/", ask, ask),
("", ask, ask),
] {
assert_eq!((l.exists(Path::new(path)), l.is_file(Path::new(path))), (exists, is_file), "{path:?}");
}
// a name that is no text: nothing in its folder is surely not there
#[cfg(unix)]
{
use std::os::unix::ffi::OsStringExt;
let mut l = Listing::default();
l.note(
Path::new("/pics/raw"),
vec![(std::ffi::OsString::from("IMG_1.CR3"), File), (std::ffi::OsString::from_vec(vec![0x66, 0xff, 0x6f]), File)],
true,
);
assert_eq!((l.exists(Path::new("/pics/raw/IMG_1.CR3")), l.exists(Path::new("/pics/raw/IMG_2.CR3"))), (there, ask));
}
}
fn photo(c: &mut Catalog, path: &Path) -> Arc<Photo> {
let id = c.alloc_photo_id();
let name = path.file_name().unwrap().to_string_lossy().to_string();
c.apply(lightcraft_catalog::Op::AddPhoto {
photo: Box::new(Photo::new(id, Source::File { path: path.to_string_lossy().to_string() }, &name, "JPEG", 6, 4, "2020-01-01T00:00:00")),
})
.unwrap();
Arc::clone(c.photo(id).unwrap())
}
/// Given a folder on disk whose photos are there or gone, with sidecars under every name a
/// sidecar can have, when each photo is checked with the folder's listing, then the answer is
/// the one the disk gives; and for photos with plain names the disk was asked nothing but the
/// sidecars' dates.
#[test]
fn a_check_from_the_listing_is_the_check_from_the_disk() {
let dir = std::env::temp_dir().join(format!("lc-listing-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("sub")).unwrap();
let news = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:xmp="http://ns.adobe.com/xap/1.0/" xmp:Rating="5"/></rdf:RDF></x:xmpmeta>"#;
for name in [
"plain.jpg",
"stem.jpg",
"stem.xmp",
"full.jpg",
"full.jpg.xmp",
"upper.jpg",
"upper.XMP",
"sub/deep.jpg",
"sub/deep.xmp",
"Größe.jpg",
"Größe.xmp",
"dir.jpg.xmp",
] {
std::fs::write(dir.join(name), if name.ends_with("mp") || name.ends_with("MP") { news.as_bytes() } else { b"jpeg" }).unwrap();
}
let mut c = Catalog::new();
let photos: Vec<Arc<Photo>> =
["plain.jpg", "stem.jpg", "full.jpg", "upper.jpg", "sub/deep.jpg", "Größe.jpg", "gone.jpg", "sub/gone.jpg", "dir.jpg"]
.iter()
.map(|n| photo(&mut c, &dir.join(n)))
.collect();
// (`dir.jpg` is gone, though a sidecar of its name is there)
let mut l = Listing::default();
let walked =
crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| l.note(d, names, complete));
assert_eq!(walked.files.len(), 12);
let nothing = Listing::default();
for naming in [SidecarNaming::Stem, SidecarNaming::Full] {
for p in &photos {
let from_disk = check_photo(p, naming, &c, &nothing);
assert_eq!(check_photo(p, naming, &c, &l), from_disk, "{:?} {naming:?}", p.file_name);
}
}
let expected = [Check::Nothing, Check::News, Check::News, Check::News, Check::News, Check::News, Check::Missing, Check::Missing, Check::Missing];
assert_eq!(photos.iter().map(|p| check_photo(p, SidecarNaming::Stem, &c, &l)).collect::<Vec<_>>(), expected);
// what the disk was asked about the same photos, with the listing and without: fewer questions with it
let asked = |l: &Listing| {
for p in &photos {
check_photo(p, SidecarNaming::Stem, &c, l);
}
l.asked.load(std::sync::atomic::Ordering::Relaxed)
};
let mut with = Listing::default();
crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| with.note(d, names, complete));
let (with, without) = (asked(&with), asked(&Listing::default()));
// (without: each photo's file, and the names of its sidecar up to the one that is there: 20 questions where
// names are found in any case, 21 where they are not)
assert!(without >= 20, "{without}");
// (with: this folder holds `Größe.*`, so what it does not list is still asked about: the nine names of the
// sidecar `plain.jpg` has not (`.xmp`, `.XMP` and `.Xmp` of `plain`, `plain.jpg` and `plain.JPG`), the three
// `full.jpg` has not, `upper.xmp`, and the two photos that are gone. What it lists is not asked about, nor
// anything in `sub`, whose names are all ASCII)
assert_eq!(with, 15, "asked {with} times with the listing, {without} without");
// a folder of plain names only: the disk is asked nothing at all
let sub = dir.join("sub");
let mut only_plain = Listing::default();
crate::walk::files_in_noting(&sub, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| only_plain.note(d, names, complete));
let mut c2 = Catalog::new();
for (name, expected) in [("deep.jpg", Check::News), ("gone.jpg", Check::Missing), ("gone.jpg.xmp", Check::Missing)] {
let p = photo(&mut c2, &sub.join(name));
assert_eq!(check_photo(&p, SidecarNaming::Stem, &c2, &only_plain), expected, "{name}");
}
assert_eq!(only_plain.asked.load(std::sync::atomic::Ordering::Relaxed), 0, "every answer came from the listing");
let _ = std::fs::remove_dir_all(&dir);
}
+67
View File
@@ -313,6 +313,44 @@ fn permanently_deleting_explicit_targets_keeps_the_surviving_selection() {
assert_eq!(s.catalog.to_snapshot(), snapshot);
}
/// `mask.brushStroke {connect: true}` (a ⇧-click) starts from the last point of the mask's last stroke: one straight
/// stroke when the brush is unchanged, pieces whose size, feather and flow step from the last stroke's to the new
/// values when it changed, and a plain dab when there is nothing to connect to.
#[test]
fn connected_brush_strokes_start_where_the_last_one_ended() {
use lightcraft_develop::MaskShape;
use lightcraft_geom::Point;
let mut s = demo();
let id = s.active().unwrap();
let r = s.execute("mask.brushStroke", &json!({"points": [[0.2, 0.5]], "size": 0.02, "connect": true})).unwrap();
assert_eq!(r["connected"], false, "nothing to connect to: a dab");
let strokes = |s: &Session| {
let d = s.develop_of(id).unwrap();
let MaskShape::Brush { strokes } = &d.masks[0].components[0].shape else { panic!("brush") };
strokes.clone()
};
assert_eq!(strokes(&s)[0].points.len(), 1);
let r = s.execute("mask.brushStroke", &json!({"points": [[0.6, 0.5]], "size": 0.02, "connect": true})).unwrap();
assert_eq!(r["connected"], true);
let line = strokes(&s);
assert_eq!(line.len(), 2);
assert_eq!(line[1].points, vec![Point::new(0.2, 0.5), Point::new(0.6, 0.5)]);
// a bigger, softer brush: the segment tapers from the last stroke's values to the new ones
let r = s.execute("mask.brushStroke", &json!({"points": [[0.6, 0.9]], "size": 0.04, "feather": 80, "connect": true})).unwrap();
assert_eq!(r["connected"], true);
let all = strokes(&s);
let taper = &all[2..];
assert!(taper.len() > 4, "{} pieces", taper.len());
assert_eq!(taper[0].points[0], Point::new(0.6, 0.5), "from the last stroke's end");
assert_eq!(*taper.last().unwrap().points.last().unwrap(), Point::new(0.6, 0.9), "to the click");
assert!(taper.windows(2).all(|w| w[1].size >= w[0].size && w[1].feather >= w[0].feather && w[0].points[1] == w[1].points[0]));
assert!((taper.last().unwrap().size - 0.04).abs() < 1e-12 && (taper.last().unwrap().feather - 80.0).abs() < 1e-9);
assert!(taper[0].size > 0.02 && taper[0].size < 0.04);
// all of it is one undo step
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(strokes(&s).len(), 2);
}
#[test]
fn brush_strokes_keep_pen_pressure_and_refuse_bad_lists() {
let mut s = demo();
@@ -330,6 +368,35 @@ fn brush_strokes_keep_pen_pressure_and_refuse_bad_lists() {
assert!(!json.contains("pressure"), "and stores none: {json}");
}
/// A ⇧-click line carries pen pressure: the last dab's at its start and the click's at its end, stepping along a
/// tapered line; with no pressure on either side there is none.
#[test]
fn connected_brush_strokes_carry_pen_pressure() {
use lightcraft_develop::MaskShape;
let mut s = demo();
let id = s.active().unwrap();
let strokes = |s: &Session| {
let d = s.develop_of(id).unwrap();
let MaskShape::Brush { strokes } = &d.masks[0].components[0].shape else { panic!("brush") };
strokes.clone()
};
s.execute("mask.brushStroke", &json!({"points": [[0.2, 0.5]], "size": 0.02, "pressure": [0.4]})).unwrap();
s.execute("mask.brushStroke", &json!({"points": [[0.6, 0.5]], "size": 0.02, "pressure": [1.0], "connect": true})).unwrap();
assert_eq!(strokes(&s)[1].pressure, vec![0.4, 1.0]);
s.execute("mask.brushStroke", &json!({"points": [[0.6, 0.9]], "size": 0.04, "pressure": [0.2], "connect": true})).unwrap();
let taper = &strokes(&s)[2..];
assert!(taper.iter().all(|t| t.pressure.len() == 2));
assert!((taper[0].pressure[0] - 1.0).abs() < 1e-6 && (taper.last().unwrap().pressure[1] - 0.2).abs() < 1e-6);
assert!(taper.windows(2).all(|w| w[1].pressure[0] <= w[0].pressure[0]));
// a mouse: no pressure anywhere
let mut m = demo();
m.execute("mask.brushStroke", &json!({"points": [[0.2, 0.5]], "size": 0.02})).unwrap();
m.execute("mask.brushStroke", &json!({"points": [[0.6, 0.5]], "size": 0.02, "connect": true})).unwrap();
let d = m.develop_of(m.active().unwrap()).unwrap();
let MaskShape::Brush { strokes } = &d.masks[0].components[0].shape else { panic!("brush") };
assert!(strokes[1].pressure.is_empty());
}
#[test]
fn masks_crop_and_render() {
let mut s = demo();
+12 -9
View File
@@ -1024,21 +1024,24 @@ fn same_folder_as_original_refuses_photos_without_a_folder() {
let mut s = Session::with_demo();
let ids: Vec<_> = s.catalog.photos().take(2).map(|p| p.id).collect();
let mut written = Vec::new();
let mut write = |p: &str, _: &[u8]| -> Result<(), String> {
written.push(p.to_string());
Ok(())
// the writer lives in a block of its own, so its borrow of `written` has ended by the time
// the assertion below reads it
let files = {
let mut write = |p: &str, _: &[u8]| -> Result<(), String> {
written.push(p.to_string());
Ok(())
};
let err = crate::export::export_batch(&mut s, &ids, &o, &to, &mut write, &|_| false).unwrap_err();
assert!(err.contains("no folder on disk"), "{err}");
// without stopping: one error per photo, nothing written
let items = crate::export::prepare_batch(&mut s, &ids, &o).unwrap();
crate::export::run_batch(items, &o, &to, &mut write, &|_| false, false, &mut |_, _| true).unwrap()
};
let err = crate::export::export_batch(&mut s, &ids, &o, &to, &mut write, &|_| false).unwrap_err();
assert!(err.contains("no folder on disk"), "{err}");
// without stopping: one error per photo, nothing written
let items = crate::export::prepare_batch(&mut s, &ids, &o).unwrap();
let files = crate::export::run_batch(items, &o, &to, &mut write, &|_| false, false, &mut |_, _| true).unwrap();
assert_eq!(files.len(), 2);
for (f, id) in files.iter().zip(&ids) {
assert_eq!(f["photo"], id.0, "{f}");
assert!(f["error"].as_str().unwrap().contains("no folder on disk"), "{f}");
}
drop(write);
assert!(written.is_empty(), "{written:?}");
// a photo on disk whose folder is gone (moved, an unplugged drive)
+14
View File
@@ -627,3 +627,17 @@ fn recognition_cannot_be_turned_on_without_a_chosen_model() {
assert!(s.execute("faces.enable", &json!({"enabled": true})).is_err());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_cpu_only_switch_is_kept_and_bad_values_are_refused() {
let d = temp("cpu-only");
let mut s = session(&d);
assert_eq!(s.execute("faces.gpu", &json!({})).unwrap()["cpuOnly"], false);
assert_eq!(s.execute("faces.gpu", &json!({"cpuOnly": true})).unwrap()["cpuOnly"], true);
// kept with the models, so a new session starts on the CPU too
let mut again = session(&d);
assert_eq!(again.execute("faces.models.list", &json!({})).unwrap()["gpu"]["cpuOnly"], true);
assert!(again.execute("faces.gpu", &json!({"cpuOnly": "yes"})).is_err());
assert_eq!(again.execute("faces.gpu", &json!({"cpuOnly": false})).unwrap()["cpuOnly"], false);
let _ = std::fs::remove_dir_all(d);
}
+297
View File
@@ -0,0 +1,297 @@
//! Reading focus points for photos already in the library (`photo.readFocusPoints`, `focus.pump`): header-only reads,
//! batches committed without undo steps, "looked, found nothing" remembered across sessions, and the activity row.
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use lightcraft_catalog::{Op, Photo, PhotoId, Source};
use serde_json::{Value, json};
use crate::Session;
use crate::focus_scan::{read_focus, record_contains, record_len};
fn temp_dir(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-focus-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn sony_arw(mode: u8, samples: usize) -> Vec<u8> {
lightcraft_meta::focus::synthetic_arw(mode, samples)
}
/// A photo of `camera` at `path` (an import made before focus points were read: none stored).
fn photo(id: u64, path: &Path, camera: &str) -> Photo {
let name = path.file_name().unwrap().to_string_lossy().into_owned();
let mut p = Photo::new(PhotoId(id), Source::File { path: path.to_string_lossy().into_owned() }, &name, "ARW", 8640, 5760, "2026-10-13T00:00:00");
p.meta.camera = camera.into();
p
}
fn add(s: &mut Session, photos: Vec<Photo>) {
s.execute_fn("test.add", |s| {
for p in photos {
s.apply_system(Op::AddPhoto { photo: Box::new(p) })?;
}
Ok(Value::Null)
})
.unwrap();
}
fn focus(s: &Session, id: u64) -> Option<lightcraft_meta::FocusPoint> {
s.catalog.photo(PhotoId(id)).unwrap().meta.focus.clone()
}
fn pump_until_idle(s: &mut Session) -> Value {
let started = Instant::now();
loop {
let v = s.execute("focus.pump", &json!({"pace": "normal"})).unwrap();
if v["active"] == false {
return v;
}
assert!(started.elapsed() < Duration::from_secs(30), "the job never ended: {v}");
std::thread::sleep(Duration::from_millis(5));
}
}
#[test]
fn the_header_read_reads_a_small_part_of_the_file() {
let dir = temp_dir("header");
let path = dir.join("a.ARW");
let file = sony_arw(3, 4_000_000);
std::fs::write(&path, &file).unwrap();
let r = read_focus(&path.to_string_lossy()).unwrap();
assert!(r.focus.is_some());
assert_eq!(r.focus, lightcraft_meta::extract(&file).focus, "the same point as a whole-file read");
assert!(r.bytes_read <= 32 * 1024 && r.bytes_read * 100 < file.len() as u64, "read {} of {}", r.bytes_read, file.len());
assert!(read_focus(&dir.join("gone.ARW").to_string_lossy()).is_err());
}
/// Fills the missing points; leaves other cameras, other formats and known points alone without opening them; records
/// "looked, found nothing" (manual focus, garbage) but not a file it could not read; makes no undo step.
#[test]
fn fills_missing_points_and_remembers_photos_without_one() {
let dir = temp_dir("fill");
let af = dir.join("af.ARW");
std::fs::write(&af, sony_arw(3, 100_000)).unwrap();
let manual = dir.join("manual.ARW");
std::fs::write(&manual, sony_arw(0, 100_000)).unwrap();
let garbage = dir.join("garbage.ARW");
std::fs::write(&garbage, vec![0x5a; 70_000]).unwrap();
let heif = dir.join("x.HIF");
std::fs::write(&heif, sony_arw(3, 1000)).unwrap();
let known = lightcraft_meta::FocusPoint {
rect: lightcraft_geom::Rect { x0: 0.1, y0: 0.1, x1: 0.2, y1: 0.2 },
mode: Some("AF-S".into()),
area: None,
tracking: None,
};
let mut with_point = photo(4, &af, "SONY ILCE-1");
with_point.meta.focus = Some(known.clone());
let mut s = Session::new();
add(
&mut s,
vec![
photo(1, &af, "SONY ILCE-1"),
photo(2, &manual, "SONY ILCE-1"),
// another body's photo is not opened, even when its file would have a point
photo(3, &af, "Canon EOS R5"),
with_point,
photo(5, &dir.join("offline.ARW"), "SONY ILCE-7SM3"),
photo(6, &garbage, "SONY ILCE-1"),
photo(7, &heif, "SONY ILCE-1"),
],
);
let undo = s.undo.len();
let r = s.execute("photo.readFocusPoints", &json!({"wait": true})).unwrap();
assert_eq!(r["queued"], 4, "{r}");
let last = &r["last"];
assert_eq!(
(last["done"].as_u64(), last["found"].as_u64(), last["none"].as_u64(), last["unavailable"].as_u64()),
(Some(4), Some(1), Some(2), Some(1)),
"{r}"
);
assert!(last["bytesRead"].as_u64().unwrap() <= 3 * 32 * 1024, "{r}");
assert_eq!(focus(&s, 1), lightcraft_meta::extract(&std::fs::read(&af).unwrap()).focus);
assert!(focus(&s, 1).is_some());
assert_eq!(focus(&s, 1).unwrap().words(), vec!["AF-C", "Wide", "Animal Eye Tracking"]);
assert_eq!(focus(&s, 2), None);
assert_eq!(focus(&s, 3), None);
assert_eq!(focus(&s, 4), Some(known), "a stored point is never replaced");
assert_eq!(focus(&s, 5), None);
assert_eq!(focus(&s, 7), None);
assert!(record_contains(&s, PhotoId(2)) && record_contains(&s, PhotoId(6)));
assert!(!record_contains(&s, PhotoId(5)), "an unreadable file is tried again later");
assert_eq!(record_len(&s), 2);
assert_eq!(s.undo.len(), undo, "no undo step");
// asked again: only the offline photo is left
assert_eq!(s.execute("photo.readFocusPoints", &json!({"wait": true})).unwrap()["queued"], 1);
// named photos are looked at again, even those recorded as having none
assert_eq!(s.execute("photo.readFocusPoints", &json!({"ids": [2, 3, 4], "wait": true})).unwrap()["queued"], 1);
}
/// What was read survives a restart: the next session reads nothing again, and a photo whose file was offline is read
/// once it is back. The automatic start (once per library opening, from the pump) picks it up.
#[test]
fn resumes_after_a_restart_without_reading_again() {
let dir = temp_dir("resume");
let lib = dir.join("library");
let af = dir.join("af.ARW");
std::fs::write(&af, sony_arw(3, 50_000)).unwrap();
let manual = dir.join("manual.ARW");
std::fs::write(&manual, sony_arw(0, 50_000)).unwrap();
let later = dir.join("later.ARW");
{
let mut s = Session::new();
s.open_library(&lib, false).unwrap();
add(&mut s, vec![photo(1, &af, "SONY ILCE-1"), photo(2, &manual, "SONY ILCE-1"), photo(3, &later, "SONY ILCE-1")]);
// the pump starts it by itself
let first = s.execute("focus.pump", &json!({"pace": "normal"})).unwrap();
assert_eq!(first["active"], true, "{first}");
let v = pump_until_idle(&mut s);
assert_eq!(v["last"]["found"], 1, "{v}");
assert!(lib.join(crate::focus_scan::RECORD_FILE).exists());
}
let mut s = Session::new();
s.open_library(&lib, false).unwrap();
assert!(focus(&s, 1).is_some(), "the point was saved with the library");
// the file that was offline is back: only it is read
std::fs::write(&later, sony_arw(3, 50_000)).unwrap();
let v = s.execute("focus.pump", &json!({"pace": "normal"})).unwrap();
assert!(record_contains(&s, PhotoId(2)), "the record was read back");
assert_eq!(v["running"]["total"], 1, "{v}");
let v = pump_until_idle(&mut s);
assert_eq!((v["last"]["total"].as_u64(), v["last"]["found"].as_u64()), (Some(1), Some(1)), "{v}");
assert!(focus(&s, 3).is_some());
assert!(record_contains(&s, PhotoId(2)));
// and it doesn't start again in this opening
assert_eq!(s.execute("focus.pump", &json!({})).unwrap()["active"], false);
}
/// The job has a row in the activity stack (n of N, cancellable); paused, it reads nothing new; cancelled (the row's ✕),
/// it commits what it read and the row goes away.
#[test]
fn the_activity_row_progresses_pauses_and_cancels() {
let dir = temp_dir("row");
let mut photos = Vec::new();
for i in 0..40u64 {
let p = dir.join(format!("{i:03}.ARW"));
std::fs::write(&p, sony_arw(3, 20_000)).unwrap();
photos.push(photo(i + 1, &p, "SONY ILCE-7SM3"));
}
let mut s = Session::new();
add(&mut s, photos);
// the pump starts the job by itself, at the pace it is given: paused, while the user drags
let v = s.execute("focus.pump", &json!({"pace": "pause"})).unwrap();
assert_eq!(v["active"], true);
let rows = s.activity.list();
let row = rows.iter().find(|t| t.kind == "focus").expect("an activity row");
assert_eq!((row.label.as_str(), row.total, row.cancellable), ("Reading focus points", 40, true));
// already running: asking again starts nothing new
let r = s.execute("photo.readFocusPoints", &json!({})).unwrap();
assert_eq!((r["queued"].as_u64(), r["running"].as_bool()), (Some(0), Some(true)));
std::thread::sleep(Duration::from_millis(300));
let still = s.execute("focus.pump", &json!({"pace": "pause"})).unwrap()["running"]["done"].as_u64().unwrap();
assert_eq!(still, 0, "paused: nothing read");
// a few more at a light pace, then cancel
let started = Instant::now();
let mut done = still;
while done < still + 3 && started.elapsed() < Duration::from_secs(20) {
std::thread::sleep(Duration::from_millis(20));
done = s.execute("focus.pump", &json!({"pace": "light"})).unwrap()["running"]["done"].as_u64().unwrap_or(40);
}
let row = s.activity.list().into_iter().find(|t| t.kind == "focus").expect("still running");
assert!(row.done >= 3, "progress shows: {}", row.done);
s.execute("focus.pump", &json!({"pace": "pause"})).unwrap();
s.activity.cancel(row.id).unwrap();
let v = s.execute("focus.pump", &json!({"pace": "pause"})).unwrap();
assert_eq!(v["active"], false, "{v}");
assert_eq!(v["last"]["cancelled"], true, "{v}");
assert!(s.activity.list().iter().all(|t| t.kind != "focus"), "the row is gone");
let found = s.catalog.photos().filter(|p| p.meta.focus.is_some()).count() as u64;
assert_eq!(found, v["last"]["found"].as_u64().unwrap(), "what was read before the cancel is kept");
assert!(found < 40);
// the rest are read next time
let r = s.execute("photo.readFocusPoints", &json!({"wait": true})).unwrap();
assert_eq!(r["queued"].as_u64().unwrap(), 40 - found);
assert_eq!(s.catalog.photos().filter(|p| p.meta.focus.is_some()).count(), 40);
}
/// Photos deleted, moved or given a point while the job runs are left alone; a library closed mid-run takes the job
/// with it (nothing lands in the next library), and nothing panics.
#[test]
fn changes_under_a_running_job_are_respected() {
let dir = temp_dir("changes");
let mut photos = Vec::new();
for i in 0..6u64 {
let p = dir.join(format!("{i}.ARW"));
std::fs::write(&p, sony_arw(3, 10_000)).unwrap();
photos.push(photo(i + 1, &p, "SONY ILCE-1"));
}
let mut s = Session::new();
s.open_library(dir.join("lib-a"), false).unwrap();
add(&mut s, photos);
// started by the pump, paused
assert_eq!(s.execute("focus.pump", &json!({"pace": "pause"})).unwrap()["running"]["total"], 6);
// meanwhile: photo 1 removed, photo 2 relinked elsewhere, photo 3 given a point by the user's Reload, photo 4's file
// replaced by garbage, photo 5's deleted
let mine =
lightcraft_meta::FocusPoint { rect: lightcraft_geom::Rect { x0: 0.5, y0: 0.5, x1: 0.6, y1: 0.6 }, mode: None, area: None, tracking: None };
s.execute_fn("test.change", |s| {
s.apply_system(Op::RemovePhoto { id: PhotoId(1) })?;
let moved = dir.join("moved.ARW");
std::fs::copy(dir.join("1.ARW"), &moved).map_err(|e| crate::EngineError::Other(e.to_string()))?;
s.apply_system(Op::Relink {
id: PhotoId(2),
file_name: "moved.ARW".into(),
source: Source::File { path: moved.to_string_lossy().into() },
format: None,
})?;
let mut meta = s.catalog.photo(PhotoId(3)).unwrap().meta.clone();
meta.focus = Some(mine.clone());
s.apply_system(Op::SetMeta { id: PhotoId(3), meta: Box::new(meta) })?;
Ok(Value::Null)
})
.unwrap();
std::fs::write(dir.join("3.ARW"), b"not a raw any more").unwrap();
std::fs::remove_file(dir.join("4.ARW")).unwrap();
let v = pump_until_idle(&mut s);
assert!(s.catalog.photo(PhotoId(1)).is_none());
assert_eq!(focus(&s, 2), None, "read at the old path: left for another run");
assert_eq!(focus(&s, 3), Some(mine));
assert_eq!(focus(&s, 5), None);
assert!(focus(&s, 6).is_some());
assert!(v["last"]["done"].as_u64().unwrap() <= 6, "{v}");
// a library closed mid-run: the job goes with it
s.execute("focus.pump", &json!({"pace": "pause"})).unwrap();
assert_eq!(s.execute("photo.readFocusPoints", &json!({"ids": [2, 4, 5]})).unwrap()["queued"], 3);
s.open_library(dir.join("lib-b"), false).unwrap();
let v = s.execute("focus.pump", &json!({"pace": "normal"})).unwrap();
assert_eq!(v["active"], false, "{v}");
assert!(s.activity.list().iter().all(|t| t.kind != "focus"));
assert_eq!(s.catalog.photos().count(), 0);
}
/// Opt-in measurement on real files: `LIGHTCRAFT_FOCUS_FILES=<a text file listing raw paths, one per line>` reads each
/// whole and by header, asserts they give the same focus point, and prints the bytes the header read took. Skipped
/// without it; no user file is part of the repository.
#[test]
fn real_files_header_read_matches_whole_file() {
let Ok(list) = std::env::var("LIGHTCRAFT_FOCUS_FILES") else { return };
let paths: Vec<String> = std::fs::read_to_string(list).unwrap().lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect();
let (mut n, mut with_point, mut bytes, mut max) = (0u64, 0u64, 0u64, 0u64);
for p in &paths {
let whole = lightcraft_meta::extract(&std::fs::read(p).unwrap()).focus;
let r = read_focus(p).unwrap();
assert_eq!(r.focus, whole, "{p}");
n += 1;
with_point += u64::from(whole.is_some());
bytes += r.bytes_read;
max = max.max(r.bytes_read);
}
eprintln!(
"focus header reads: {n} files ({with_point} with a point) agree with whole-file reads; {} bytes each on average, {max} at most",
bytes / n.max(1)
);
}
+44
View File
@@ -778,6 +778,50 @@ fn assisted_culling_groups_bursts_and_scores_focus() {
let _ = std::fs::remove_dir_all(&dir);
}
/// Assisted culling: two look-alike shots one second apart across midnight form a burst at the end of
/// every month, including those with fewer than 31 days.
#[test]
fn assisted_culling_groups_bursts_across_month_end() {
let dir = temp_dir("cull-month-end");
for (name, (w, h)) in [("a.png", (480, 320)), ("b.png", (482, 321))] {
let img = lightcraft_scenes::demo_library()[0].render(w, h);
let data: Vec<[u8; 4]> = img
.data
.iter()
.map(|c| {
let e = |v: f32| (lightcraft_color::transfer::linear_to_srgb(v.clamp(0.0, 1.0)) * 255.0).round() as u8;
[e(c[0]), e(c[1]), e(c[2]), 255]
})
.collect();
let png = lightcraft_codecs::encode_png(
&lightcraft_codecs::EncodeImage::rgba8(&lightcraft_raster::Rgba8 { width: img.width, height: img.height, data }),
&Default::default(),
)
.unwrap();
std::fs::write(dir.join(name), png).unwrap();
}
let mut s = Session::new().with_fs();
let r = s.execute("library.import", &json!({"paths": [dir.to_string_lossy()]})).unwrap();
let ids: Vec<u64> = r["imported"].as_array().unwrap().iter().filter_map(Value::as_u64).collect();
let by_name = |s: &Session, n: &str| s.catalog.photos().find(|p| p.file_name == n).unwrap().id;
let (a, b) = (by_name(&s, "a.png"), by_name(&s, "b.png"));
for (before, after) in [
("2026-01-31T23:59:59", "2026-02-01T00:00:00"),
("2026-02-28T23:59:59", "2026-03-01T00:00:00"),
("2026-04-30T23:59:59", "2026-05-01T00:00:00"),
("2026-06-30T23:59:59", "2026-07-01T00:00:00"),
("2026-09-30T23:59:59", "2026-10-01T00:00:00"),
("2026-11-30T23:59:59", "2026-12-01T00:00:00"),
("2026-12-31T23:59:59", "2027-01-01T00:00:00"),
] {
s.commit("t", lightcraft_catalog::Op::SetCaptured { id: a, captured: Some(before.into()) }).unwrap();
s.commit("t", lightcraft_catalog::Op::SetCaptured { id: b, captured: Some(after.into()) }).unwrap();
let r = s.execute("photo.analyze", &json!({"ids": ids})).unwrap();
assert_eq!(r["groups"], 1, "shots 1 s apart ({before} / {after}) must form a burst: {r}");
}
let _ = std::fs::remove_dir_all(&dir);
}
/// Find Dust Spots: soft dark spots on a smooth sky are found and healed (one undo step).
#[test]
fn dust_spots_are_found_and_healed() {
+286
View File
@@ -0,0 +1,286 @@
//! Moving photos to a folder on a NAS or network share (`photo.moveToFolder`, which dropping them
//! on a Folders row runs), where the share can go away between the move and its undo:
//!
//! * The share offline when Undo runs: Undo is refused with the reason and keeps the step;
//! nothing is deleted, the library still points at the files where they are, and once the share
//! is back the same Undo moves them home.
//! * The share read-only when Undo runs: the same.
//! * The connection lost partway through a move: the files already moved go back, the library is
//! unchanged and no history step is left.
//! * One file the share can't give back during Undo: the files already brought back go out again,
//! so every photo points at its file and the step can run once the share answers.
//! * Across devices (a share is another volume): the file is copied, checked and only then
//! removed; the copy goes to a hidden name and takes the photo's name only once complete.
use lightcraft_catalog::{Op, Photo, PhotoId, Source};
use serde_json::json;
use crate::Session;
/// A scratch folder that goes away with the test, however it ends.
struct Scratch(std::path::PathBuf);
impl Scratch {
fn new(tag: &str) -> Scratch {
let dir = std::env::temp_dir().join(format!("lc-share-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("local")).unwrap();
std::fs::create_dir_all(dir.join("share")).unwrap();
Scratch(dir)
}
fn path(&self, rel: &str) -> String {
self.0.join(rel).components().collect::<std::path::PathBuf>().to_string_lossy().to_string()
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn add_file(s: &mut Session, path: &str, body: &str) -> PhotoId {
std::fs::write(path, body).unwrap();
let id = s.catalog.alloc_photo_id();
let name = std::path::Path::new(path).file_name().unwrap().to_string_lossy().to_string();
let p = Photo::new(id, Source::File { path: path.into() }, &name, "JPEG", 60, 40, "2026-01-01T10:00:00");
s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
id
}
fn file_of(s: &Session, id: PhotoId) -> String {
match &s.catalog.photo(id).unwrap().source {
Source::File { path } => path.clone(),
_ => panic!("not a file"),
}
}
/// Two photos on the local disk moved to the share.
fn moved_to_share(dir: &Scratch) -> (Session, PhotoId, PhotoId) {
let mut s = Session::new();
let a = add_file(&mut s, &dir.path("local/a.jpg"), "A");
std::fs::write(dir.path("local/a.xmp"), "<edits/>").unwrap();
let b = add_file(&mut s, &dir.path("local/b.jpg"), "B");
let r = s.execute("photo.moveToFolder", &json!({"ids": [a.0, b.0], "folder": dir.path("share")})).unwrap();
assert_eq!(r["moved"], 2);
(s, a, b)
}
#[test]
fn undo_with_the_share_offline_is_refused_keeps_the_step_and_works_once_it_is_back() {
let dir = Scratch::new("offline");
let (mut s, a, b) = moved_to_share(&dir);
let steps = s.undo.len();
// the share goes offline: its folder is no longer there
let away = dir.0.join("share-unmounted");
std::fs::rename(dir.0.join("share"), &away).unwrap();
let e = s.execute("edit.undo", &json!({})).unwrap_err().to_string();
assert!(e.contains("can't move the files back"), "says why: {e}");
assert_eq!(s.undo.len(), steps, "the step stays, to undo once the share is back");
// nothing deleted, nothing half-moved, and the library still points at the share
assert!(away.join("a.jpg").is_file() && away.join("a.xmp").is_file() && away.join("b.jpg").is_file());
assert!(!dir.0.join("local/a.jpg").exists() && !dir.0.join("local/b.jpg").exists());
assert_eq!(file_of(&s, a), dir.path("share/a.jpg"));
assert_eq!(file_of(&s, b), dir.path("share/b.jpg"));
// the share is back: the same Undo moves them home, sidecar and all
std::fs::rename(&away, dir.0.join("share")).unwrap();
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(std::fs::read_to_string(dir.path("local/a.jpg")).unwrap(), "A");
assert!(dir.0.join("local/a.xmp").is_file() && dir.0.join("local/b.jpg").is_file());
assert_eq!(file_of(&s, a), dir.path("local/a.jpg"));
assert_eq!(file_of(&s, b), dir.path("local/b.jpg"));
}
#[cfg(unix)]
#[test]
fn undo_from_a_read_only_share_is_refused_and_deletes_nothing() {
use std::os::unix::fs::PermissionsExt;
let dir = Scratch::new("readonly");
let (mut s, a, _) = moved_to_share(&dir);
let steps = s.undo.len();
std::fs::set_permissions(dir.0.join("share"), std::fs::Permissions::from_mode(0o555)).unwrap();
// running as root ignores permissions: nothing to check there
let root = std::fs::write(dir.0.join("share/probe"), "x").is_ok();
if !root {
assert!(s.execute("edit.undo", &json!({})).is_err());
assert_eq!(s.undo.len(), steps, "the step stays");
assert!(dir.0.join("share/a.jpg").is_file() && dir.0.join("share/b.jpg").is_file(), "nothing deleted");
assert!(!dir.0.join("local/a.jpg").exists(), "no second copy left behind");
assert_eq!(file_of(&s, a), dir.path("share/a.jpg"));
}
std::fs::set_permissions(dir.0.join("share"), std::fs::Permissions::from_mode(0o755)).unwrap();
s.execute("edit.undo", &json!({})).unwrap();
assert!(dir.0.join("local/a.jpg").is_file());
}
/// The model volume spells paths with `/`; the code under test joins them with the host's separator.
fn host(path: &str) -> String {
path.replace('/', std::path::MAIN_SEPARATOR_STR)
}
fn model_session(fs_files: &[&str]) -> (Session, Vec<PhotoId>) {
let mut s = Session::new();
let ids = fs_files
.iter()
.map(|path| {
let id = s.catalog.alloc_photo_id();
let name = std::path::Path::new(path).file_name().unwrap().to_string_lossy().to_string();
let p = Photo::new(id, Source::File { path: host(path) }, &name, "JPEG", 1, 1, "2026-01-01T00:00:00");
s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
id
})
.collect();
(s, ids)
}
fn plans(s: &Session, ids: &[PhotoId], into: &str) -> Vec<crate::rename::RenamePlan> {
ids.iter()
.map(|id| {
let from = file_of(s, *id);
let name = std::path::Path::new(&from).file_name().unwrap().to_string_lossy().to_string();
crate::rename::RenamePlan {
id: id.0,
from: name.clone(),
to: name.clone(),
from_path: Some(from),
to_path: Some(host(&format!("{into}/{name}"))),
}
})
.collect()
}
/// Every photo points at a file the model volume holds (or, for `offline`, one it held).
fn assert_linked(s: &Session, fs: &crate::rename::tests::FakeFs, offline: &[&str]) {
let files = fs.listing();
for p in s.catalog.photos() {
let Source::File { path } = &p.source else { continue };
let path = path.replace('\\', "/");
assert!(
files.iter().any(|(f, _)| *f == path) || offline.contains(&path.as_str()),
"photo {} points at {path}, which isn't there: {files:?}",
p.id.0
);
}
}
#[test]
fn a_connection_lost_partway_through_a_move_puts_back_what_moved() {
let fs = crate::rename::tests::FakeFs::new(false, &[("/local/a.jpg", "A"), ("/local/b.jpg", "B"), ("/local/c.jpg", "C")]);
let (mut s, ids) = model_session(&["/local/a.jpg", "/local/b.jpg", "/local/c.jpg"]);
let plans = plans(&s, &ids, "/share");
// the share stops answering when the second file goes
fs.fail.borrow_mut().push("/local/b.jpg".into());
let steps = s.undo.len();
assert!(s.apply_file_plans(&fs, &plans, &|_| "Move to Folder".into()).is_err());
assert_eq!(
fs.listing(),
vec![("/local/a.jpg".into(), "A".into()), ("/local/b.jpg".into(), "B".into()), ("/local/c.jpg".into(), "C".into())],
"all back home"
);
assert_eq!(s.undo.len(), steps, "no step for a move that didn't happen");
assert_linked(&s, &fs, &[]);
}
#[test]
fn a_file_the_share_cannot_give_back_rolls_the_whole_undo_back() {
let fs = crate::rename::tests::FakeFs::new(false, &[("/local/a.jpg", "A"), ("/local/b.jpg", "B")]);
let (mut s, ids) = model_session(&["/local/a.jpg", "/local/b.jpg"]);
let plans = plans(&s, &ids, "/share");
s.apply_file_plans(&fs, &plans, &|_| "Move to Folder".into()).unwrap();
assert_eq!(fs.listing(), vec![("/share/a.jpg".into(), "A".into()), ("/share/b.jpg".into(), "B".into())]);
// b can't be read back: the share answers for a but not for b
fs.fail.borrow_mut().push("/share/b.jpg".into());
let e = s.undo_step_with(&fs).unwrap_err().to_string();
assert!(e.contains("can't move the files back"), "{e}");
// a came back before b failed: it went out again, so the step is whole and every photo
// points at its file, none lost
assert_linked(&s, &fs, &[]);
let contents: Vec<String> = fs.listing().into_iter().map(|(_, c)| c).collect();
assert_eq!(contents, vec!["A".to_string(), "B".to_string()], "both files still exist once: {:?}", fs.listing());
// the share answers again: undoing whatever is left brings both home
fs.fail.borrow_mut().clear();
while s.undo_step_with(&fs).is_ok() && fs.listing().iter().any(|(p, _)| p.starts_with("/share")) {}
assert_eq!(fs.listing(), vec![("/local/a.jpg".into(), "A".into()), ("/local/b.jpg".into(), "B".into())]);
assert_linked(&s, &fs, &[]);
}
/// Where this machine has a second file system to write to (Linux's `/dev/shm`), a real move
/// across devices: copied, checked, the original removed, and Undo copies it back. Skipped where
/// there is none (a network share can't be mounted in a test).
#[test]
fn a_move_across_devices_copies_checks_and_removes_and_undoes_the_same_way() {
let dir = Scratch::new("devices");
let Some(other) = other_device(&dir.0) else { return };
let share = other.join(format!("lc-share-other-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&share);
std::fs::create_dir_all(&share).unwrap();
let mut s = Session::new();
let body: String = (0..300_000).map(|i| char::from(b'a' + (i % 26) as u8)).collect();
let a = add_file(&mut s, &dir.path("local/a.jpg"), &body);
let r = s.execute("photo.moveToFolder", &json!({"ids": [a.0], "folder": share.to_string_lossy()}));
let moved =
r.is_ok() && std::fs::read_to_string(share.join("a.jpg")).ok().as_deref() == Some(body.as_str()) && !dir.0.join("local/a.jpg").exists();
let undone = moved
&& s.execute("edit.undo", &json!({})).is_ok()
&& std::fs::read_to_string(dir.0.join("local/a.jpg")).ok().as_deref() == Some(body.as_str());
let gone_from_share = !share.join("a.jpg").exists();
let _ = std::fs::remove_dir_all(&share);
assert!(moved, "moved across devices: {r:?}");
assert!(undone, "and back");
assert!(gone_from_share, "no copy left on the other device");
}
/// A writable folder on another device than `here`, if there is one.
fn other_device(here: &std::path::Path) -> Option<std::path::PathBuf> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let dev = std::fs::metadata(here).ok()?.dev();
["/dev/shm", "/run/shm"]
.iter()
.map(std::path::PathBuf::from)
.find(|p| {
std::fs::metadata(p).is_ok_and(|m| m.dev() != dev)
&& std::fs::write(p.join(format!(".lc-probe-{}", std::process::id())), b"x").is_ok()
})
.inspect(|p| {
let _ = std::fs::remove_file(p.join(format!(".lc-probe-{}", std::process::id())));
})
}
#[cfg(not(unix))]
{
let _ = here;
None
}
}
#[test]
fn the_copy_across_volumes_is_whole_or_absent_and_never_replaces() {
use crate::rename::MoveFs;
let dir = Scratch::new("copy");
let body: Vec<u8> = (0..(3usize << 20) + 17).map(|i| (i % 251) as u8).collect();
let (from, to) = (dir.0.join("local/big.raw"), dir.0.join("share/big.raw"));
std::fs::write(&from, &body).unwrap();
crate::rename::RealFs.copy_no_replace(&from, &to).unwrap();
assert_eq!(std::fs::read(&to).unwrap(), body);
// nothing left under a hidden name beside it
let left: Vec<_> = std::fs::read_dir(dir.0.join("share")).unwrap().flatten().map(|e| e.file_name()).collect();
assert_eq!(left, vec![std::ffi::OsString::from("big.raw")], "{left:?}");
// a partial left by a copy the share cut off doesn't block the next copy
let stale = dir.0.join(format!("share/.again.raw.lc-partial-{}-0", std::process::id()));
std::fs::write(&stale, b"cut off").unwrap();
crate::rename::RealFs.copy_no_replace(&from, &dir.0.join("share/again.raw")).unwrap();
assert_eq!(std::fs::read(dir.0.join("share/again.raw")).unwrap(), body);
std::fs::remove_file(&stale).unwrap();
std::fs::remove_file(dir.0.join("share/again.raw")).unwrap();
// a copy that can't start (its source unreadable) leaves nothing at all under the photo's name
let gone = dir.0.join("local/missing.raw");
assert!(crate::rename::RealFs.copy_no_replace(&gone, &dir.0.join("share/missing.raw")).is_err());
assert!(!dir.0.join("share/missing.raw").exists());
let left: Vec<_> = std::fs::read_dir(dir.0.join("share")).unwrap().flatten().map(|e| e.file_name()).collect();
assert_eq!(left.len(), 1, "no partial file left: {left:?}");
// a name taken is never replaced, and what is there stays
std::fs::write(&to, b"other").unwrap();
assert!(crate::rename::RealFs.copy_no_replace(&from, &to).is_err());
assert_eq!(std::fs::read(&to).unwrap(), b"other");
}
+20
View File
@@ -336,3 +336,23 @@ fn deleting_an_album_folder_preserves_photos_and_restores_its_subtree() {
drop(reopened);
let _ = std::fs::remove_dir_all(dir);
}
/// A huge `start` is a validation error, not an arithmetic overflow in the numbering.
#[test]
fn rename_huge_start_is_a_bad_param() {
let mut s = Session::with_demo();
let ids: Vec<u64> = s.catalog.photos().take(3).map(|p| p.id.0).collect();
let names_before: Vec<String> = s.catalog.photos().map(|p| p.file_name.clone()).collect();
for cmd in ["photo.renamePreview", "photo.rename"] {
let e = s.execute(cmd, &json!({"template": "{seq}", "start": u64::MAX, "ids": ids})).unwrap_err();
assert!(e.to_string().contains("start out of range"), "{cmd}: {e}");
}
let names_after: Vec<String> = s.catalog.photos().map(|p| p.file_name.clone()).collect();
assert_eq!(names_before, names_after, "a rejected rename changes nothing");
// the last start whose whole sequence fits is accepted, one more is not
let fits = usize::MAX as u64 - 2;
let r = s.execute("photo.renamePreview", &json!({"template": "{seq}", "start": fits, "ids": ids}));
assert!(r.is_ok(), "{r:?}");
let r = s.execute("photo.renamePreview", &json!({"template": "{seq}", "start": fits + 1, "ids": ids}));
assert!(r.is_err(), "{r:?}");
}
+19
View File
@@ -142,6 +142,25 @@ fn preferences_persist_with_the_library() {
let _ = std::fs::remove_dir_all(&lib);
}
#[test]
fn last_merge_settings_persist_with_the_library() {
let lib = temp_dir("merge");
let opts = json!({"align": false, "deghost": "high"});
{
let mut s = Session::new().with_fs();
s.open_library(&lib, false).unwrap();
s.last_merge.insert("merge.hdr".into(), opts.clone());
s.save_prefs().unwrap();
s.close_library().unwrap();
}
let mut s = Session::new().with_fs();
s.open_library(&lib, false).unwrap();
assert_eq!(s.last_merge.get("merge.hdr"), Some(&opts));
assert!(!s.last_merge.contains_key("merge.panorama"));
drop(s);
let _ = std::fs::remove_dir_all(&lib);
}
#[test]
fn default_copyright_and_creator_fill_gaps_on_import() {
let dir = temp_dir("copyright");
+20
View File
@@ -341,3 +341,23 @@ fn every_print_command_is_registered_with_a_description() {
assert!(c.params.len() > 3, "{id}");
}
}
#[test]
fn page_options_are_print_settings_and_template_fields() {
let mut s = Session::with_demo();
let g = run(&mut s, "print.get", json!({}));
assert_eq!(g["settings"]["page"], json!({"background": [255, 255, 255], "pageNumbers": false, "pageInfo": false, "cropMarks": false}));
let g = run(&mut s, "print.set", json!({"page": {"pageNumbers": true, "cropMarks": true}}));
assert_eq!((g["settings"]["page"]["pageNumbers"].as_bool(), g["settings"]["page"]["pageInfo"].as_bool()), (Some(true), Some(false)));
assert_eq!(g["settings"]["page"]["background"], json!([255, 255, 255]), "a patch merges");
fails(&mut s, "print.set", json!({"page": {"pageInfo": "yes"}}));
fails(&mut s, "print.set", json!({"page": {"pageNumber": true}}));
// a template keeps them
run(&mut s, "print.template.save", json!({"name": "Marked"}));
run(&mut s, "print.reset", json!({}));
assert!(!s.print.page.crop_marks);
run(&mut s, "print.template.apply", json!({"name": "Marked"}));
assert!(s.print.page.page_numbers && s.print.page.crop_marks && !s.print.page.page_info);
let set = crate::find_command("print.set").unwrap();
assert!(set.params.contains("pageNumbers") && set.params.contains("pageInfo") && set.params.contains("cropMarks"), "{}", set.params);
}
+1 -1
View File
@@ -975,7 +975,7 @@ fn dngs_of_raws_without_colour_matrices_carry_the_colour_they_open_with() {
std::fs::write(&path, &bytes).unwrap();
let mut opened = lightcraft_raw::decode(&bytes).unwrap();
assert!(!has_matrix(&opened.color));
let (_, want, look) = crate::camera_preview::starting_colour(&mut opened, &bytes, None);
let (_, want, look) = crate::camera_preview::starting_colour(&mut opened, &bytes, None, None);
assert_eq!((want.matrix_is_fallback, look.is_none()), (!measured, true), "{model}");
let mut s = Session::new().with_fs();
+93 -4
View File
@@ -47,10 +47,35 @@ fn hidden(p: &Path) -> bool {
p.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.'))
}
/// What a folder's listing says an entry is.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum EntryKind {
File,
Folder,
/// A link, or an entry whose kind the listing did not say.
Other,
}
/// What [`files_in_noting`] hands over for each folder it lists: the folder, the name and kind
/// of every entry in it (hidden ones too), and whether that is all of them (not when the walk
/// stopped inside the folder, or an entry could not be read).
pub type NoteFolder<'a> = &'a mut dyn FnMut(&Path, Vec<(std::ffi::OsString, EntryKind)>, bool);
/// The files below folder `root` that `keep` accepts. Hidden entries (names starting with `.`)
/// are left out, the folder `skip` (e.g. the library's own) is never entered, and the walk stops
/// at `limits` (see [`Walked::truncated`]). Unreadable folders are skipped.
pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool) -> Walked {
pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool) -> Walked {
walk(root, skip, limits, keep, None)
}
/// [`files_in`], handing the listing of every folder it reads to `note`: who walks a folder to
/// find its files has, with that, the answer to "is this name there" for everything in it, and
/// need not ask the disk again (a round trip for each name on a network share).
pub fn files_in_noting(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool, note: NoteFolder) -> Walked {
walk(root, skip, limits, keep, Some(note))
}
fn walk(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool, mut note: Option<NoteFolder>) -> Walked {
let mut out = Walked::default();
// canonical folders reached through a link, so a link loop is walked at most once
let mut linked: HashSet<PathBuf> = HashSet::new();
@@ -77,17 +102,41 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
}
let Ok(rd) = std::fs::read_dir(&path) else { continue };
let mut children: Vec<(PathBuf, bool)> = Vec::new();
for e in rd.flatten() {
// (the folder's listing, for who asked for it; all of it unless something below says otherwise)
let mut listed = note.is_some().then(Vec::new);
let mut complete = true;
for e in rd {
let Ok(e) = e else {
complete = false;
continue;
};
if out.entries >= limits.max_entries {
out.truncated = true;
complete = false;
break;
}
out.entries = out.entries.saturating_add(1);
let p = e.path();
if hidden(&p) {
// (a hidden entry's kind is only asked for when the listing is wanted; every other
// entry's is asked once)
let is_hidden = hidden(&p);
if is_hidden && listed.is_none() {
continue;
}
let Ok(ft) = e.file_type() else { continue };
let ft = e.file_type();
if let Some(listed) = listed.as_mut() {
let kind = match &ft {
Ok(t) if t.is_symlink() => EntryKind::Other,
Ok(t) if t.is_dir() => EntryKind::Folder,
Ok(t) if t.is_file() => EntryKind::File,
_ => EntryKind::Other,
};
listed.push((e.file_name(), kind));
}
if is_hidden {
continue;
}
let Ok(ft) = ft else { continue };
let dir = if ft.is_symlink() {
// a link (or a Windows junction): a folder is entered once, a file is a file
match std::fs::metadata(&p) {
@@ -105,6 +154,9 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
};
children.push((p, dir));
}
if let (Some(note), Some(listed)) = (note.as_mut(), listed) {
note(&path, listed, complete);
}
children.sort();
let below = depth.saturating_add(1);
stack.extend(children.into_iter().rev().map(|(p, d)| (p, below, d)));
@@ -123,6 +175,43 @@ mod tests {
d
}
/// Given a folder with files, a folder inside it, a hidden file and a link, when it is walked
/// with its listings noted, then the walk finds what it finds without them, and each folder's
/// listing holds every entry by name with its kind (hidden ones too, a link as neither file nor
/// folder) and says it is all of them; a walk that stops inside a folder says it is not.
#[test]
fn a_walk_notes_each_folders_listing() {
let root = tmp("noting");
std::fs::create_dir_all(root.join("sub")).unwrap();
for name in ["a.jpg", ".hidden", "sub/b.jpg"] {
std::fs::write(root.join(name), b"x").unwrap();
}
#[cfg(unix)]
std::os::unix::fs::symlink(root.join("a.jpg"), root.join("link.jpg")).unwrap();
let mut noted: Vec<(PathBuf, Vec<(String, EntryKind)>, bool)> = Vec::new();
let walked = files_in_noting(&root, None, Limits::default(), |_| true, &mut |dir, entries, complete| {
let mut entries: Vec<(String, EntryKind)> = entries.into_iter().map(|(n, k)| (n.to_string_lossy().to_string(), k)).collect();
entries.sort_by(|a, b| a.0.cmp(&b.0));
noted.push((dir.to_path_buf(), entries, complete));
});
assert_eq!(walked, files_in(&root, None, Limits::default(), |_| true), "the same walk, noted or not");
let named = |n: &str, k: EntryKind| (n.to_string(), k);
let mut top = vec![named(".hidden", EntryKind::File), named("a.jpg", EntryKind::File), named("sub", EntryKind::Folder)];
if cfg!(unix) {
top.insert(2, named("link.jpg", EntryKind::Other));
}
assert_eq!(noted, [(root.clone(), top, true), (root.join("sub"), vec![named("b.jpg", EntryKind::File)], true)]);
// stopped inside the first folder: its listing is not all of it
let mut cut = Vec::new();
let walked = files_in_noting(&root, None, Limits { max_depth: 8, max_entries: 2 }, |_| true, &mut |dir, entries, complete| {
cut.push((dir.to_path_buf(), entries.len(), complete))
});
assert!(walked.truncated);
assert_eq!(cut.first().map(|(dir, n, complete)| (dir.clone(), *n, *complete)), Some((root.clone(), 2, false)));
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn files_come_in_path_order_without_hidden_ones_or_the_skipped_folder() {
let d = tmp("order");
+2 -1
View File
@@ -5,7 +5,8 @@
//!
//! Everything here treats model files and manifests as hostile input: sizes are capped, numbers must be
//! finite and sane, and a malformed file is an error, never a panic. Detection and recognition run
//! on the same checked, single-thread pure-Rust CPU interpreter.
//! on the same checked, single-thread pure-Rust CPU interpreter; an [`net::Accelerator`] (the GPU, in `lightcraft-gpu`) can
//! take its convolutions once it agrees with it.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
+451 -3
View File
@@ -9,8 +9,10 @@
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::{Arc, PoisonError, RwLock};
mod kernels;
pub mod program;
mod shape;
use crate::graph::{Data, Graph, Node};
@@ -64,9 +66,64 @@ impl Tensor {
}
}
/// One convolution, as handed to an [`Accelerator`]: batch 1, planar (NCHW) float32, already checked.
///
/// `out[co][oy][ox] = bias[co] + Σ weights[co][ci][ky][kx] · input[g·cg + ci][oy·sh + ky − pt][ox·sw + kx − pl]`
/// with `g = co / (cout / groups)` and zero outside the input.
#[derive(Clone, Copy, Debug)]
pub struct ConvJob<'a> {
/// The node's index in the graph: the same for every run of a net, so its weights can stay on the device.
pub layer: usize,
pub cin: usize,
pub h: usize,
pub w: usize,
pub cout: usize,
pub groups: usize,
pub kh: usize,
pub kw: usize,
pub sh: usize,
pub sw: usize,
pub pt: usize,
pub pl: usize,
pub oh: usize,
pub ow: usize,
/// `[cout][cin / groups][kh][kw]`.
pub weights: &'a [f32],
/// `[cout]`, if the node has one.
pub bias: Option<&'a [f32]>,
/// `[cin][h][w]`.
pub input: &'a [f32],
}
impl ConvJob<'_> {
/// Multiply-adds the convolution takes.
pub fn macs(&self) -> usize {
let per_group = (self.cin / self.groups.max(1)).saturating_mul(self.kh).saturating_mul(self.kw);
self.cout.saturating_mul(per_group).saturating_mul(self.oh).saturating_mul(self.ow)
}
}
/// Something that can run a network's convolutions faster than the CPU kernels (the GPU, in `lightcraft-gpu`).
///
/// The CPU interpreter stays the reference: a net only uses an accelerator that agreed with it on a test input
/// ([`Net::attach_checked`]), and the first error makes the net drop it and run on the CPU again.
pub trait Accelerator: Send + Sync {
/// What it runs on, for the user ("AMD Radeon RX 7900 (Vulkan)").
fn device(&self) -> String;
/// `[cout][oh][ow]`, or `Ok(None)` to leave this one to the CPU (too small to be worth a dispatch, say).
fn conv(&self, job: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String>;
/// The whole network on the device, keeping every value there between steps, for each of `inputs` (all of
/// `program.lens[INPUT]` values) in one go: per input, each of `program.outputs` in order. `Ok(None)` leaves the
/// run to the CPU path (which may still use [`Accelerator::conv`]).
fn run_program(&self, _program: &program::Program, _inputs: &[&[f32]]) -> std::result::Result<Option<Vec<Vec<Vec<f32>>>>, String> {
Ok(None)
}
}
/// A loaded network, ready to run.
pub struct Net {
graph: Graph,
accelerator: RwLock<Option<Arc<dyn Accelerator>>>,
}
/// Validate semantics rather than silently ignoring unknown or mistyped attributes.
@@ -80,7 +137,75 @@ impl Net {
for n in &graph.nodes {
check(n, &graph)?;
}
Ok(Net { graph })
Ok(Net { graph, accelerator: RwLock::new(None) })
}
/// The accelerator in use, if any.
pub fn accelerator(&self) -> Option<Arc<dyn Accelerator>> {
self.accelerator.read().unwrap_or_else(PoisonError::into_inner).clone()
}
/// Use `accelerator` for this net's convolutions, but only after it gave the CPU's outputs on `probe` (every
/// output value within `tolerance` of the CPU's, relative to that output's largest magnitude) and, with
/// `require_faster`, ran it faster than the CPU did (a GPU's per-layer round trip can cost more than a small
/// model's layers). Returns why it was not used, so the caller can say so; the net then stays on the CPU.
pub fn attach_checked(
&self,
accelerator: Arc<dyn Accelerator>,
probe: &Tensor,
tolerance: f32,
require_faster: bool,
) -> std::result::Result<(), String> {
self.detach();
let started = web_time::Instant::now();
let cpu = self.run(probe.clone()).map_err(|e| e.to_string())?;
let cpu_time = started.elapsed();
*self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = Some(accelerator);
// the first run also uploads the weights and builds the kernels: time a second one
let fast = self.run(probe.clone()).map_err(|e| e.to_string());
let mut verdict = match fast {
Err(e) => Err(e),
Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
Ok(fast) => agree(&cpu, &fast, tolerance),
};
if verdict.is_ok() && require_faster {
let started = web_time::Instant::now();
let again = self.run(probe.clone()).map_err(|e| e.to_string());
let fast_time = started.elapsed();
verdict = match again {
Err(e) => Err(e),
Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
Ok(_) if fast_time >= cpu_time => Err(format!(
"the device is not faster for this model ({:.0} ms against {:.0} ms on the CPU)",
fast_time.as_secs_f64() * 1000.0,
cpu_time.as_secs_f64() * 1000.0
)),
Ok(_) => Ok(()),
};
}
if verdict.is_err() {
self.detach();
}
verdict
}
/// Run on the CPU only from now on.
pub fn detach(&self) {
*self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = None;
}
/// A convolution on the accelerator, if one is attached and takes it. A failing accelerator is dropped.
fn accelerated_conv(&self, layer: usize, node: &Node, x: &Tensor) -> Result<Option<Tensor>> {
let Some(accelerator) = self.accelerator() else { return Ok(None) };
let job = kernels::conv_job(layer, node, &x.shape, &x.data, &self.graph)?;
match accelerator.conv(&job) {
Ok(Some(out)) if out.len() == job.cout * job.oh * job.ow => Ok(Some(Tensor::new(vec![1, job.cout, job.oh, job.ow], out)?)),
Ok(None) => Ok(None),
Ok(Some(_)) | Err(_) => {
self.detach();
Ok(None)
}
}
}
/// The shape the model declares for its input (0 for a dynamic size).
@@ -96,7 +221,67 @@ impl Net {
/// Run the network on `input` (its single input); returns each output by name.
pub fn run(&self, input: Tensor) -> Result<Vec<(String, Tensor)>> {
shape::tensor(&input.shape, input.data.len(), &input.data)?;
self.run_batch(vec![input])?.pop().ok_or_else(|| NetError::Shape("the run gave nothing".into()))
}
/// Run the network on each of `inputs`; an accelerator that keeps the values on its device runs inputs of one shape
/// together (one upload and one read-back for all of them).
pub fn run_batch(&self, inputs: Vec<Tensor>) -> Result<Vec<Vec<(String, Tensor)>>> {
for input in &inputs {
shape::tensor(&input.shape, input.data.len(), &input.data)?;
}
if let Some(out) = self.run_on_device(&inputs)? {
return Ok(out);
}
inputs.into_iter().map(|input| self.run_one(input)).collect()
}
/// The whole batch on the attached accelerator, or `None` to run it on the CPU path. A failing or misbehaving
/// accelerator is dropped.
fn run_on_device(&self, inputs: &[Tensor]) -> Result<Option<Vec<Vec<(String, Tensor)>>>> {
let (Some(accelerator), Some(first)) = (self.accelerator(), inputs.first()) else { return Ok(None) };
if inputs.iter().any(|t| t.shape != first.shape) {
return Ok(None);
}
let Some(program) = self.program(&first.shape)? else { return Ok(None) };
let slices: Vec<&[f32]> = inputs.iter().map(|t| t.data.as_slice()).collect();
let started = web_time::Instant::now();
let outs = match accelerator.run_program(&program, &slices) {
Ok(Some(outs)) => outs,
Ok(None) => return Ok(None),
Err(_) => {
self.detach();
return Ok(None);
}
};
let gathered: Option<Vec<Vec<(String, Tensor)>>> = (outs.len() == inputs.len())
.then(|| {
outs.into_iter()
.map(|values| {
(values.len() == program.outputs.len())
.then(|| {
values
.into_iter()
.zip(&program.outputs)
.map(|(data, (name, _, shape))| Tensor::new(shape.clone(), data).ok().map(|t| (name.clone(), t)))
.collect::<Option<Vec<_>>>()
})
.flatten()
})
.collect()
})
.flatten();
if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
eprintln!("[faces-gpu] {} steps x {}: {:.3} ms", program.steps.len(), inputs.len(), started.elapsed().as_secs_f64() * 1000.0);
}
if gathered.is_none() {
// wrong sizes or values that are not finite: the device is not trusted any more
self.detach();
}
Ok(gathered)
}
fn run_one(&self, input: Tensor) -> Result<Vec<(String, Tensor)>> {
self.validate_input(&input.shape)?;
let mut scratch = kernels::Scratch::default();
let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
@@ -127,7 +312,13 @@ impl Net {
};
let started = web_time::Instant::now();
let out = match node.op.as_str() {
"Conv" => kernels::conv(node, &*get(0)?, g, &mut scratch)?,
"Conv" => {
let x = get(0)?;
match self.accelerated_conv(i, node, &x)? {
Some(out) => out,
None => kernels::conv(node, &x, g, &mut scratch)?,
}
}
"Relu" => map(&*get(0)?, |v| v.max(0.0)),
"Sigmoid" => map(&*get(0)?, |v| 1.0 / (1.0 + (-v).exp())),
"Add" | "Sub" | "Mul" => kernels::binary(node.op.as_str(), &*get(0)?, &*get(1)?)?,
@@ -168,6 +359,25 @@ impl Net {
}
}
/// Whether two runs' outputs agree: same names and shapes, values within `tolerance` of each other relative to the
/// output's largest magnitude.
fn agree(a: &[(String, Tensor)], b: &[(String, Tensor)], tolerance: f32) -> std::result::Result<(), String> {
if a.len() != b.len() {
return Err("the device gave a different number of outputs".into());
}
for ((name, x), (other, y)) in a.iter().zip(b) {
if name != other || x.shape != y.shape {
return Err(format!("the device gave `{other}` a different shape"));
}
let scale = x.data.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-6);
let worst = x.data.iter().zip(&y.data).fold(0f32, |m, (p, q)| m.max((p - q).abs()));
if worst.is_nan() || worst > tolerance * scale {
return Err(format!("the device's `{name}` differs from the CPU's by {:.2e} (of {scale:.2e})", worst));
}
}
Ok(())
}
fn map(x: &Tensor, f: impl Fn(f32) -> f32) -> Tensor {
Tensor { shape: x.shape.clone(), data: x.data.iter().map(|v| f(*v)).collect() }
}
@@ -534,6 +744,244 @@ mod tests {
assert_eq!(rs.shape, vec![1, 6, 2]);
}
/// An accelerator built on the slow reference convolution, optionally wrong or failing.
struct Fake {
skew: f32,
fail_after: std::sync::atomic::AtomicUsize,
}
impl Accelerator for Fake {
fn device(&self) -> String {
"test".into()
}
fn conv(&self, j: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String> {
if self.fail_after.fetch_sub(1, std::sync::atomic::Ordering::Relaxed) == 0 {
return Err("device lost".into());
}
assert!(j.pt == j.pl && j.sh == j.sw);
let x = Tensor::new(vec![1, j.cin, j.h, j.w], j.input.to_vec()).unwrap();
let zero = vec![0.0; j.cout];
let dims = [j.cout, j.cin / j.groups, j.kh, j.kw];
let out = reference_conv(&x, j.weights, dims, j.bias.unwrap_or(&zero), j.sh, j.pt, j.groups);
Ok(Some(out.data.iter().map(|v| v + self.skew).collect()))
}
}
fn two_convs() -> (Graph, Tensor) {
let attrs = [("strides", Attr::Ints(vec![2, 2])), ("pads", Attr::Ints(vec![1; 4]))];
let mut a = node("Conv", &["x", "w1", "b1"], &attrs);
a.outputs = vec!["h".into()];
let b = node("Conv", &["h", "w2"], &[("group", Attr::Int(2)), ("pads", Attr::Ints(vec![1; 4]))]);
let mut g = graph(
vec![a, b],
vec![
("w1", w(&[4, 3, 3, 3], (0..108).map(|i| ((i * 7 % 13) as f32 - 6.0) * 0.05).collect())),
("b1", w(&[4], vec![0.1, -0.2, 0.3, 0.0])),
("w2", w(&[4, 2, 3, 3], (0..72).map(|i| ((i * 5 % 11) as f32 - 5.0) * 0.05).collect())),
],
);
g.inputs = vec![("x".into(), vec![1, 3, 9, 8])];
(g, input(3, 9, 8, |i| ((i * 13 % 17) as f32 - 8.0) * 0.1))
}
#[test]
fn an_accelerator_is_used_only_when_it_agrees_with_the_cpu() {
let (g, x) = two_convs();
let net = Net::new(g).unwrap();
let cpu = net.run(x.clone()).unwrap();
let good = Arc::new(Fake { skew: 0.0, fail_after: usize::MAX.into() });
net.attach_checked(good, &x, 1e-4, false).unwrap();
assert!(net.accelerator().is_some());
assert_eq!(net.run(x.clone()).unwrap()[0].1.shape, cpu[0].1.shape);
let wrong = Arc::new(Fake { skew: 0.5, fail_after: usize::MAX.into() });
assert!(net.attach_checked(wrong, &x, 1e-4, false).is_err());
assert!(net.accelerator().is_none(), "a device that disagrees is not used");
assert_eq!(net.run(x).unwrap(), cpu);
}
#[test]
fn a_failing_accelerator_falls_back_to_the_cpu_mid_run() {
let (g, x) = two_convs();
let net = Net::new(g).unwrap();
let cpu = net.run(x.clone()).unwrap();
// agrees on the probe (two convs), then fails on the second conv of the next run
net.attach_checked(Arc::new(Fake { skew: 0.0, fail_after: 3.into() }), &x, 1e-4, false).unwrap();
let out = net.run(x).unwrap();
assert!(net.accelerator().is_none());
for (a, b) in out[0].1.data.iter().zip(&cpu[0].1.data) {
assert!((a - b).abs() < 1e-4);
}
}
/// An accelerator that runs whole programs with plain loops (the definition of each step), counting its calls;
/// `poison` makes it give a value that is not finite.
struct ProgramFake {
calls: std::sync::atomic::AtomicUsize,
poison: bool,
}
fn run_steps(p: &program::Program, input: &[f32]) -> Vec<Vec<f32>> {
use program::{Arg, Binary, Op, Unary};
let mut values: Vec<Vec<f32>> = vec![Vec::new(); p.lens.len()];
values[program::INPUT] = input.to_vec();
for step in &p.steps {
let arg = |i: usize| -> Vec<f32> {
match &step.args[i] {
Arg::Value(v) => values[*v].clone(),
Arg::Const { data, .. } => data.to_vec(),
}
};
let len = p.lens[step.out];
let x = arg(0);
let out: Vec<f32> = match &step.op {
Op::Conv(j) => {
let (m, cg) = (j.cout / j.groups, j.cin / j.groups);
let mut out = vec![0f32; len];
for co in 0..j.cout {
for oy in 0..j.oh {
for ox in 0..j.ow {
let mut v = j.bias.map_or(0.0, |b| b[co]);
for ci in 0..cg {
for ky in 0..j.kh {
for kx in 0..j.kw {
let (iy, ix) = ((oy * j.sh + ky) as isize - j.pt as isize, (ox * j.sw + kx) as isize - j.pl as isize);
if iy >= 0 && ix >= 0 && (iy as usize) < j.h && (ix as usize) < j.w {
v += j.weights[((co * cg + ci) * j.kh + ky) * j.kw + kx]
* x[((co / m * cg + ci) * j.h + iy as usize) * j.w + ix as usize];
}
}
}
}
out[(co * j.oh + oy) * j.ow + ox] = v;
}
}
}
out
}
Op::Unary(Unary::Relu) => x.iter().map(|v| v.max(0.0)).collect(),
Op::Unary(Unary::Sigmoid) => x.iter().map(|v| 1.0 / (1.0 + (-v).exp())).collect(),
Op::ChannelAffine { channels, plane } => {
let (s, o) = (arg(1), arg(2));
x.iter().enumerate().map(|(i, v)| v * s[(i / plane) % channels] + o[(i / plane) % channels]).collect()
}
Op::ChannelPrelu { channels, plane } => {
let a = arg(1);
x.iter().enumerate().map(|(i, v)| if *v < 0.0 { v * a[(i / plane) % channels] } else { *v }).collect()
}
Op::Binary { op, dims, a, b } => {
let y = arg(1);
(0..len)
.map(|i| {
let (mut rest, mut ia, mut ib) = (i, 0, 0);
for d in (0..4).rev() {
let at = rest % dims[d];
rest /= dims[d];
ia += at * a[d];
ib += at * b[d];
}
match op {
Binary::Add => x[ia] + y[ib],
Binary::Sub => x[ia] - y[ib],
Binary::Mul => x[ia] * y[ib],
}
})
.collect()
}
Op::Transpose { dims, strides } => (0..len)
.map(|i| {
let (mut rest, mut from) = (i, 0);
for d in (0..4).rev() {
from += (rest % dims[d]) * strides[d];
rest /= dims[d];
}
x[from]
})
.collect(),
Op::MaxPool { h, w, kh, kw, sh, sw, oh, ow, .. } => (0..len)
.map(|i| {
let (ox, oy, c) = (i % ow, (i / ow) % oh, i / (ow * oh));
let mut m = f32::NEG_INFINITY;
for ky in 0..*kh {
for kx in 0..*kw {
m = m.max(x[(c * h + oy * sh + ky) * w + ox * sw + kx]);
}
}
m
})
.collect(),
Op::Resize { h, w, sh, sw, .. } => (0..len)
.map(|i| {
let (ow, oh) = (w * sw, h * sh);
let (ox, oy, c) = (i % ow, (i / ow) % oh, i / (ow * oh));
x[(c * h + oy / sh) * w + ox / sw]
})
.collect(),
Op::GlobalAverage { plane, .. } => x.chunks(*plane).map(|c| c.iter().sum::<f32>() / *plane as f32).collect(),
Op::Gemv { k, n, trans_b, alpha } => {
let (wt, bias) = (arg(1), arg(2));
(0..*n).map(|o| alpha * (0..*k).map(|j| x[j] * wt[if *trans_b { o * k + j } else { j * n + o }]).sum::<f32>() + bias[o]).collect()
}
};
assert_eq!(out.len(), len, "{:?}", step.op);
values[step.out] = out;
for f in &step.free {
values[*f] = Vec::new();
}
}
p.outputs.iter().map(|(_, v, _)| values[*v].clone()).collect()
}
impl Accelerator for ProgramFake {
fn device(&self) -> String {
"program test".into()
}
fn conv(&self, _: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String> {
Ok(None)
}
fn run_program(&self, p: &program::Program, inputs: &[&[f32]]) -> std::result::Result<Option<Vec<Vec<Vec<f32>>>>, String> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut out: Vec<Vec<Vec<f32>>> = inputs.iter().map(|x| run_steps(p, x)).collect();
if self.poison {
out[0][0][0] = f32::NAN;
}
Ok(Some(out))
}
}
fn every_operator_input(seed: usize) -> Tensor {
input(4, 8, 8, |i| (((i * 31 + seed * 7) % 37) as f32 - 18.0) * 0.05)
}
#[test]
fn a_program_of_every_operator_gives_what_the_interpreter_gives() {
let net = Net::new(crate::synthetic::every_operator_graph()).unwrap();
let x = every_operator_input(0);
let cpu = net.run(x.clone()).unwrap();
let fake = Arc::new(ProgramFake { calls: 0.into(), poison: false });
net.attach_checked(fake.clone(), &x, 1e-5, false).unwrap();
let calls = fake.calls.load(std::sync::atomic::Ordering::Relaxed);
assert!(calls > 0, "the whole network ran as one program");
// several inputs go to the device together, and each gets its own outputs
let inputs: Vec<Tensor> = (0..3).map(every_operator_input).collect();
let together = net.run_batch(inputs.clone()).unwrap();
assert_eq!(fake.calls.load(std::sync::atomic::Ordering::Relaxed), calls + 1, "one dispatch for the batch");
net.detach();
for (x, got) in inputs.into_iter().zip(&together) {
let want = net.run(x).unwrap();
agree(&want, got, 1e-5).unwrap();
}
assert_eq!(cpu.len(), 2);
}
#[test]
fn a_device_that_gives_values_that_are_not_finite_is_dropped() {
let net = Net::new(crate::synthetic::every_operator_graph()).unwrap();
let x = every_operator_input(1);
let cpu = net.run(x.clone()).unwrap();
assert!(net.attach_checked(Arc::new(ProgramFake { calls: 0.into(), poison: true }), &x, 1e-5, false).is_err());
assert!(net.accelerator().is_none());
assert_eq!(net.run(x).unwrap(), cpu);
}
#[test]
fn unsupported_and_inconsistent_models_are_errors() {
let x = input(1, 3, 3, |i| i as f32);
+28 -1
View File
@@ -59,6 +59,33 @@ fn shifted_plane(plane: &[f32], width: usize, ky: usize, kx: usize, pt: usize, p
}
Ok(())
}
/// The checked geometry of a convolution node, for an accelerator.
pub(super) fn conv_job<'a>(layer: usize, n: &Node, shape: &[usize], input: &'a [f32], g: &'a Graph) -> Result<ConvJob<'a>> {
let (cin, h, w) = program::chw_of(shape)?;
let wt = shape::weight(n, g, 1)?;
let weights = shape::floats(n, g, 1)?;
let [cout, cg, kh, kw] = wt.dims.as_slice() else {
return shape_err("invalid convolution weights");
};
let (cout, cg, kh, kw) = (*cout, *cg, *kh, *kw);
let groups = usize::try_from(n.int("group").unwrap_or(1)).map_err(|_| NetError::Shape("invalid group".into()))?;
let (sh, sw) = pair(n, "strides", 1)?;
let (pt, pl, pb, pr) = pads(n)?;
let ph = h.checked_add(pt).and_then(|v| v.checked_add(pb)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
let pw = w.checked_add(pl).and_then(|v| v.checked_add(pr)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
if ph < kh || pw < kw || groups == 0 || cg == 0 || kh == 0 || kw == 0 || cin != groups * cg || cout % groups != 0 {
return shape_err("invalid convolution shape");
}
if cout.checked_mul(cg).and_then(|v| v.checked_mul(kh * kw)) != Some(weights.len()) {
return shape_err("short convolution weights");
}
let bias = if n.inputs.get(2).is_some_and(|s| !s.is_empty()) { Some(shape::floats(n, g, 2)?) } else { None };
if bias.is_some_and(|b| b.len() != cout) {
return shape_err("the convolution bias does not match");
}
let (oh, ow) = ((ph - kh) / sh + 1, (pw - kw) / sw + 1);
Ok(ConvJob { layer, cin, h, w, cout, groups, kh, kw, sh, sw, pt, pl, oh, ow, weights, bias, input })
}
pub(super) fn conv(n: &Node, x: &Tensor, g: &Graph, scratch: &mut Scratch) -> Result<Tensor> {
let (cin, h, w) = chw(x)?;
let wt = shape::weight(n, g, 1)?;
@@ -181,7 +208,7 @@ pub(super) fn conv(n: &Node, x: &Tensor, g: &Graph, scratch: &mut Scratch) -> Re
Ok(out)
}
/// Flat offset under ONNX's trailing-dimension broadcasting; shapes are checked before execution.
fn broadcast_offset(mut at: usize, out: &[usize], input: &[usize]) -> usize {
pub(super) fn broadcast_offset(mut at: usize, out: &[usize], input: &[usize]) -> usize {
let mut offset = 0;
let mut stride = 1;
for (axis, &size) in out.iter().enumerate().rev() {
+315
View File
@@ -0,0 +1,315 @@
//! A network lowered to a flat list of steps over numbered values, for an [`Accelerator`] that keeps every activation
//! on its device between layers (one upload, one read-back per run instead of one per convolution).
//!
//! Shapes are resolved for one input shape; every step names the values it reads and the one it writes, and which
//! values nothing reads after it. Shape-only operators (`Reshape`, `Flatten`, `Identity`, `Dropout`) make no step: their
//! output is the same value under another name. Anything this cannot express (a broadcast past four dimensions, a
//! transposed or batched `Gemm`, a slope that is not per channel) makes the whole lowering `None`, and the net runs
//! the CPU path instead.
use std::borrow::Cow;
use super::*;
/// Value 0 is the network's input.
pub const INPUT: usize = 0;
/// An operand: a value computed on the device, or a constant (uploaded once and kept, by `id`).
#[derive(Clone, Debug)]
pub enum Arg<'a> {
Value(usize),
/// `id` is the same for every run of the same net, so the device can keep the data.
Const {
id: usize,
data: Cow<'a, [f32]>,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Unary {
Relu,
Sigmoid,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Binary {
Add,
Sub,
Mul,
}
/// What a step computes. Element counts come from the program's `lens` of the step's output.
#[derive(Clone, Debug)]
pub enum Op<'a> {
/// A convolution; its `input` is empty (it reads the step's first argument).
Conv(ConvJob<'a>),
/// `out[i] = f(x[i])`. Args: `[x]`.
Unary(Unary),
/// `out[i] = x[i] * scale[c] + offset[c]`, `c = (i / plane) % channels`. Args: `[x, scale, offset]`.
ChannelAffine { channels: usize, plane: usize },
/// `out[i] = x[i] < 0 ? x[i] * slope[c] : x[i]`. Args: `[x, slope]` (`channels` values).
ChannelPrelu { channels: usize, plane: usize },
/// Broadcasting over four dimensions: `dims` of the output, element strides of each operand (0 where it repeats).
/// Args: `[a, b]`.
Binary { op: Binary, dims: [usize; 4], a: [usize; 4], b: [usize; 4] },
/// `out[n] = alpha · Σk x[k] · B(k, n) + bias[n]`, `B(k, n) = weights[n·k_len + k]` when `trans_b`, else
/// `weights[k·n_len + n]`. Args: `[x, weights, bias]` (bias already times beta; zeros when the node has none).
Gemv { k: usize, n: usize, trans_b: bool, alpha: f32 },
/// The mean of each `plane`-sized run. Args: `[x]`.
GlobalAverage { channels: usize, plane: usize },
/// Unpadded max pooling. Args: `[x]`.
MaxPool { c: usize, h: usize, w: usize, kh: usize, kw: usize, sh: usize, sw: usize, oh: usize, ow: usize },
/// Nearest-neighbour upscale by whole numbers. Args: `[x]`.
Resize { c: usize, h: usize, w: usize, sh: usize, sw: usize },
/// `out` in its own order (`dims`), read from the input at `strides` (the input's strides, permuted). Args: `[x]`.
Transpose { dims: [usize; 4], strides: [usize; 4] },
}
#[derive(Clone, Debug)]
pub struct Step<'a> {
pub op: Op<'a>,
pub args: Vec<Arg<'a>>,
pub out: usize,
/// Values nothing reads after this step (never an output): their memory can be reused.
pub free: Vec<usize>,
}
/// A whole network for one input shape.
#[derive(Clone, Debug)]
pub struct Program<'a> {
/// Elements of each value; `lens[INPUT]` is the input's.
pub lens: Vec<usize>,
pub steps: Vec<Step<'a>>,
/// Each graph output: its name, the value that holds it and its shape.
pub outputs: Vec<(String, usize, Vec<usize>)>,
}
/// Leading ones up to rank four, or `None` past four.
fn rank4(s: &[usize]) -> Option<[usize; 4]> {
let pad = 4usize.checked_sub(s.len())?;
let mut out = [1; 4];
for (i, d) in s.iter().enumerate() {
*out.get_mut(pad + i)? = *d;
}
Some(out)
}
/// Element strides of `input` read as `out` under trailing-dimension broadcasting (0 along repeated axes).
fn broadcast_strides(out: [usize; 4], input: &[usize]) -> Option<[usize; 4]> {
let dims = rank4(input)?;
let mut strides = [0; 4];
let mut step = 1;
for axis in (0..4).rev() {
let (d, o) = (*dims.get(axis)?, *out.get(axis)?);
if d == o && d != 1 {
*strides.get_mut(axis)? = step;
} else if d != 1 {
return None;
}
step *= d;
}
Some(strides)
}
impl Net {
/// This network lowered for `input` (its shape), or `None` when a step has no device form.
pub fn program(&self, input: &[usize]) -> Result<Option<Program<'_>>> {
let g = &self.graph;
let (shapes, _) = shape::infer_all(g, input)?;
let Some((input_name, _)) = g.inputs.first() else { return shape_err("the model has no input") };
let mut lens = vec![shape::elements(input)?];
let mut names: HashMap<&str, (usize, Vec<usize>)> = HashMap::new();
names.insert(input_name.as_str(), (INPUT, input.to_vec()));
let mut steps: Vec<Step> = Vec::new();
for (i, node) in g.nodes.iter().enumerate() {
let Some(out_shape) = shapes.get(i) else { return shape_err("a node has no shape") };
let Some(out_name) = node.outputs.first() else { return shape_err("a node has no output") };
let value = |idx: usize| -> Option<&(usize, Vec<usize>)> { node.inputs.get(idx).and_then(|n| names.get(n.as_str())) };
let constant = |idx: usize| -> Option<(Cow<'_, [f32]>, &[usize])> {
let w = node.inputs.get(idx).and_then(|n| g.weights.get(n))?;
match &w.data {
Data::F32(v) => Some((Cow::Borrowed(v.as_slice()), w.dims.as_slice())),
_ => None,
}
};
let id = |k: usize| i * 4 + k;
let Some((x, x_shape)) = value(0).cloned() else { return Ok(None) };
let (op, args) = match node.op.as_str() {
"Reshape" | "Flatten" | "Identity" | "Dropout" => {
names.insert(out_name.as_str(), (x, out_shape.clone()));
continue;
}
"Conv" => (Op::Conv(kernels::conv_job(i, node, &x_shape, &[], g)?), vec![Arg::Value(x)]),
"Relu" => (Op::Unary(Unary::Relu), vec![Arg::Value(x)]),
"Sigmoid" => (Op::Unary(Unary::Sigmoid), vec![Arg::Value(x)]),
"BatchNormalization" => {
let channels = *x_shape.get(1).ok_or_else(|| NetError::Shape("BatchNormalization rank".into()))?;
let plane = shape::elements(x_shape.get(2..).unwrap_or(&[]))?;
let (s, b, m, v) =
(shape::floats(node, g, 1)?, shape::floats(node, g, 2)?, shape::floats(node, g, 3)?, shape::floats(node, g, 4)?);
let eps = shape::float(node, "epsilon", 1e-5);
let mut scale = Vec::with_capacity(channels);
let mut offset = Vec::with_capacity(channels);
for c in 0..channels {
let (Some(s), Some(b), Some(m), Some(v)) = (s.get(c), b.get(c), m.get(c), v.get(c)) else {
return shape_err("short BatchNormalization parameters");
};
let k = s / (v + eps).sqrt();
scale.push(k);
offset.push(b - m * k);
}
(
Op::ChannelAffine { channels, plane },
vec![Arg::Value(x), Arg::Const { id: id(1), data: scale.into() }, Arg::Const { id: id(2), data: offset.into() }],
)
}
"PRelu" => {
let Some((slope, dims)) = constant(1) else { return Ok(None) };
let channels = x_shape.get(1).copied().unwrap_or(1);
let plane = shape::elements(x_shape.get(2..).unwrap_or(&[]))?;
let mut expanded = vec![1; x_shape.len().saturating_sub(dims.len())];
expanded.extend_from_slice(dims);
let per_channel = expanded.iter().enumerate().all(|(i, &d)| d == 1 || i == 1) && slope.len() == channels;
let slope: Cow<[f32]> = if per_channel {
slope
} else if let [one] = slope.as_ref() {
vec![*one; channels].into()
} else {
return Ok(None);
};
(Op::ChannelPrelu { channels, plane }, vec![Arg::Value(x), Arg::Const { id: id(1), data: slope }])
}
"Add" | "Sub" | "Mul" => {
let op = match node.op.as_str() {
"Add" => Binary::Add,
"Sub" => Binary::Sub,
_ => Binary::Mul,
};
let (b, b_shape) = match (value(1), constant(1)) {
(Some((v, s)), _) => (Arg::Value(*v), s.clone()),
(None, Some((data, dims))) => (Arg::Const { id: id(1), data }, dims.to_vec()),
_ => return Ok(None),
};
let Some(dims) = rank4(out_shape) else { return Ok(None) };
let (Some(sa), Some(sb)) = (broadcast_strides(dims, &x_shape), broadcast_strides(dims, &b_shape)) else { return Ok(None) };
(Op::Binary { op, dims, a: sa, b: sb }, vec![Arg::Value(x), b])
}
"Gemm" => {
let Some((weights, wdims)) = constant(1) else { return Ok(None) };
let ([1, k], [n_out]) = (x_shape.as_slice(), out_shape.get(1..).unwrap_or(&[])) else { return Ok(None) };
if node.int("transA").unwrap_or(0) != 0 || out_shape.first() != Some(&1) {
return Ok(None);
}
let trans_b = node.int("transB").unwrap_or(0) != 0;
if wdims.len() != 2 || weights.len() != k * n_out {
return shape_err("Gemm weights do not match");
}
let mut bias = vec![0.0f32; *n_out];
if node.inputs.get(2).is_some_and(|s| !s.is_empty()) {
let wt = shape::weight(node, g, 2)?;
let b = shape::floats(node, g, 2)?;
let beta = shape::float(node, "beta", 1.0);
for (j, to) in bias.iter_mut().enumerate() {
*to = beta
* b.get(kernels::broadcast_offset(j, out_shape, &wt.dims))
.copied()
.ok_or_else(|| NetError::Shape("short matrix bias".into()))?;
}
}
(
Op::Gemv { k: *k, n: *n_out, trans_b, alpha: shape::float(node, "alpha", 1.0) },
vec![Arg::Value(x), Arg::Const { id: id(1), data: weights }, Arg::Const { id: id(2), data: bias.into() }],
)
}
"GlobalAveragePool" => {
let (c, h, w) = chw_of(&x_shape)?;
(Op::GlobalAverage { channels: c, plane: h * w }, vec![Arg::Value(x)])
}
"MaxPool" => {
let (c, h, w) = chw_of(&x_shape)?;
let (kh, kw) = pair(node, "kernel_shape", 1)?;
let (sh, sw) = pair(node, "strides", 1)?;
if pads(node)? != (0, 0, 0, 0) {
return Ok(None);
}
let (_, oh, ow) = chw_of(out_shape)?;
(Op::MaxPool { c, h, w, kh, kw, sh, sw, oh, ow }, vec![Arg::Value(x)])
}
"Resize" => {
let (c, h, w) = chw_of(&x_shape)?;
let (_, oh, ow) = chw_of(out_shape)?;
if oh % h != 0 || ow % w != 0 {
return shape_err("a Resize that is not by whole numbers");
}
(Op::Resize { c, h, w, sh: oh / h, sw: ow / w }, vec![Arg::Value(x)])
}
"Transpose" => {
let rank = x_shape.len();
let default_perm: Vec<i64> = (0..rank as i64).rev().collect();
let perm = node.ints("perm").unwrap_or(&default_perm);
if rank > 4 || perm.len() != rank {
return Ok(None);
}
let mut in_strides = vec![1usize; rank];
for a in (0..rank.saturating_sub(1)).rev() {
let next = in_strides.get(a + 1).copied().unwrap_or(1) * x_shape.get(a + 1).copied().unwrap_or(1);
if let Some(s) = in_strides.get_mut(a) {
*s = next;
}
}
let Some(dims) = rank4(out_shape) else { return Ok(None) };
let mut strides = [0usize; 4];
for (j, p) in perm.iter().enumerate() {
let p = usize::try_from(*p).map_err(|_| NetError::Shape("a negative permutation".into()))?;
let s = in_strides.get(p).copied().ok_or_else(|| NetError::Shape("a bad permutation".into()))?;
*strides.get_mut(4 - rank + j).ok_or_else(|| NetError::Shape("a bad permutation".into()))? = s;
}
(Op::Transpose { dims, strides }, vec![Arg::Value(x)])
}
_ => return Ok(None),
};
let out = lens.len();
lens.push(shape::elements(out_shape)?);
names.insert(out_name.as_str(), (out, out_shape.clone()));
steps.push(Step { op, args, out, free: Vec::new() });
}
let outputs = g
.outputs
.iter()
.map(|name| {
let (v, s) = names.get(name.as_str()).ok_or_else(|| NetError::Shape(format!("output `{name}` was never computed")))?;
Ok((name.clone(), *v, s.clone()))
})
.collect::<Result<Vec<_>>>()?;
// the last step that reads each value; a value nothing reads is free right after it is made
let mut last: Vec<Option<usize>> = vec![None; lens.len()];
for (at, step) in steps.iter().enumerate() {
if let Some(l) = last.get_mut(step.out) {
*l = Some(at);
}
for a in &step.args {
if let Arg::Value(v) = a
&& let Some(l) = last.get_mut(*v)
{
*l = Some(at);
}
}
}
for (v, at) in last.iter().enumerate() {
if outputs.iter().any(|(_, o, _)| *o == v) {
continue;
}
if let Some(step) = at.and_then(|at| steps.get_mut(at)) {
step.free.push(v);
}
}
Ok(Some(Program { lens, steps, outputs }))
}
}
pub(super) fn chw_of(s: &[usize]) -> Result<(usize, usize, usize)> {
match s {
[1, c, h, w] => Ok((*c, *h, *w)),
s => unsupported(format!("a tensor of shape {s:?} (only [1, channels, height, width])")),
}
}
+9 -1
View File
@@ -257,7 +257,12 @@ fn image(s: &[usize]) -> Result<(usize, usize, usize)> {
}
}
pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
Ok(infer_all(g, input)?.1)
}
/// Each node's output shape, and the graph's outputs' shapes.
pub(super) fn infer_all(g: &Graph, input: &[usize]) -> Result<(Vec<Vec<usize>>, Vec<Vec<usize>>)> {
elements(input)?;
let mut per_node = Vec::with_capacity(g.nodes.len());
let mut shapes: HashMap<&str, Vec<usize>> = g.weights.iter().map(|(n, w)| (n.as_str(), w.dims.clone())).collect();
shapes.insert(g.inputs.first().ok_or_else(|| NetError::Shape("no input".into()))?.0.as_str(), input.to_vec());
let mut live = elements(input)?;
@@ -480,6 +485,7 @@ pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
.checked_add(out_count)
.filter(|&n| n <= 128 * 1024 * 1024)
.ok_or_else(|| NetError::Shape("live tensors exceed the 512 MiB working limit".into()))?;
per_node.push(out.clone());
shapes.insert(n.outputs.first().ok_or_else(|| NetError::Shape("no output".into()))?, out);
let mut removed = std::collections::HashSet::new();
for name in &n.inputs {
@@ -488,5 +494,7 @@ pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
}
}
}
g.outputs.iter().map(|s| shapes.get(s.as_str()).cloned().ok_or_else(|| NetError::Shape("missing output".into()))).collect()
let outputs =
g.outputs.iter().map(|s| shapes.get(s.as_str()).cloned().ok_or_else(|| NetError::Shape("missing output".into()))).collect::<Result<_>>()?;
Ok((per_node, outputs))
}
+93 -8
View File
@@ -40,6 +40,9 @@ pub struct SelfTest {
type Plan = Net;
/// Most faces [`Embedder::embed_batch`] sends to a GPU in one go.
pub const BATCH: usize = 16;
/// A loaded recognition model.
#[derive(Clone)]
pub struct Embedder {
@@ -99,6 +102,31 @@ impl Embedder {
})
}
/// The network, e.g. to see which device it runs on.
pub fn net(&self) -> &Net {
&self.plan
}
/// Run the model's convolutions on `accelerator` if it gives the CPU's embedding on a test face (and, with
/// `require_faster`, runs it faster than the CPU); otherwise (or on
/// its first error) recognition stays on the CPU. Returns why it was not used.
pub fn accelerate(&self, accelerator: Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), RuntimeError> {
let (mean, std) = (self.manifest.input.mean, self.manifest.input.std);
let plane = self.width * self.height;
let data: Vec<f32> = crate::yunet::probe_pixels(3 * plane)
.into_iter()
.enumerate()
.map(|(i, v)| {
let c = (i / plane.max(1)).min(2);
(v - mean.get(c).copied().unwrap_or(0.0)) / std.get(c).copied().filter(|s| *s != 0.0).unwrap_or(1.0)
})
.collect();
let probe = Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))?;
guarded("checking the device", || {
self.plan.attach_checked(accelerator, &probe, crate::yunet::ACCELERATOR_TOLERANCE, require_faster).map_err(RuntimeError::Run)
})
}
/// The size of the aligned face picture the model wants (width, height).
pub fn input_size(&self) -> (usize, usize) {
(self.width, self.height)
@@ -110,6 +138,66 @@ impl Embedder {
/// The face as a unit-length vector. `rgb` is an aligned face at exactly [`Self::input_size`], 8-bit RGB.
pub fn embed(&self, rgb: &[u8]) -> Result<Vec<f32>, RuntimeError> {
let input = self.input(rgb)?;
let out = guarded("running it", || {
let mut result = self.plan.run(input).map_err(|e| RuntimeError::Run(e.to_string()))?;
if result.len() != 1 {
return Err(RuntimeError::Run("the model must give one embedding".into()));
}
Ok(result.pop().ok_or_else(|| RuntimeError::Run("no output".into()))?.1.data)
})?;
self.unit(out)
}
/// Several faces at once, each as [`Self::embed`] gives it. On a GPU they run together, up to [`BATCH`] per
/// dispatch (one upload and one read-back for all of them); on the CPU one after another.
pub fn embed_batch(&self, faces: &[&[u8]]) -> Vec<Result<Vec<f32>, RuntimeError>> {
if self.plan.accelerator().is_none() || faces.len() < 2 {
return faces.iter().map(|f| self.embed(f)).collect();
}
let mut results: Vec<Option<Result<Vec<f32>, RuntimeError>>> = faces.iter().map(|_| None).collect();
let mut ready: Vec<(usize, Tensor)> = Vec::new();
for (i, face) in faces.iter().enumerate() {
match self.input(face) {
Ok(t) => ready.push((i, t)),
Err(e) => {
if let Some(r) = results.get_mut(i) {
*r = Some(Err(e));
}
}
}
}
while !ready.is_empty() {
let chunk: Vec<(usize, Tensor)> = ready.drain(..ready.len().min(BATCH)).collect();
let (at, inputs): (Vec<usize>, Vec<Tensor>) = chunk.into_iter().unzip();
let ran = guarded("running it", || self.plan.run_batch(inputs).map_err(|e| RuntimeError::Run(e.to_string())));
match ran {
Ok(outs) if outs.len() == at.len() => {
for (i, mut out) in at.into_iter().zip(outs) {
let r = match (out.len(), out.pop()) {
(1, Some((_, t))) => self.unit(t.data),
_ => Err(RuntimeError::Run("the model must give one embedding".into())),
};
if let Some(slot) = results.get_mut(i) {
*slot = Some(r);
}
}
}
// one bad face spoils a batch: run them one by one so the others still get theirs
_ => {
for i in at {
if let (Some(slot), Some(face)) = (results.get_mut(i), faces.get(i)) {
*slot = Some(self.embed(face));
}
}
}
}
}
results.into_iter().map(|r| r.unwrap_or_else(|| Err(RuntimeError::Run("the face was not embedded".into())))).collect()
}
/// The model's input for an aligned face.
fn input(&self, rgb: &[u8]) -> Result<Tensor, RuntimeError> {
if rgb.len() != self.width * self.height * 3 {
return Err(RuntimeError::Input("the face picture is not the size the model wants".into()));
}
@@ -124,14 +212,11 @@ impl Embedder {
}
}
}
let out = guarded("running it", || {
let input = Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))?;
let mut result = self.plan.run(input).map_err(|e| RuntimeError::Run(e.to_string()))?;
if result.len() != 1 {
return Err(RuntimeError::Run("the model must give one embedding".into()));
}
Ok(result.pop().ok_or_else(|| RuntimeError::Run("no output".into()))?.1.data)
})?;
Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))
}
/// The model's output, checked and scaled to unit length.
fn unit(&self, out: Vec<f32>) -> Result<Vec<f32>, RuntimeError> {
if out.len() != self.dim {
return Err(RuntimeError::Run(format!("the model gave {} numbers per face, not the {} its description says", out.len(), self.dim)));
}
+60
View File
@@ -248,3 +248,63 @@ pub fn tiny_detector_manifest() -> crate::manifest::ModelManifest {
m.size_bytes = Some(bytes.len() as u64);
m
}
/// A small network that uses every operator the interpreter runs (both kinds of convolution, batch norm, PRelu,
/// sigmoid, broadcasting add/sub/mul with values and constants, max pooling, nearest upscaling, transposes, reshape,
/// global pooling, flatten and a fully connected layer), with two outputs, for checking an accelerator's program
/// against the CPU. Its input is `[1, 4, 8, 8]`.
pub fn every_operator_graph() -> crate::graph::Graph {
use crate::graph::{Attr, Data, Graph, Node, Weight};
let node = |op: &str, inputs: &[&str], output: &str, attrs: &[(&str, Attr)]| Node {
op: op.into(),
inputs: inputs.iter().map(|s| s.to_string()).collect(),
outputs: vec![output.into()],
attrs: attrs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
};
let wave = |n: usize, k: usize| -> Vec<f32> { (0..n).map(|i| (((i * k) % 23) as f32 - 11.0) * 0.04).collect() };
let w = |dims: &[usize], v: Vec<f32>| Weight { dims: dims.to_vec(), data: Data::F32(v) };
let weights = [
("w1", w(&[8, 4, 3, 3], wave(288, 7))),
("b1", w(&[8], wave(8, 5))),
("bn_s", w(&[8], (0..8).map(|i| 0.5 + i as f32 * 0.1).collect())),
("bn_b", w(&[8], wave(8, 3))),
("bn_m", w(&[8], wave(8, 11))),
("bn_v", w(&[8], (0..8).map(|i| 0.2 + i as f32 * 0.05).collect())),
("slope", w(&[8, 1, 1], (0..8).map(|i| 0.05 * i as f32).collect())),
("w2", w(&[8, 1, 3, 3], wave(72, 13))),
("scales", w(&[4], vec![1.0, 1.0, 2.0, 2.0])),
("half", w(&[1], vec![0.5])),
("per_channel", w(&[1, 8, 1, 1], wave(8, 17))),
("wg", w(&[5, 8], wave(40, 19))),
("bg", w(&[5], wave(5, 2))),
("flat", Weight { dims: vec![2], data: Data::I64(vec![1, -1]) }),
];
let pads = |p: i64| ("pads", Attr::Ints(vec![p; 4]));
let nodes = vec![
node("Conv", &["x", "w1", "b1"], "c1", &[pads(1)]),
node("BatchNormalization", &["c1", "bn_s", "bn_b", "bn_m", "bn_v"], "bn", &[]),
node("PRelu", &["bn", "slope"], "pr", &[]),
node("Conv", &["pr", "w2"], "c2", &[("group", Attr::Int(8)), ("strides", Attr::Ints(vec![2, 2])), pads(1)]),
node("Sigmoid", &["c2"], "sg", &[]),
node("Mul", &["c2", "sg"], "swish", &[]),
node("Resize", &["swish", "", "scales"], "up", &[]),
node("Add", &["up", "pr"], "sum", &[]),
node("Sub", &["sum", "half"], "less", &[]),
node("MaxPool", &["less"], "pool", &[("kernel_shape", Attr::Ints(vec![2, 2])), ("strides", Attr::Ints(vec![2, 2]))]),
node("Relu", &["pool"], "relu", &[]),
node("Transpose", &["relu"], "nhwc", &[("perm", Attr::Ints(vec![0, 2, 3, 1]))]),
node("Transpose", &["nhwc"], "nchw", &[("perm", Attr::Ints(vec![0, 3, 1, 2]))]),
node("Mul", &["nchw", "per_channel"], "scaled", &[]),
node("Identity", &["scaled"], "same", &[]),
node("GlobalAveragePool", &["same"], "avg", &[]),
node("Flatten", &["avg"], "row", &[]),
node("Reshape", &["row", "flat"], "row2", &[]),
node("Gemm", &["row2", "wg", "bg"], "y", &[("transB", Attr::Int(1)), ("alpha", Attr::Float(0.5))]),
];
Graph {
nodes,
weights: weights.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
inputs: vec![("x".into(), vec![1, 4, 8, 8])],
outputs: vec!["y".into(), "nhwc".into()],
}
}
+22
View File
@@ -60,6 +60,14 @@ pub struct Face {
pub landmarks: [(f32, f32); 5],
}
/// How far an accelerator's outputs may stray from the CPU's on the test input, relative to each output's largest value.
pub const ACCELERATOR_TOLERANCE: f32 = 1e-3;
/// A deterministic, photo-like test input (smooth gradients plus texture, in 0..255).
pub fn probe_pixels(n: usize) -> Vec<f32> {
(0..n).map(|i| ((i % 640) as f32 * 0.3 + ((i * 7919) % 97) as f32 + (i / 640 % 480) as f32 * 0.2) % 255.0).collect()
}
pub struct Detector {
net: Net,
/// Side of the square network input.
@@ -90,6 +98,20 @@ impl Detector {
Ok(Detector { net, side })
}
/// The network, e.g. to see which device it runs on.
pub fn net(&self) -> &Net {
&self.net
}
/// Run the network's convolutions on `accelerator` if it gives the CPU's outputs on a test picture (and, with `require_faster`,
/// runs it faster than the CPU); otherwise
/// (or on its first error) detection stays on the CPU. Returns why it was not used.
pub fn accelerate(&self, accelerator: std::sync::Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), String> {
let side = self.side;
let probe = Tensor::new(vec![1, 3, side, side], probe_pixels(3 * side * side)).map_err(|e| e.to_string())?;
self.net.attach_checked(accelerator, &probe, ACCELERATOR_TOLERANCE, require_faster)
}
/// The square input side, used to check an installed manifest.
pub fn input_side(&self) -> usize {
self.side
+127 -20
View File
@@ -10,7 +10,7 @@
//! the downloading thread. No proxies (see docs/ai-masks.md).
use std::io::{ErrorKind, Read, Write};
use std::net::{TcpStream, ToSocketAddrs};
use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
@@ -88,6 +88,9 @@ impl Url {
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
let host = v6.get(..end).unwrap_or_default();
let after = v6.get(end + 1..).unwrap_or_default();
if !after.is_empty() && !after.starts_with(':') {
return Err(bad("unexpected text after IPv6 address"));
}
(host.to_string(), after.strip_prefix(':'))
} else {
match authority.rsplit_once(':') {
@@ -182,27 +185,68 @@ fn tls_config() -> Result<Arc<rustls::ClientConfig>, HttpError> {
Ok(Arc::new(config))
}
/// Resolve a host name within `deadline`, checking the cancel flag while it runs. The system resolver blocks with no
/// timeout of its own, so it runs on a helper thread (which ends when the resolver returns); a lookup that takes too
/// long is given up on, and Cancel works during it (issue #1532).
fn lookup(
host: &str,
resolve: impl FnOnce() -> std::io::Result<Vec<SocketAddr>> + Send + 'static,
deadline: Instant,
limits: &Limits,
) -> Result<Vec<SocketAddr>, HttpError> {
if limits.cancel.load(Ordering::Relaxed) {
return Err(HttpError::Cancelled);
}
let (tx, rx) = std::sync::mpsc::channel();
std::thread::Builder::new()
.name("lc-fetch-lookup".into())
.spawn(move || {
let _ = tx.send(resolve());
})
.map_err(|e| HttpError::Connect(format!("{host}: {e}")))?;
loop {
if limits.cancel.load(Ordering::Relaxed) {
return Err(HttpError::Cancelled);
}
let left = deadline.saturating_duration_since(Instant::now());
if left.is_zero() {
return Err(HttpError::Connect(format!("{host}: the name lookup timed out")));
}
match rx.recv_timeout(left.min(TICK)) {
Ok(r) => return r.map_err(|e| HttpError::Connect(format!("{host}: {e}"))),
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Err(HttpError::Connect(format!("{host}: the name lookup failed"))),
}
}
}
/// Connect to the first of `addrs` that answers, all of them together within `deadline` (each used to get the whole
/// connect timeout of its own, issue #1532).
fn connect_any(host: &str, addrs: &[SocketAddr], deadline: Instant, limits: &Limits) -> Result<TcpStream, HttpError> {
let mut last = format!("{host}: no address");
for a in addrs {
if limits.cancel.load(Ordering::Relaxed) {
return Err(HttpError::Cancelled);
}
let left = deadline.saturating_duration_since(Instant::now());
if left.is_zero() {
return Err(HttpError::Connect(format!("{host}: connecting timed out")));
}
match TcpStream::connect_timeout(a, left) {
Ok(s) => return Ok(s),
Err(e) => last = format!("{host}: {e}"),
}
}
Err(HttpError::Connect(last))
}
impl Stream {
fn connect(url: &Url, limits: &Limits) -> Result<Stream, HttpError> {
let addrs: Vec<_> = (url.host.as_str(), url.port).to_socket_addrs().map_err(|e| HttpError::Connect(format!("{}: {e}", url.host)))?.collect();
if addrs.is_empty() {
return Err(HttpError::Connect(format!("{}: no address", url.host)));
}
let mut last = String::new();
let mut sock = None;
for a in addrs {
if limits.cancel.load(Ordering::Relaxed) {
return Err(HttpError::Cancelled);
}
match TcpStream::connect_timeout(&a, limits.connect) {
Ok(s) => {
sock = Some(s);
break;
}
Err(e) => last = format!("{}: {e}", url.host),
}
}
let sock = sock.ok_or(HttpError::Connect(last))?;
// the lookup and the connection attempts share one connect timeout
let deadline = Instant::now() + limits.connect;
let (host, port) = (url.host.clone(), url.port);
let addrs = lookup(&url.host, move || (host.as_str(), port).to_socket_addrs().map(Iterator::collect), deadline, limits)?;
let sock = connect_any(&url.host, &addrs, deadline, limits)?;
sock.set_read_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
sock.set_write_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
let _ = sock.set_nodelay(true);
@@ -512,6 +556,58 @@ pub fn get(url: &Url, headers: &[(&str, String)], limits: &Limits) -> Result<Res
Ok(Response { status, headers, conn, body })
}
#[cfg(test)]
mod tests_connect {
use super::*;
fn limits(cancel: &AtomicBool) -> Limits<'_> {
Limits { connect: Duration::from_millis(300), stall: Duration::from_secs(1), cancel }
}
/// A name lookup that hangs is given up on at the connect timeout, and Cancel stops it at once (issue #1532).
#[test]
fn a_hung_name_lookup_times_out_and_can_be_cancelled() {
let cancel = AtomicBool::new(false);
let slow = || {
std::thread::sleep(Duration::from_secs(10));
Ok(Vec::new())
};
let t = Instant::now();
let r = lookup("slow.example", slow, Instant::now() + Duration::from_millis(300), &limits(&cancel));
assert!(matches!(&r, Err(HttpError::Connect(m)) if m.contains("timed out")), "{r:?}");
assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed());
// Cancel while the lookup is waiting on the resolver, not before it starts
let t = Instant::now();
let r = std::thread::scope(|scope| {
scope.spawn(|| {
std::thread::sleep(Duration::from_millis(100));
cancel.store(true, Ordering::Relaxed);
});
lookup("slow.example", slow, Instant::now() + Duration::from_secs(30), &limits(&cancel))
});
assert!(matches!(r, Err(HttpError::Cancelled)), "{r:?}");
assert!(t.elapsed() >= Duration::from_millis(100) && t.elapsed() < Duration::from_secs(1), "{:?}", t.elapsed());
// and a lookup asked for after Cancel doesn't start
assert!(matches!(lookup("slow.example", slow, Instant::now() + Duration::from_secs(30), &limits(&cancel)), Err(HttpError::Cancelled)));
// a lookup that answers is used
let cancel = AtomicBool::new(false);
let local: SocketAddr = "127.0.0.1:9".parse().unwrap();
assert_eq!(lookup("local", move || Ok(vec![local]), Instant::now() + Duration::from_secs(5), &limits(&cancel)).unwrap(), [local]);
}
/// Several addresses that don't answer share one deadline instead of each getting the full timeout.
#[test]
fn addresses_share_one_connect_deadline() {
let cancel = AtomicBool::new(false);
// non-routable addresses: each either hangs until the deadline or fails at once, never longer
let addrs: Vec<SocketAddr> = (1..=5).map(|i| format!("10.255.255.{i}:81").parse().unwrap()).collect();
let t = Instant::now();
let r = connect_any("blackhole", &addrs, Instant::now() + Duration::from_millis(400), &limits(&cancel));
assert!(r.is_err());
assert!(t.elapsed() < Duration::from_millis(1500), "{:?}", t.elapsed());
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -537,6 +633,17 @@ mod tests {
}
}
#[test]
fn rejects_text_after_a_bracketed_ipv6_host() {
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
assert!(matches!(Url::parse(bad), Err(HttpError::BadUrl(_))), "{bad}");
}
let u = Url::parse("http://[::1]:8080/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
let u = Url::parse("http://[::1]/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
}
#[test]
fn resolves_redirects() {
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
+10 -1
View File
@@ -104,7 +104,8 @@ pub struct GeoNames {
fn cell(p: LatLon) -> (i32, i32) {
// valid positions only: |lat| ≤ 90, |lon| ≤ 180, so the casts are in range
(p.lat.floor() as i32, p.lon.floor() as i32)
// longitude 180 is the same meridian as -180: key it as column -180, the wrap `reverse` probes
(p.lat.floor() as i32, (p.lon.floor() as i32 + 180).rem_euclid(360) - 180)
}
fn short(s: &str) -> String {
@@ -253,6 +254,14 @@ mod tests {
assert_eq!(fiji.reverse(LatLon::new(-17.0, -179.95)).unwrap().city, "X");
}
#[test]
fn city_on_the_antimeridian_is_found() {
let g = GeoNames::parse("1\tTaveuni\tTaveuni\t\t0.0\t180.0\tP\tPPL\tFJ\t\t01\n", None, None);
for lon in [179.9, -179.9, 180.0, -180.0] {
assert_eq!(g.reverse(LatLon::new(0.0, lon)).map(|p| p.city).as_deref(), Some("Taveuni"), "from {lon}");
}
}
#[test]
fn high_latitude_cities_two_cells_away() {
let city = |lat: f64, lon: f64| GeoNames::parse(&format!("1\tX\tX\t\t{lat}\t{lon}\tP\tP\tNO\t\t\n"), None, None);
+1
View File
@@ -14,6 +14,7 @@ lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
lightcraft-develop = { workspace = true }
lightcraft-faces = { workspace = true }
lightcraft-pipeline = { workspace = true }
log = { workspace = true }
File diff suppressed because it is too large. Load diff
+4 -2
View File
@@ -35,6 +35,8 @@ pub mod backend;
mod ctx;
#[cfg(not(target_arch = "wasm32"))]
mod exit;
#[cfg(not(target_arch = "wasm32"))]
pub mod faces;
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
pub mod nn;
#[cfg(not(target_arch = "wasm32"))]
@@ -239,9 +241,9 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {
}
}
/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the
/// Why GPU compute that has a device of its own (denoise, faces) is switched off by the user's settings, or `None`: the
/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn switched_off() -> Option<String> {
if env_disabled() {
return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
+171
View File
@@ -0,0 +1,171 @@
// Convolution for the face networks (lightcraft-faces), planar (NCHW) float32, batch 1, any stride, padding and grouping.
//
// `tiled` is an implicit matrix product per group: rows are the group's output channels (M), columns its output
// pixels (N), and K runs over input channel x kernel tap. The input is gathered (im2col) straight into workgroup
// memory, so the lowered matrix is never made. 256 threads make a 64 x 64 block, 4 x 4 values each.
// `direct` gives one output value per thread, for depthwise convolutions (one output channel per group), where a
// 64-row tile would be almost all waste.
//
// out[co][p] = bias[co] + sum_k W[co][k] * X[k][p] X[k][p] = input[g*cg + k/taps][oy*sh + ky - pt][ox*sw + kx - pl]
struct P {
m: u32, // output channels per group
n: u32, // output pixels (oh * ow)
k: u32, // cg * kh * kw
cg: u32, // input channels per group
h: u32,
w: u32,
ow: u32,
kh: u32,
kw: u32,
sh: u32,
sw: u32,
pt: u32,
pl: u32,
bias: u32, // 1 when there is a bias
total: u32, // cout * n
row: u32, // workgroups along x, for `direct` and `reduce` (so a large layer can use y too)
splits: u32, // K is cut into this many slices (z = group * splits + slice), summed by `reduce`
kchunk: u32, // K values per slice, a multiple of BK
_a: u32,
_b: u32,
}
@group(0) @binding(0) var<uniform> p: P;
@group(0) @binding(1) var<storage, read> src: array<f32>;
@group(0) @binding(2) var<storage, read> wgt: array<f32>;
@group(0) @binding(3) var<storage, read> bias: array<f32>;
@group(0) @binding(4) var<storage, read_write> dst: array<f32>;
const BM: u32 = 64u;
const BN: u32 = 64u;
const BK: u32 = 16u;
var<workgroup> As: array<f32, 1024>; // [BK][BM]
var<workgroup> Bs: array<f32, 1024>; // [BK][BN]
fn gather(g: u32, k: u32, pix: u32) -> f32 {
let taps = p.kh * p.kw;
let ci = g * p.cg + k / taps;
let tap = k % taps;
let oy = pix / p.ow;
let ox = pix % p.ow;
let iy = i32(oy * p.sh + tap / p.kw) - i32(p.pt);
let ix = i32(ox * p.sw + tap % p.kw) - i32(p.pl);
if (iy < 0 || ix < 0 || iy >= i32(p.h) || ix >= i32(p.w)) {
return 0.0;
}
return src[(ci * p.h + u32(iy)) * p.w + u32(ix)];
}
@compute @workgroup_size(256)
fn tiled(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
let n0 = wg.x * BN;
let m0 = wg.y * BM;
let g = wg.z / p.splits;
let slice = wg.z % p.splits;
let k_lo = slice * p.kchunk;
let k_hi = min(p.k, k_lo + p.kchunk);
let tx = tid % 16u;
let ty = tid / 16u;
var acc: array<array<f32, 4>, 4>;
for (var i = 0u; i < 4u; i++) {
for (var j = 0u; j < 4u; j++) {
acc[i][j] = 0.0;
}
}
for (var k0 = k_lo; k0 < k_hi; k0 += BK) {
// 1024 values of each tile, 4 per thread; every slot has exactly one writer
for (var e = tid; e < 1024u; e += 256u) {
// weights: neighbouring threads read neighbouring k of one row
let mm = e / BK;
let ka = k0 + e % BK;
let m = m0 + mm;
var a = 0.0;
if (ka < k_hi && m < p.m) {
a = wgt[(g * p.m + m) * p.k + ka];
}
As[(e % BK) * BM + mm] = a;
// input: neighbouring threads read neighbouring pixels
let kk = e / BN;
let k = k0 + kk;
let pix = n0 + e % BN;
var b = 0.0;
if (k < k_hi && pix < p.n) {
b = gather(g, k, pix);
}
Bs[e] = b;
}
workgroupBarrier();
for (var kk = 0u; kk < BK; kk++) {
var a: array<f32, 4>;
var b: array<f32, 4>;
for (var i = 0u; i < 4u; i++) {
a[i] = As[kk * BM + ty * 4u + i];
b[i] = Bs[kk * BN + tx + 16u * i];
}
for (var i = 0u; i < 4u; i++) {
for (var j = 0u; j < 4u; j++) {
acc[i][j] += a[i] * b[j];
}
}
}
workgroupBarrier();
}
for (var i = 0u; i < 4u; i++) {
let m = m0 + ty * 4u + i;
if (m >= p.m) {
continue;
}
let co = g * p.m + m;
var b0 = 0.0;
if (p.bias != 0u && p.splits == 1u) {
b0 = bias[co];
}
// with slices, each writes a partial sum of its own: [slice][cout][n]
let base = slice * p.total + co * p.n;
for (var j = 0u; j < 4u; j++) {
// columns tx, tx + 16, tx + 32, tx + 48: neighbouring threads write neighbouring pixels
let pix = n0 + tx + 16u * j;
if (pix < p.n) {
dst[base + pix] = acc[i][j] + b0;
}
}
}
}
@compute @workgroup_size(256)
fn direct(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
let at = (wg.y * p.row + wg.x) * 256u + tid;
if (at >= p.total) {
return;
}
let co = at / p.n;
let pix = at % p.n;
let g = co / p.m;
var v = 0.0;
if (p.bias != 0u) {
v = bias[co];
}
for (var k = 0u; k < p.k; k++) {
v += wgt[co * p.k + k] * gather(g, k, pix);
}
dst[at] = v;
}
// The sum of the K slices' partial results, plus the bias. `src` holds [splits][cout][n].
@compute @workgroup_size(256)
fn reduce(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
let at = (wg.y * p.row + wg.x) * 256u + tid;
if (at >= p.total) {
return;
}
var v = 0.0;
if (p.bias != 0u) {
v = bias[at / p.n];
}
for (var s = 0u; s < p.splits; s++) {
v += src[s * p.total + at];
}
dst[at] = v;
}
Loaded 100 of 205 files, more files were not shown because too many files have changed in this diff. Show more