Merge PR #356

# Conflicts:
#	docs/parity.md

crates/raw/src/vendor/cr2.rs

@@ -15,14 +15,15 @@
 //! - `SensorInfo` (maker note `0x00e0`): sensor width/height and the left/top/right/bottom borders of the image
 //!   area; the masked columns left of it give the black level.
 //! - `ColorBalance` (maker note `0x4001`): as-shot `RGGB` levels at a model-dependent offset; we probe the known
-//!   offsets and accept the first plausible quadruple.
+//!   offsets and accept the first plausible quadruple. The array is 16-bit words; some models store it as UNDEFINED
+//!   bytes (ColorData versions -3 and -4), which are paired up in the maker note's byte order first.
 //! - sRAW / mRAW (YCbCr, subsampled) are not supported yet.
 
 use super::{black_from_columns, white_from_data};
 use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg};
 use lightcraft_geom::Orientation;
 use lightcraft_tiff::image::chunk_bytes;
-use lightcraft_tiff::{Ifd, Tiff, makernote, tags as t};
+use lightcraft_tiff::{ByteOrder, Ifd, Tiff, Value, makernote, tags as t};
 
 const CR2_SLICE: u16 = 0xc640;
 const SRAW_TYPE: u16 = 0xc6c5;
@@ -46,6 +47,17 @@ fn cfa_from_tag(v: Option<u64>) -> Option<Cfa> {
     Some(Cfa::bayer_static(name))
 }
 
+/// The 16-bit words of a maker-note array. Canon writes `ColorData` as SHORT in most models, but as UNDEFINED
+/// bytes (count = byte length, e.g. 5120) in the PowerShot / EOS M models with ColorData versions -3 and -4;
+/// those bytes are the same 16-bit words in the maker note's byte order, so they are paired up, not read one by
+/// one. Other types are returned value by value.
+fn words(v: &Value, order: ByteOrder) -> Vec<u64> {
+    match v {
+        Value::Byte(b) | Value::Undefined(b) => b.as_chunks::<2>().0.iter().map(|c| u64::from(order.u16(*c))).collect(),
+        _ => v.to_u64_vec(),
+    }
+}
+
 /// As-shot WB multipliers (R, G, B; G = 1) from the ColorBalance array.
 fn wb_from_color_balance(v: &[u64]) -> Option<[f32; 3]> {
     for off in [63usize, 25, 24, 34, 71, 85, 105, 69, 77] {
@@ -134,7 +146,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
             active = Rect::new(l, tp, r - l + 1, b - tp + 1);
         }
     }
-    let wb = mn.as_ref().and_then(|m| m.ifd.u64s(COLOR_BALANCE)).and_then(|v| wb_from_color_balance(&v));
+    let wb = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order))).and_then(|v| wb_from_color_balance(&v));
     let cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN)).unwrap_or_else(|| Cfa::bayer_static("RGGB"));
     let black = if active.x >= 8 {
         black_from_columns(&data, width, 2..active.x - 2, active.y..active.y + active.height, active)
@@ -175,6 +187,11 @@ pub(crate) mod tests {
 
     /// Build a synthetic CR2: IFD0..IFD2 placeholders + IFD3 holding a sliced 2-component LJ92 frame.
     pub(crate) fn synthetic_cr2(w: usize, h: usize, slices: Option<[u16; 3]>) -> (Vec<u8>, Vec<u16>) {
+        synthetic_cr2_note(w, h, slices, None)
+    }
+
+    /// Like [`synthetic_cr2`], with `note` as the Exif maker note (bytes as stored).
+    fn synthetic_cr2_note(w: usize, h: usize, slices: Option<[u16; 3]>, note: Option<Vec<u8>>) -> (Vec<u8>, Vec<u16>) {
         let img: Vec<u16> = (0..w * h).map(|i| 1024 + ((i * 7919) % 12000) as u16).collect();
         let widths: Vec<usize> = match slices {
             Some([n, sw, last]) => {
@@ -200,6 +217,9 @@ pub(crate) mod tests {
         // maker note: SensorInfo + ColorBalance, plain IFD with offsets relative to the TIFF header
         let mut exif = IfdBuilder::new();
         exif.set(t::ISO_SPEED, Value::Short(vec![400]));
+        if let Some(n) = note {
+            exif.set(t::MAKER_NOTE, Value::Undefined(n));
+        }
         ifd0.set_child(t::EXIF_IFD, exif);
         let mut ifd3 = IfdBuilder::new();
         ifd3.set(t::COMPRESSION, Value::Short(vec![6]));
@@ -251,4 +271,62 @@ pub(crate) mod tests {
         assert!((wb[0] - 2000.0 / 1025.0).abs() < 1e-4 && (wb[2] - 1500.0 / 1025.0).abs() < 1e-4);
         assert!(wb_from_color_balance(&[0; 10]).is_none());
     }
+
+    #[test]
+    fn wb_from_undefined_bytes() {
+        // hand-coded: ColorData of an S110-like file, UNDEFINED bytes, little-endian words; word 71.. = 1988 745 745 1641
+        let mut b = vec![0u8; 2 * 120];
+        b[0..2].copy_from_slice(&[0xfd, 0xff]); // version -3
+        b[142..150].copy_from_slice(&[0xc4, 0x07, 0xe9, 0x02, 0xe9, 0x02, 0x69, 0x06]);
+        let le = words(&Value::Undefined(b.clone()), ByteOrder::Little);
+        assert_eq!(&le[71..75], &[1988, 745, 745, 1641]);
+        let wb = wb_from_color_balance(&le).unwrap();
+        assert!((wb[0] - 1988.0 / 745.0).abs() < 1e-5 && wb[1] == 1.0 && (wb[2] - 1641.0 / 745.0).abs() < 1e-5);
+        // read as single bytes (the old behaviour) nothing is plausible
+        assert!(wb_from_color_balance(&b.iter().map(|&x| u64::from(x)).collect::<Vec<_>>()).is_none());
+        // big-endian note: same words, bytes swapped
+        let mut be = b.clone();
+        be.as_chunks_mut::<2>().0.iter_mut().for_each(|c| c.swap(0, 1));
+        assert_eq!(words(&Value::Undefined(be), ByteOrder::Big), le);
+        // SHORT values pass through, an odd trailing byte is ignored
+        assert_eq!(words(&Value::Short(vec![1, 2, 3]), ByteOrder::Little), vec![1, 2, 3]);
+        assert_eq!(words(&Value::Undefined(vec![1, 0, 2]), ByteOrder::Little), vec![1]);
+    }
+
+    /// A maker note holding only `ColorData` (`0x4001`) as `ty` (3 = SHORT, 7 = UNDEFINED) with `n` values, a plain
+    /// IFD whose value offset is relative to the TIFF header (patched once the note's position is known).
+    fn note_with_color_data(ty: u16, data: &[u8]) -> Vec<u8> {
+        let count = if ty == 3 { data.len() / 2 } else { data.len() } as u32;
+        let mut note = Vec::new();
+        note.extend_from_slice(&1u16.to_le_bytes());
+        note.extend_from_slice(&COLOR_BALANCE.to_le_bytes());
+        note.extend_from_slice(&ty.to_le_bytes());
+        note.extend_from_slice(&count.to_le_bytes());
+        note.extend_from_slice(&0u32.to_le_bytes()); // value offset, patched below
+        note.extend_from_slice(&0u32.to_le_bytes()); // next IFD
+        note.extend_from_slice(data);
+        note
+    }
+
+    fn decode_with_color_data(ty: u16, data: &[u8]) -> RawImage {
+        let (mut bytes, _) = synthetic_cr2_note(64, 10, None, Some(note_with_color_data(ty, data)));
+        let tiff = Tiff::parse(&bytes).unwrap();
+        let e = tiff.exif().unwrap().get(t::MAKER_NOTE).unwrap();
+        let off = e.offset as usize;
+        bytes[off + 10..off + 14].copy_from_slice(&(e.offset as u32 + 18).to_le_bytes());
+        crate::decode(&bytes).unwrap()
+    }
+
+    /// `ColorData` stored as UNDEFINED bytes (versions -3 / -4: PowerShot, EOS M) gives the as-shot WB from word 71,
+    /// exactly as the same words stored as SHORT do.
+    #[test]
+    fn color_data_stored_as_bytes() {
+        let mut bytes = vec![0u8; 2 * 120];
+        bytes[0..2].copy_from_slice(&[0xfd, 0xff]); // version -3
+        bytes[142..150].copy_from_slice(&[0xc4, 0x07, 0xe9, 0x02, 0xe9, 0x02, 0x69, 0x06]); // 1988 745 745 1641
+        for ty in [7, 3] {
+            let wb = decode_with_color_data(ty, &bytes).wb_multipliers.unwrap_or_else(|| panic!("type {ty}: no WB"));
+            assert!((wb[0] - 1988.0 / 745.0).abs() < 1e-5 && wb[1] == 1.0 && (wb[2] - 1641.0 / 745.0).abs() < 1e-5, "type {ty}: {wb:?}");
+        }
+    }
 }

crates/raw/tests/corpus.rs

@@ -166,6 +166,34 @@ fn corpus_cr2_cfa_patterns() {
     eprintln!("CR2 CFA layouts checked on {seen} files");
 }
 
+/// The as-shot white balance of Canon CR2s whose ColorData is stored as SHORT words stays readable (the PowerShot /
+/// EOS M files that store it as UNDEFINED bytes are covered by the unit test in `vendor/cr2.rs`; none is in this corpus).
+/// Daylight-ish CR2s have R/G above 1 and B/G between 1 and 2.5 (observed 1.4-2.7 on these nine bodies).
+#[test]
+fn corpus_cr2_as_shot_white_balance() {
+    let dir = corpus_root().join("raw");
+    let mut seen = 0;
+    for name in [
+        "cr2-canon-40d.cr2",
+        "cr2-canon-550d.cr2",
+        "cr2-canon-5d2.cr2",
+        "cr2-canon-5d3.cr2",
+        "cr2-canon-5dsr.cr2",
+        "cr2-canon-6d.cr2",
+        "cr2-canon-7d.cr2",
+        "cr2-canon-80d.cr2",
+    ] {
+        let Ok(bytes) = std::fs::read(dir.join(name)) else {
+            eprintln!("skip: {name} absent");
+            continue;
+        };
+        let wb = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}")).wb_multipliers.unwrap_or_else(|| panic!("{name}: no as-shot WB"));
+        assert!(wb[1] == 1.0 && wb[0] > 1.0 && wb[2] > 0.5 && wb[0] < 4.0 && wb[2] < 4.0, "{name}: WB {wb:?}");
+        seen += 1;
+    }
+    eprintln!("CR2 as-shot WB checked on {seen} files");
+}
+
 /// raw.pixls.us has the same D5100 scene as 14-bit lossless compressed and uncompressed NEF: the Huffman decode must
 /// match the uncompressed image (up to the small differences between two exposures).
 #[test]

docs/parity.md

@@ -135,7 +135,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
 | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) |
 | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars |
 | LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). CR3 opens from its full-size embedded JPEG with full metadata (Exif, GPS, XMP from the `CMT1`/`CMT2`/`CMT4` boxes, `crates/meta/src/cr3.rs`) but its raw data is not decoded yet. Missing: CR3 raw decoding (CRX), compressed RAF/ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode, palette (indexed-colour) TIFF (refused by the `tiff` crate) |
-| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. |
+| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. |
 | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic |
 | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving |
 | LR-IMP-DNG-CONVERT | Convert to DNG on import [Classic] | P2 | ✅ | `cmd:library.import` (`dng`), `crates/ui-egui/src/import.rs` | copy imports: Raw files ▸ Copy as DNG (lossless; the card is untouched); the raw copy is removed only once its DNG is verified and on disk (issue #106) |