diff --git a/Cargo.lock b/Cargo.lock index 45d8ff15c..ba0915cf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3118,6 +3118,7 @@ dependencies = [ "lightcraft-fetch", "lightcraft-geom", "lightcraft-gpu", + "lightcraft-heif", "lightcraft-merge", "lightcraft-meta", "lightcraft-pipeline", @@ -3192,6 +3193,7 @@ name = "lightcraft-heif" version = "0.4.0" dependencies = [ "heic-rs", + "rayon", ] [[package]] diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index 18d00ac3e..5c310cf09 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -44,3 +44,5 @@ avif = ["dep:ravif"] [dev-dependencies] proptest = { workspace = true } +# Lossless synthetic HEIC files for the `heif` feature's tests. +lightcraft-heif = { workspace = true, features = ["testdata"] } diff --git a/crates/codecs/README.md b/crates/codecs/README.md index 18743ad03..a4fecef62 100644 --- a/crates/codecs/README.md +++ b/crates/codecs/README.md @@ -47,7 +47,7 @@ CRW, MRW, X3F) are detected so the engine can route them to `lightcraft-raw`; `d | PSD / PSB | merged composite: 8/16/32-bit gray, RGB, CMYK, indexed, duotone (as gray); raw/RLE | — | ICC (1039), EXIF (1058), XMP (1060); ZIP-compressed composite, Lab and 1-bit unsupported | | JPEG XL | yes (jxl-oxide, feature `jxl`, default on) | — | enum colour → rendered straight to linear Rec.2020; ICC → our ICC path; orientation applied by the decoder (reported as 1) | | AVIF | **no** | yes (ravif/rav1e, native only, feature `avif`) | 8-bit sRGB, EXIF; no ICC in the muxer | -| HEIC/HEIF | **no** (sniff only) | — | see gaps | +| HEIC/HEIF | yes with feature `heif` (lightcraft-heif: heic-rs + libheif-matching colour; 4:2:0/mono, 8–10-bit, grids, irot/imir/clap, ICC/nclx/VUI, EXIF/XMP, thumbnail item) | — | 4:2:2/4:4:4 (Canon/Sony HIF) unsupported | ## Colour @@ -100,9 +100,11 @@ Run `cargo test --release -p lightcraft-codecs --test bench -- --ignored --nocap ## Gaps / limitations -- **AVIF and HEIC decode**: no permissively licensed, pure-Rust AV1/HEVC decoder exists today - (dav1d/libheif are C; rav1d-safe and imazen's heic are AGPL). Files are sniffed and rejected with - `Error::Unsupported`; a raw-style embedded-preview path could be added for HEIC thumbnails. +- **AVIF decode**: no permissively licensed, pure-Rust AV1 decoder exists today (dav1d is C; + rav1d-safe is AGPL). Files are sniffed and rejected with `Error::Unsupported`. +- **HEIC** needs the `heif` feature (off by default: HEVC patents are the distributor's call); + without it, `Error::Unsupported` says the build has no HEIC support. 4:2:2 and 4:4:4 HEIF + (Canon/Sony HIF) is refused: heic-rs 0.1.1 loses sync on those bitstreams. - JPEG: arithmetic coding unsupported (as in both decoders); extended XMP (> 64 KiB) neither read nor written; EXIF > 64 KiB not written. The encoder uses standard Huffman tables (interleaved baseline) for maximum compatibility. diff --git a/crates/codecs/src/heif.rs b/crates/codecs/src/heif.rs index fe91c0af5..702a0d27d 100644 --- a/crates/codecs/src/heif.rs +++ b/crates/codecs/src/heif.rs @@ -1,16 +1,18 @@ //! HEIF / HEIC (the iPhone and Mac photo format), read-only. //! //! Decoding lives in the optional `lightcraft-heif` crate (heic-rs, a pure-Rust HEVC -//! still-picture decoder), enabled by this crate's `heif` feature — off by default because HEVC -//! is patent-encumbered and whether a build carries an HEVC decoder is the distributor's call -//! (the same policy as the sibling PhotoCraft). Without the feature, HEIF files are still -//! detected and opening one is an [`Error::Unsupported`] error, never a panic. +//! still-picture decoder, plus a libheif-matching colour conversion), enabled by this crate's +//! `heif` feature — off by default because HEVC is patent-encumbered and whether a build carries +//! an HEVC decoder is the distributor's call (the same policy as the sibling PhotoCraft). Without +//! the feature, HEIF files are still detected and opening one is an [`Error::Unsupported`] error +//! saying so, never a panic. //! //! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF, //! so the decoder applies them and `orientation` is reported as 1: the EXIF Orientation tag only //! mirrors what the container says, and applying it as well would turn the photo twice. -//! The samples come back in the container's own primaries with its ICC profile (if any); like -//! every decoder here, the colour interpretation happens in [`crate::convert::finish`]. +//! The samples come back as 16-bit RGB in the container's own primaries, with its ICC profile +//! (iPhones: Display P3) or, without one, its `nclx` / VUI colour description; like every decoder +//! here, the colour interpretation happens in [`crate::convert::finish`]. #[cfg(feature = "heif")] use crate::convert::{Buf, Meta, Model, Raw, finish}; @@ -22,26 +24,32 @@ const F: Format = Format::Heif; #[cfg(not(feature = "heif"))] pub(crate) const NOT_IN_BUILD: &str = "HEIC/HEIF support isn't included in this build of LightCraft"; -/// The reason a HEIF file that no build can open is refused (this crate decodes HEVC stills -/// only; the wrapper names the exact construct in its own errors). -#[cfg(feature = "heif")] -const NOT_DECODED: &str = "the image is not a decodable HEVC still (image sequence, overlay, alpha-less grid or AVIF-coded item)"; - #[cfg(not(feature = "heif"))] pub(crate) fn decode(_bytes: &[u8], _opts: &DecodeOptions) -> Result { Err(Error::Unsupported(F, NOT_IN_BUILD)) } +#[cfg(not(feature = "heif"))] +pub(crate) fn header(_bytes: &[u8]) -> Result<(u32, u32, u16)> { + Err(Error::Unsupported(F, NOT_IN_BUILD)) +} + #[cfg(feature = "heif")] fn err(e: lightcraft_heif::Error) -> Error { match e { - // This crate's Unsupported takes a fixed reason; the dynamic wording lives in the - // lightcraft-heif error itself. - lightcraft_heif::Error::Unsupported(_) => Error::Unsupported(F, NOT_DECODED), + lightcraft_heif::Error::Unsupported(why) => Error::Unsupported(F, why), lightcraft_heif::Error::Limit(m) | lightcraft_heif::Error::Malformed(m) => Error::Malformed(F, m), } } +/// The displayed size (container transforms applied, so orientation 1), from the container +/// alone: no picture is decoded. +#[cfg(feature = "heif")] +pub(crate) fn header(bytes: &[u8]) -> Result<(u32, u32, u16)> { + let info = lightcraft_heif::probe(bytes).map_err(err)?; + Ok((info.width, info.height, 1)) +} + #[cfg(feature = "heif")] pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result { // The container alone: the declared size is checked before any pixel is decoded. @@ -50,16 +58,22 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result { return Err(Error::TooLarge(info.width as u64, info.height as u64)); } let options = lightcraft_heif::Options { max_pixels: opts.max_pixels, apply_transforms: true }; - let decoded = lightcraft_heif::decode(bytes, &options).map_err(err)?; - // The decoder re-reads the container; trust its output's shape, not the probe's. - let buf = if decoded.sixteen_bit { - let mut v = Vec::with_capacity(decoded.data.len() / 2); - for pair in decoded.data.as_chunks::<2>().0 { - v.push(u16::from_ne_bytes(*pair)); + // A small decode (a thumbnail or a cataloguing probe) uses the file's own thumbnail image when + // it is at least as large as asked: iPhones store one of 320 × 240, and decoding it is a + // hundredth of the work of the 12 MP photo. + let thumb = match opts.max_size { + Some((mw, mh)) if mw > 0 && mh > 0 && mw.max(mh) < info.width.max(info.height) => { + lightcraft_heif::decode_thumbnail(bytes, mw.max(mh), &options).ok().flatten().filter(|t| { + // Only a thumbnail of the same picture: same orientation and aspect within 1 %. + let aspect = |w: u32, h: u32| w as f64 / h.max(1) as f64; + (aspect(t.width, t.height) / aspect(info.width, info.height) - 1.0).abs() < 0.01 + }) } - Buf::U16(v) - } else { - Buf::U8(decoded.data) + _ => None, + }; + let mut decoded = match thumb { + Some(t) => t, + None => lightcraft_heif::decode(bytes, &options).map_err(err)?, }; let raw = Raw { width: decoded.width as usize, @@ -67,11 +81,20 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result { model: Model::Rgb, alpha: decoded.has_alpha, premultiplied: false, - buf, - bit_depth: if decoded.sixteen_bit { 16 } else { 8 }, + buf: Buf::U16(std::mem::take(&mut decoded.data)), + bit_depth: decoded.bit_depth, }; - let meta = Meta { icc: decoded.icc, exif: decoded.exif, xmp: decoded.xmp, orientation: Some(1), ..Default::default() }; - finish(F, raw, meta, (decoded.width, decoded.height), opts) + let c = decoded.colour; + let meta = Meta { + icc: decoded.icc, + exif: decoded.exif, + xmp: decoded.xmp, + orientation: Some(1), + // Used when there is no (usable) ICC profile. + hint: crate::space::SourceSpace::from_cicp(c.primaries, c.transfer), + ..Default::default() + }; + finish(F, raw, meta, (info.width, info.height), opts) } #[cfg(test)] @@ -96,6 +119,8 @@ mod tests { fn without_the_feature_heif_is_a_clear_unsupported_error() { let r = crate::decode(HEIC_HEADER, crate::DecodeOptions::default()); assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{r:?}"); + let h = crate::read_header(HEIC_HEADER); + assert!(matches!(&h, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{h:?}"); } /// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice @@ -113,6 +138,7 @@ mod tests { fn a_heic_rs_panic_is_a_malformed_error() { let r = crate::decode(&HEIC_RS_BOX_PANIC, crate::DecodeOptions::default()); assert!(matches!(&r, Err(crate::Error::Malformed(Format::Heif, _))), "{r:?}"); + assert!(crate::read_header(&HEIC_RS_BOX_PANIC).is_err()); } #[cfg(feature = "heif")] @@ -120,6 +146,6 @@ mod tests { fn an_image_sequence_is_unsupported() { // An ftyp with no meta box: a video track, not a still. let r = crate::decode(b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc", crate::DecodeOptions::default()); - assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, _))), "{r:?}"); + assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("sequence")), "{r:?}"); } } diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index 169f40c11..cac1c4aad 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -171,9 +171,9 @@ pub struct Header { } /// Read the stored dimensions and orientation of any format [`decode`] supports, agreeing with -/// what a decode reports, without decoding pixels: JPEG, PNG, TIFF, WebP and PSD are read from -/// their headers (markers, chunks, IFD, resources). GIF, BMP and JPEG XL fall back to a small -/// decode. +/// what a decode reports, without decoding pixels: JPEG, PNG, TIFF, WebP, PSD and HEIF are read +/// from their headers (markers, chunks, IFD, resources, boxes). GIF, BMP and JPEG XL fall back to +/// a small decode. /// /// The header readers also refuse what a decode would refuse before reaching the pixels (an /// unsupported layout, compression or frame type) and files whose image data is cut short @@ -196,6 +196,7 @@ pub fn read_header_unguarded(bytes: &[u8], format: Format) -> Result
{ Format::Tiff => tiff_codec::header(bytes)?, Format::WebP => webp::header(bytes)?, Format::Psd => psd::header(bytes)?, + Format::Heif => heif::header(bytes)?, // rare here (or, for JPEG XL, without a header reader yet): a small decode, as before _ => { let d = decode_unguarded(bytes, format, &DecodeOptions::fit(64, 64))?; @@ -217,6 +218,8 @@ pub fn decode_jpeg_with_fallback(bytes: &[u8], opts: DecodeOptions, fallback: Na #[doc(hidden)] pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> Result { let opts = *opts; + // The reason a format can't be decoded lives in one place, `Format::not_decodable`. + let unsupported = || Err(Error::Unsupported(format, format.not_decodable().unwrap_or("not decodable in this build"))); match format { Format::Jpeg => jpeg::decode(bytes, &opts), Format::Png => png_codec::decode(bytes, &opts), @@ -227,10 +230,9 @@ pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> R #[cfg(feature = "jxl")] Format::Jxl => jxl::decode(bytes, &opts), #[cfg(not(feature = "jxl"))] - Format::Jxl => Err(Error::Unsupported(format, "built without the `jxl` feature")), + Format::Jxl => unsupported(), Format::Heif => heif::decode(bytes, &opts), - Format::Avif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed AV1 decoder yet")), - Format::RawTiffLike | Format::RawOther => Err(Error::Unsupported(format, "camera raw: decode with lightcraft-raw")), + Format::Avif | Format::RawTiffLike | Format::RawOther => unsupported(), } } diff --git a/crates/codecs/src/png_codec.rs b/crates/codecs/src/png_codec.rs index 600f60b05..098f1cfaa 100644 --- a/crates/codecs/src/png_codec.rs +++ b/crates/codecs/src/png_codec.rs @@ -127,25 +127,10 @@ pub(crate) fn header(b: &[u8]) -> Result<(u32, u32, u16)> { } fn container_hint(info: &png::Info) -> Option { - if let Some(c) = &info.coding_independent_code_points { - let named = match c.color_primaries { - 1 => Some(NamedSpace::Srgb), - 9 => Some(NamedSpace::Rec2020), - 12 => Some(NamedSpace::DisplayP3), - _ => None, - }; - let trc = match c.transfer_function { - 13 => Some(Trc::Srgb), - 1 | 6 | 14 | 15 => Some(Trc::Rec709), - 8 => Some(Trc::Linear), - 4 => Some(Trc::Gamma(2.2)), - _ => None, - }; - if let (Some(n), Some(t)) = (named, trc) { - let mut s = SourceSpace::named(n, SpaceOrigin::Container); - s.trc = Some([t.clone(), t.clone(), t]); - return Some(s); - } + if let Some(s) = + info.coding_independent_code_points.as_ref().and_then(|c| SourceSpace::from_cicp(c.color_primaries.into(), c.transfer_function.into())) + { + return Some(s); } if info.srgb.is_some() { return Some(SourceSpace::named(NamedSpace::Srgb, SpaceOrigin::Container)); diff --git a/crates/codecs/src/sniff.rs b/crates/codecs/src/sniff.rs index 50fa3ccf7..5bc5c4027 100644 --- a/crates/codecs/src/sniff.rs +++ b/crates/codecs/src/sniff.rs @@ -64,11 +64,20 @@ impl Format { /// Whether [`crate::decode`] can decode this format in this build. pub fn can_decode(self) -> bool { + self.not_decodable().is_none() + } + + /// Why [`crate::decode`] can't decode this format in this build (`None`: it can). The + /// [`crate::Error::Unsupported`] a decode returns carries the same reason. + pub fn not_decodable(self) -> Option<&'static str> { match self { - Format::Jxl => cfg!(feature = "jxl"), - Format::Heif => cfg!(feature = "heif"), - Format::Avif | Format::RawTiffLike | Format::RawOther => false, - _ => true, + #[cfg(not(feature = "jxl"))] + Format::Jxl => Some("built without the `jxl` feature"), + #[cfg(not(feature = "heif"))] + Format::Heif => Some(crate::heif::NOT_IN_BUILD), + Format::Avif => Some("no pure-Rust, permissively licensed AV1 decoder yet"), + Format::RawTiffLike | Format::RawOther => Some("camera raw: decode with lightcraft-raw"), + _ => None, } } diff --git a/crates/codecs/src/space.rs b/crates/codecs/src/space.rs index a945fe726..d8ee0d19f 100644 --- a/crates/codecs/src/space.rs +++ b/crates/codecs/src/space.rs @@ -255,6 +255,27 @@ impl SourceSpace { SourceSpace { named: Some(n), to_xyz_d50: n.to_xyz_d50(), trc: Some([t.clone(), t.clone(), t]), origin } } + /// The space named by ITU-T H.273 code points (PNG `cICP`, HEIF `nclx`, HEVC VUI), when + /// both the primaries and the transfer are ones this crate models. + pub(crate) fn from_cicp(primaries: u16, transfer: u16) -> Option { + let named = match primaries { + 1 => NamedSpace::Srgb, + 9 => NamedSpace::Rec2020, + 12 => NamedSpace::DisplayP3, + _ => return None, + }; + let trc = match transfer { + 13 => Trc::Srgb, + 1 | 6 | 14 | 15 => Trc::Rec709, + 8 => Trc::Linear, + 4 => Trc::Gamma(2.2), + _ => return None, + }; + let mut s = SourceSpace::named(named, SpaceOrigin::Container); + s.trc = Some([trc.clone(), trc.clone(), trc]); + Some(s) + } + /// Linear source RGB → linear `dst` RGB (Bradford from D50 to `dst`'s white). pub fn to_space(&self, dst: &RgbSpace) -> Mat3 { if let Some(n) = self.named { diff --git a/crates/codecs/tests/heif.rs b/crates/codecs/tests/heif.rs new file mode 100644 index 000000000..d14f51ac4 --- /dev/null +++ b/crates/codecs/tests/heif.rs @@ -0,0 +1,98 @@ +//! HEIC/HEIF through the codec API (feature `heif`), on lossless synthetic files from +//! `lightcraft_heif::testdata`: an iPhone-like grid with a Display P3 profile, rotation, EXIF, a +//! thumbnail and an HDR gain map; `nclx`-only colour; 10-bit; and hostile input. +#![cfg(feature = "heif")] + +use lightcraft_codecs::*; +use lightcraft_heif::testdata::{Prop, Spec, build}; + +fn grey(x: u32, y: u32) -> [u16; 3] { + [((x * 7 + y * 13) % 220 + 20) as u16, 128, 128] +} + +/// EXIF whose Orientation (6) only mirrors the container's `irot`: it must not turn the photo again. +fn exif_orientation_6() -> Vec { + let mut t = b"MM\0\x2a\0\0\0\x08\0\x01".to_vec(); + t.extend_from_slice(&[0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, 6, 0, 0]); + t.extend_from_slice(&[0, 0, 0, 0]); + t +} + +/// The thumbnail is flat mid grey so a test can tell it from the photo. +fn iphone_like() -> Vec { + build(&Spec { + props: vec![Prop::Icc(icc::write_named(NamedSpace::DisplayP3)), Prop::Irot(3)], + exif: Some(exif_orientation_6()), + thumbnail: Some(&|_, _| [126, 128, 128]), + ignored_aux: vec!["urn:com:apple:photo:2020:aux:hdrgainmap"], + ..Spec::new(96, 64, &grey) + }) +} + +#[test] +fn an_iphone_like_heic_decodes_upright_in_display_p3() { + let bytes = iphone_like(); + assert_eq!(sniff(&bytes), Some(Format::Heif)); + let d = decode(&bytes, DecodeOptions::default()).unwrap(); + // irot 3 = 90° clockwise, applied by the decoder; the EXIF Orientation is not applied again. + assert_eq!((d.width, d.height, d.source_width, d.source_height, d.orientation), (64, 96, 64, 96, 1)); + assert_eq!(d.space.named, Some(NamedSpace::DisplayP3)); + assert_eq!(d.space.origin, SpaceOrigin::IccMatrixTrc); + assert_eq!((d.bit_depth, d.has_alpha, d.float), (8, false, false)); + assert!(d.exif.is_some() && d.icc.is_some()); + // No nclx and no VUI: limited range, BT.601; neutral chroma stays neutral. + let srgb = d.to_srgb8(); + let px = srgb.data[0]; + assert!(px[0].abs_diff(px[1]) <= 1 && px[1].abs_diff(px[2]) <= 1, "{px:?}"); + + let h = read_header(&bytes).unwrap(); + assert_eq!((h.format, h.width, h.height, h.orientation), (Format::Heif, 64, 96, 1)); +} + +#[test] +fn a_small_decode_uses_the_embedded_thumbnail() { + let bytes = iphone_like(); + // The 48 × 32 thumbnail, turned like the photo, covers a 32-pixel request. + let d = decode(&bytes, DecodeOptions::fit(32, 32)).unwrap(); + assert!(d.width.max(d.height) <= 32 && d.height > d.width, "{}x{}", d.width, d.height); + assert_eq!((d.source_width, d.source_height), (64, 96)); + assert_eq!(d.space.named, Some(NamedSpace::DisplayP3)); + let flat = d.to_srgb8(); + assert!(flat.data.iter().all(|p| *p == flat.data[0]), "the flat thumbnail, not the photo"); + // A thumbnail too small for the request: the photo is decoded and resized. + let d = decode(&bytes, DecodeOptions::fit(60, 60)).unwrap(); + assert_eq!((d.width.max(d.height), d.source_width), (60, 64)); + let photo = d.to_srgb8(); + assert!(photo.data.iter().any(|p| *p != photo.data[0])); + let t = decode_thumbnail(&bytes, 32).unwrap(); + assert_eq!((t.source_width, t.source_height), (64, 96)); +} + +#[test] +fn nclx_colour_and_ten_bit() { + // Display P3 primaries, sRGB transfer, BT.709 matrix, full range: the nclx alone names the space. + let bytes = build(&Spec { bit_depth: 10, props: vec![Prop::Nclx(12, 13, 1, true)], ..Spec::new(64, 32, &|x, _| [(x * 16) as u16, 512, 512]) }); + let d = decode(&bytes, DecodeOptions::default()).unwrap(); + assert_eq!(d.space.named, Some(NamedSpace::DisplayP3)); + assert_eq!(d.space.origin, SpaceOrigin::Container); + assert_eq!(d.bit_depth, 10); + // Full-range 10-bit neutral steps of 16 code values: linear values rise monotonically. + let row: Vec = (0..64).map(|x| d.image.row(0)[x][1]).collect(); + assert!(row.windows(2).all(|w| w[1] > w[0]), "{row:?}"); +} + +#[test] +fn hostile_heic_is_an_error_never_a_panic() { + let bytes = iphone_like(); + let full = decode(&bytes, DecodeOptions::default()).unwrap(); + for cut in (12..bytes.len()).step_by(11) { + let b = &bytes[..cut]; + // Cut inside the photo: an error. Cut in a trailing item (gain map, thumbnail): the photo. + if let (Ok(_), Ok(d)) = (read_header(b), decode(b, DecodeOptions::default())) { + assert_eq!(d.image.data, full.image.data, "cut at {cut}"); + } + let _ = decode_thumbnail(b, 16); + } + let r = decode(&bytes, DecodeOptions { max_pixels: 64 * 96 - 1, ..Default::default() }); + assert!(matches!(r, Err(Error::TooLarge(64, 96))), "{r:?}"); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 9bd9845fa..4036a5c94 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -49,5 +49,7 @@ denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-g rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"] [dev-dependencies] +# Lossless synthetic HEIC files (the import test; decoded when the codecs' `heif` feature is on). +lightcraft-heif = { workspace = true, features = ["testdata"] } libc = { workspace = true } lightcraft-tiff = { workspace = true } diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index db762f009..63e61ddaa 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -151,8 +151,9 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result { }); } let fmt = lightcraft_codecs::sniff(bytes).ok_or("unrecognized file format")?; - if !fmt.can_decode() { - return Err(format!("{fmt:?} files are not supported yet")); + if let Some(why) = fmt.not_decodable() { + // e.g. "Heif files can't be opened: HEIC/HEIF support isn't included in this build of LightCraft" + return Err(format!("{fmt:?} files can't be opened: {why}")); } // headers only (issue #367: decoding the pixels was nearly all of an import's CPU time) let header = lightcraft_codecs::read_header(bytes).map_err(|e| e.to_string())?; diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 833d92e8e..b9b45fcaa 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -33,7 +33,10 @@ use crate::media::ProbeInfo; /// File extensions LightCraft imports (lower case). pub const EXTENSIONS: &[&str] = &[ "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "srw", "psd", - "jxl", "gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG) + "jxl", "gif", "bmp", + // Decoded when built with the codecs' `heif` feature; otherwise the import reports them as + // failed with the reason ("HEIC/HEIF support isn't included in this build"). + "heic", "heif", "avif", // no AV1 decoder yet: reported as failed, like a HEIC without the feature "iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff", ]; diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs index 4a4cad539..3496d957d 100644 --- a/crates/engine/src/tests_import.rs +++ b/crates/engine/src/tests_import.rs @@ -1105,3 +1105,40 @@ fn folder_import_picks_up_undecodable_raw_containers_as_preview_only() { assert!(s.catalog.photos().all(|p| p.preview_only.is_some() && p.kind == lightcraft_catalog::MediaKind::Raw && (p.width, p.height) == (40, 30))); let _ = std::fs::remove_dir_all(&src); } + +/// A HEIC laid out like an iPhone's (a grid of HEVC pictures, a Display P3 profile, a quarter turn +/// in the container, a thumbnail) imports as a normal photo that develops and exports upright like +/// a JPEG. In a build without the codecs' `heif` feature it is reported as failed, saying why. +#[test] +fn heic_imports_as_a_normal_photo_or_says_why_not() { + use lightcraft_heif::testdata::{Prop, Spec, build}; + let src = temp_dir("heic"); + let photo = |x: u32, y: u32| [((x * 7 + y * 13) % 200 + 30) as u16, 110, 150]; + let p3 = lightcraft_codecs::icc::write_named(lightcraft_codecs::NamedSpace::DisplayP3); + let bytes = build(&Spec { props: vec![Prop::Icc(p3), Prop::Irot(1)], thumbnail: Some(&photo), ..Spec::new(96, 64, &photo) }); + std::fs::write(src.join("IMG_0001.HEIC"), &bytes).unwrap(); + let mut s = Session::new().with_fs(); + let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + if !lightcraft_codecs::Format::Heif.can_decode() { + assert_eq!((ids(&r, "imported"), ids(&r, "failed")), (0, 1), "{r}"); + assert!(r["failed"].to_string().contains("isn't included in this build"), "{r}"); + let _ = std::fs::remove_dir_all(&src); + return; + } + assert_eq!((ids(&r, "imported"), ids(&r, "failed")), (1, 0), "{r}"); + let p = s.catalog.photos().next().unwrap(); + let id = p.id; + // Upright (the container's turn applied) and not preview-only. + assert_eq!((p.kind, p.preview_only.clone(), p.width, p.height), (lightcraft_catalog::MediaKind::Image, None, 64, 96)); + s.execute("library.select", &json!({"ids": [id.0], "active": id.0})).unwrap(); + let plain = crate::export::export_photo(&mut s, id, &crate::export::ExportOptions::from_json(&json!({"format": "png"})), 1).unwrap(); + assert_eq!((plain.width, plain.height), (64, 96)); + s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.0})).unwrap(); + let brighter = crate::export::export_photo(&mut s, id, &crate::export::ExportOptions::from_json(&json!({"format": "png"})), 1).unwrap(); + let mean = |png: &[u8]| { + let d = lightcraft_codecs::decode(png, Default::default()).unwrap(); + d.image.data.iter().map(|p| p[1]).sum::() / d.image.data.len() as f32 + }; + assert!(mean(&brighter.bytes) > mean(&plain.bytes) * 1.3, "exposure +1 brightens the HEIC"); + let _ = std::fs::remove_dir_all(&src); +} diff --git a/crates/heif/Cargo.toml b/crates/heif/Cargo.toml index 9a12bb2e5..b3bc69d62 100644 --- a/crates/heif/Cargo.toml +++ b/crates/heif/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightcraft-heif" -description = "Optional HEIF/HEIC still-image decoder for LightCraft (a thin, panic-guarded wrapper around heic-rs)" +description = "Optional HEIF/HEIC still-image decoder for LightCraft (heic-rs for the container and HEVC, libheif-matching colour conversion, panic-guarded)" version.workspace = true edition.workspace = true license.workspace = true @@ -9,12 +9,18 @@ license.workspace = true # Pure Rust, no unsafe. Pinned exactly: a young, single-owner crate that parses untrusted files; # moving the pin is a reviewed change (re-run the heif tests and the codecs corpus tests). # Mirrors PhotoCraft's photocraft-heif (same team, same decision): heic-rs was chosen over the -# other permissive candidates (heif-oxide lacks 4:2:2; heifer was published the same week). +# other permissive candidates (heif-oxide's HEVC decoder, rust_h265, follows FFmpeg's (LGPL) +# code structure; gamut-heic has no HEVC decoder of its own). heic-rs = { version = "=0.1.1", default-features = false, features = ["std"] } # The tiles of a HEIC photo decode in parallel on native; wasm stays single-threaded. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] heic-rs = { version = "=0.1.1", default-features = false, features = ["std", "parallel"] } +rayon = { workspace = true } + +[features] +# `lightcraft_heif::testdata`: lossless synthetic HEIF files, for the tests of the crates above. +testdata = [] [lints] workspace = true diff --git a/crates/heif/README.md b/crates/heif/README.md index e008a01c3..75db814b0 100644 --- a/crates/heif/README.md +++ b/crates/heif/README.md @@ -1,33 +1,47 @@ # lightcraft-heif -The optional HEIF / HEIC decoder of LightCraft: iPhone and Mac photos. A thin, panic-guarded wrapper -around [`heic-rs`](https://github.com/tbraun96/heic-rs), a pure-Rust HEVC still-picture decoder -(no `unsafe`, MIT OR Apache-2.0): single pictures and grid-tiled photos, 8- and 10-bit (10-bit -decodes to 16-bit), alpha auxiliary images, the container's rotation/mirror/crop, ICC, EXIF and XMP. -Read-only: writing would need an HEVC encoder. +The optional HEIF / HEIC decoder of LightCraft: iPhone and Mac photos. [`heic-rs`](https://github.com/tbraun96/heic-rs) +(pure Rust, no `unsafe`, MIT OR Apache-2.0, written from the H.265 / HEIF specifications) reads the +container and reconstructs the HEVC pictures; this crate turns them into 16-bit RGB itself so the +result matches libheif to within one code value: + +- the colour description is the item's `colr nclx` (or its first tile's), else the HEVC VUI + (iPhones: full range), else the H.265 defaults; an unspecified matrix is BT.601. heic-rs alone + assumes BT.709 limited range, about 8 code values off on every iPhone photo; +- chroma is upsampled by nearest neighbour, or centred bilinear when a `clap` starts on an odd + pixel, as libheif does; +- `imir` axis 0 flips top and bottom, as libheif writes and reads it (heic-rs names it the other way); +- a grid's ICC profile may sit on its tiles. + +Single pictures and grid-tiled photos, 4:2:0 and monochrome, 8- to 10-bit, alpha auxiliary images +(other auxiliary images — HDR gain maps, depth — are ignored), the container's rotation/mirror/crop, +ICC, EXIF and XMP, and the file's thumbnail item. 4:2:2 and 4:4:4 (Canon/Sony "HIF") are refused +with `Error::Unsupported`: heic-rs 0.1.1 loses sync on them. Read-only. ```rust let info = lightcraft_heif::probe(&bytes)?; // size, depth, alpha: check limits first let img = lightcraft_heif::decode(&bytes, &lightcraft_heif::Options::default())?; -// img.width, img.height, img.has_alpha, img.sixteen_bit, img.data (RGB/RGBA), img.icc, img.exif, img.xmp +// img.width, img.height, img.has_alpha, img.bit_depth, img.data (RGB/RGBA u16), img.colour, img.icc, img.exif, img.xmp +let thumb = lightcraft_heif::decode_thumbnail(&bytes, 256, &Default::default())?; // Option ``` -Errors are `Error::Unsupported` (image sequences, overlays, …), `Error::Limit` or +Errors are `Error::Unsupported` (image sequences, overlays, 4:2:2/4:4:4, …), `Error::Limit` or `Error::Malformed`. It never panics: every call into heic-rs runs under `catch_unwind`, and a panic -inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`. The crate is a port -of PhotoCraft's `photocraft-heif` — the sibling apps deliberately share one decoder, one pin and -one set of regression fixtures. +inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`. + +Feature `testdata` exposes `lightcraft_heif::testdata`: lossless synthetic HEIF files (PCM-coded +32 × 32 HEVC pictures, grids, every property above) for the tests of the crates above. ## Why it is a separate, optional crate - **A young decoder.** heic-rs is new and has a single maintainer. It is pinned exactly (`=0.1.1`); moving the pin is a reviewed change (re-run the crate tests and - `cargo test -p lightcraft-codecs --features heif`). -- **HEVC patents are a distributor's call.** HEVC is patent-encumbered in some jurisdictions, so - whether a build includes an HEVC decoder is a build-time choice. `lightcraft-codecs` uses this - crate only behind its non-default `heif` feature; the apps forward it - (`cargo build -p lightcraft --features heif`). Without it, HEIC files are still recognised and - opening one is a clear "HEIC/HEIF support isn't included in this build" error — the same policy - as PhotoCraft. + `cargo test -p lightcraft-codecs --features heif`). The other permissive candidates were + rejected: heif-oxide's HEVC decoder (rust_h265) follows FFmpeg's (LGPL) code structure, and + gamut-heic has no HEVC decoder of its own. +- **HEVC patents are a distributor's call.** `lightcraft-codecs` uses this crate only behind its + non-default `heif` feature; the apps forward it and the release packages enable it. Without it, + HEIC files are still recognised and opening one is a clear "HEIC/HEIF support isn't included in + this build" error — the same policy as PhotoCraft. The crate depends on no other LightCraft crate; `codecs` → `heif` is the only edge between them. diff --git a/crates/heif/src/lib.rs b/crates/heif/src/lib.rs index 7b19b9b94..8292024b1 100644 --- a/crates/heif/src/lib.rs +++ b/crates/heif/src/lib.rs @@ -1,14 +1,17 @@ //! `lightcraft-heif`: the optional HEIF / HEIC (iPhone and Mac photo) decoder. //! -//! A thin wrapper around `heic-rs`, a pure-Rust HEVC still-picture decoder: single pictures and -//! grid-tiled photos, 8- and 10-bit (10-bit decodes to 16-bit), alpha auxiliary images, ICC, -//! EXIF and XMP. Its API is plain data ([`Info`], [`Decoded`], [`Error`]) so the crate knows -//! nothing about the rest of LightCraft; `lightcraft-codecs` adapts it behind its `heif` feature. -//! A port of PhotoCraft's `photocraft-heif` (same team, same licence) so the sibling apps share -//! one decision, one dependency pin and one set of regression fixtures. +//! `heic-rs`, a pure-Rust HEIF container and HEVC still-picture decoder, reads the container and +//! reconstructs the coded pictures (bit-exact against libde265 on iPhone photos); this crate turns +//! them into RGB itself ([`ycc`]) so the colours match libheif: it honours the HEVC VUI's range +//! and matrix ([`vui`]) where heic-rs assumes BT.709 limited range, and upsamples chroma like +//! libheif. Single pictures and grid-tiled photos, 8- to 16-bit, monochrome, alpha auxiliary +//! images, ICC, EXIF and XMP. The API is plain data ([`Info`], [`Decoded`], [`Error`]) so the +//! crate knows nothing about the rest of LightCraft; `lightcraft-codecs` adapts it behind its +//! `heif` feature. Shares heic-rs and its regression fixtures with PhotoCraft's `photocraft-heif`. //! //! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF. //! [`Options::apply_transforms`] applies them; with it off, the pixels come back as coded. +//! Auxiliary images other than alpha (HDR gain maps, depth maps) are ignored. //! //! Never panics: heic-rs is young, so every call into it runs under `catch_unwind` and a panic //! inside it becomes [`Error::Malformed`]. @@ -16,9 +19,20 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod orient; +/// Lossless synthetic HEIF files for tests, here and in the crates above (feature `testdata`). +#[cfg(any(test, feature = "testdata"))] +#[doc(hidden)] +pub mod testdata; +mod vui; +mod ycc; + use std::borrow::Cow; -use heic_rs::PixelLayout; +use heic_rs::context::Context; +use heic_rs::hevc::Frame; +use heic_rs::props::ItemProps; +use heic_rs::props::colr::Range; /// What a HEIF file declares, read from the container alone (no pixel is decoded). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -31,8 +45,8 @@ pub struct Info { pub coded_height: u32, /// The primary image has an alpha auxiliary image. pub has_alpha: bool, - /// More than 8 bits per sample: it decodes to 16-bit. - pub sixteen_bit: bool, + /// Bits per sample as coded (8 for most photos, 10 for HDR-capable ones). + pub bit_depth: u8, } /// Decode settings. @@ -50,17 +64,28 @@ impl Default for Options { } } -/// A decoded image: interleaved RGB or RGBA samples, row-major, no padding. 16-bit samples are -/// native-endian `u16`s. +/// How the file describes its colour: the code points of ITU-T H.273, from the `colr` `nclx` +/// box, else the HEVC VUI, else the standard defaults (2 = unspecified, limited range). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Colour { + pub primaries: u16, + pub transfer: u16, + pub matrix: u16, + pub full_range: bool, +} + +/// A decoded image: interleaved RGB or RGBA, 16 bits per sample whatever the coded depth, +/// row-major, no padding. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Decoded { pub width: u32, pub height: u32, /// Four channels (RGBA) instead of three (RGB). pub has_alpha: bool, - /// Two bytes per sample instead of one. - pub sixteen_bit: bool, - pub data: Vec, + /// Bits per sample as coded. + pub bit_depth: u8, + pub data: Vec, + pub colour: Colour, /// The primary image's ICC profile. pub icc: Option>, /// EXIF as a TIFF structure (without HEIF's offset header). Its Orientation tag only mirrors @@ -74,7 +99,7 @@ pub struct Decoded { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Error { /// A valid file using something this decoder does not handle (image sequences, overlays…). - Unsupported(String), + Unsupported(&'static str), /// The image is larger than [`Options::max_pixels`] or holds an oversized box. Limit(String), /// Broken or truncated data (or a decoder bug, reported the same way). @@ -95,11 +120,15 @@ impl std::error::Error for Error {} const SEQUENCE: &str = "this file holds an image sequence; only HEIF still images can be opened"; -fn err(e: heic_rs::Error) -> Error { +/// heic-rs 0.1.1 reconstructs 4:2:0 and monochrome pictures exactly but fails on the 4:2:2 and +/// 4:4:4 ones some cameras write (Canon and Sony "HIF"); those failures are reported as this. +const CHROMA_422_444: &str = "4:2:2 and 4:4:4 HEIF images (as some Canon and Sony cameras write) can't be decoded yet"; + +pub(crate) fn err(e: heic_rs::Error) -> Error { match e { - heic_rs::Error::Unsupported(what) => Error::Unsupported(what.to_string()), + heic_rs::Error::Unsupported(what) => Error::Unsupported(what), // The primary item is neither an HEVC picture nor a grid of them. - heic_rs::Error::MissingBox("hvcC") => Error::Unsupported("the image is an overlay, an identity derivation or not HEVC-coded".into()), + heic_rs::Error::MissingBox("hvcC") => Error::Unsupported("the image is an overlay, an identity derivation or not HEVC-coded"), heic_rs::Error::PixelLimit { .. } | heic_rs::Error::BoxTooLarge { .. } => Error::Limit(e.to_string()), e => Error::Malformed(e.to_string()), } @@ -113,7 +142,8 @@ fn guarded(f: impl FnOnce() -> Result) -> Result { } /// Reads the container's declared size, depth and alpha without decoding pixels, so callers can -/// check their limits first. +/// check their limits first. Also refuses a file cut short: every coded picture of the primary +/// image must lie within `bytes`. pub fn probe(bytes: &[u8]) -> Result { guarded(|| probe_unguarded(bytes)) } @@ -121,16 +151,26 @@ pub fn probe(bytes: &[u8]) -> Result { fn probe_unguarded(bytes: &[u8]) -> Result { let info = heic_rs::probe(bytes).map_err(|e| match e { // No `meta` box: no still image, only a `moov` image sequence (a video track). - heic_rs::Error::MissingBox("meta") => Error::Unsupported(SEQUENCE.into()), + heic_rs::Error::MissingBox("meta") => Error::Unsupported(SEQUENCE), e => err(e), })?; + let ctx = Context::open(bytes).map_err(err)?; + let id = ctx.meta.primary; + let coded = match ctx.grid(id).map_err(err)? { + Some((_, tiles)) => tiles, + None => vec![id], + }; + for item in coded { + // Borrowed from `bytes` unless split across extents: cheap. + ctx.item_data(item).map_err(err)?; + } Ok(Info { width: info.width, height: info.height, coded_width: info.coded_width, coded_height: info.coded_height, has_alpha: info.has_alpha, - sixteen_bit: info.bit_depth > 8, + bit_depth: info.bit_depth, }) } @@ -139,49 +179,186 @@ pub fn decode(bytes: &[u8], options: &Options) -> Result { guarded(|| decode_unguarded(bytes, options)) } -fn decode_unguarded(bytes: &[u8], options: &Options) -> Result { - let info = probe_unguarded(bytes)?; - let pixel_layout = match (info.has_alpha, info.sixteen_bit) { - (true, true) => PixelLayout::Rgba16, - (true, false) => PixelLayout::Rgba8, - (false, true) => PixelLayout::Rgb16, - (false, false) => PixelLayout::Rgb8, - }; - let raw = heic_rs::DecodeOptions::default() - .with_layout(pixel_layout) - .with_max_pixels(Some(options.max_pixels)) - .with_transforms(options.apply_transforms) - .with_alpha(info.has_alpha); - let decoded = heic_rs::decode(bytes, &raw).map_err(err)?; - let (icc, exif, xmp) = metadata(bytes); - Ok(Decoded { - width: decoded.width, - height: decoded.height, - has_alpha: info.has_alpha, - sixteen_bit: info.sixteen_bit, - data: decoded.data, - icc, - exif, - xmp, +/// Decodes the smallest thumbnail image the file carries whose long edge is at least +/// `min_edge`, if any (iPhones store one of about 320 × 240). Transforms are applied as for the +/// primary image; the metadata (and, when the thumbnail has no profile of its own, the ICC +/// profile) are the primary image's. +pub fn decode_thumbnail(bytes: &[u8], min_edge: u32, options: &Options) -> Result, Error> { + guarded(|| { + let ctx = Context::open(bytes).map_err(err)?; + let primary = ctx.meta.primary; + let mut best: Option<(u32, u32)> = None; + for r in ctx.meta.refs.iter().filter(|r| r.kind == heic_rs::meta::iref::RefKind::Thmb && r.to.contains(&primary)) { + let Ok(p) = ctx.props(r.from) else { continue }; + let Ok((w, h)) = ctx.coded_size(r.from, &p) else { continue }; + let edge = w.max(h); + if edge >= min_edge && best.is_none_or(|(_, e)| edge < e) { + best = Some((r.from, edge)); + } + } + let Some((id, _)) = best else { return Ok(None) }; + let mut decoded = decode_item(&ctx, id, options, false)?; + if decoded.icc.is_none() { + decoded.icc = item_icc(&ctx, primary); + } + (decoded.exif, decoded.xmp) = metadata(&ctx); + Ok(Some(decoded)) }) } -/// The primary image's ICC profile, EXIF (TIFF structure) and XMP packet. Metadata is a -/// courtesy: a broken metadata item never fails a decode whose pixels came out fine. -fn metadata(bytes: &[u8]) -> (Option>, Option>, Option) { - let Ok(ctx) = heic_rs::context::Context::open(bytes) else { - return (None, None, None); - }; +/// An item's ICC profile; a grid whose own properties carry none takes its first tile's (as on +/// some iPhones). +fn item_icc(ctx: &Context<'_>, id: u32) -> Option> { + let p = ctx.props(id).ok()?; + let icc = p.icc.or_else(|| { + let (_, tiles) = ctx.grid(id).ok()??; + ctx.props(*tiles.first()?).ok()?.icc + }); + icc.map(<[u8]>::to_vec) +} + +fn decode_unguarded(bytes: &[u8], options: &Options) -> Result { + // Refuses sequences with a clear message before anything else is read. + probe_unguarded(bytes)?; + let ctx = Context::open(bytes).map_err(err)?; + let mut decoded = decode_item(&ctx, ctx.meta.primary, options, true)?; + let (exif, xmp) = metadata(&ctx); + decoded.exif = exif; + decoded.xmp = xmp; + Ok(decoded) +} + +/// Decodes one image item (a coded picture or a grid of them) to RGB(A), alpha included when +/// `with_alpha` and the item has an alpha auxiliary image. +fn decode_item(ctx: &Context<'_>, id: u32, options: &Options, with_alpha: bool) -> Result { + let p = ctx.props(id).map_err(err)?; + let limit = options.max_pixels; + // The declared sizes are refused before anything is decoded or allocated. + let (cw, ch) = ctx.coded_size(id, &p).map_err(err)?; + heic_rs::image::check_pixels(cw, ch, limit).map_err(err)?; + let (tw, th) = heic_rs::transform::transformed_size(cw, ch, &p.transforms).map_err(err)?; + heic_rs::image::check_pixels(tw, th, limit).map_err(err)?; + + let coded = Coded::decode(ctx, id, &p, limit)?; + let planes = coded.planes()?; + let colour = colour(&p, &coded.first_props); + let signal = ycc::Signal { matrix: colour.matrix, full_range: colour.full_range }; + let alpha_id = if with_alpha { ctx.alpha_item(id).map_err(err)? } else { None }; + let (w, h) = planes.size(); + let mode = if options.apply_transforms && orient::odd_crop(w, h, &p.transforms)? { ycc::Upsampling::Bilinear } else { ycc::Upsampling::Nearest }; + let mut data = ycc::to_rgb16(&planes, signal, alpha_id.is_some(), mode) + .ok_or_else(|| Error::Malformed("the decoded pictures don't cover the image".into()))?; + if let Some(aux) = alpha_id { + let ap = ctx.props(aux).map_err(err)?; + let alpha = Coded::decode(ctx, aux, &ap, limit)?; + let alpha_planes = alpha.planes()?; + if alpha_planes.size() != (w, h) { + return Err(Error::Unsupported("an alpha image of a different size than the picture")); + } + ycc::put_alpha(&alpha_planes, &mut data).ok_or_else(|| Error::Malformed("the alpha image is incomplete".into()))?; + } + let bit_depth = planes.bit_depth(); + let has_alpha = alpha_id.is_some(); + let mut img = orient::Pixels { width: w, height: h, ch: if has_alpha { 4 } else { 3 }, data }; + if options.apply_transforms { + img = orient::apply(img, &p.transforms)?; + } + Ok(Decoded { + width: u32::try_from(img.width).map_err(|_| Error::Limit("image too wide".into()))?, + height: u32::try_from(img.height).map_err(|_| Error::Limit("image too tall".into()))?, + has_alpha, + bit_depth, + data: img.data, + colour, + // A grid whose own properties carry no profile takes its tiles' (as on some iPhones). + icc: p.icc.or(coded.first_props.icc).map(<[u8]>::to_vec), + exif: None, + xmp: None, + }) +} + +/// The decoded pictures of an item: one, or a grid's tiles in `dimg` order. +struct Coded<'a> { + frames: Vec, + grid: Option, + /// The properties of the first coded picture (the item itself, or the first tile): where the + /// HEVC configuration and its VUI live. + first_props: ItemProps<'a>, + /// A failed 4:2:2 / 4:4:4 picture, for the error message. + chroma_format: u8, +} + +impl<'a> Coded<'a> { + fn decode(ctx: &Context<'a>, id: u32, p: &ItemProps<'a>, limit: u64) -> Result, Error> { + let (grid, ids) = match ctx.grid(id).map_err(err)? { + Some((g, tiles)) => (Some(g), tiles), + None => (None, vec![id]), + }; + let first = *ids.first().ok_or_else(|| Error::Malformed("grid lists no tiles".into()))?; + let first_props = if first == id { p.clone() } else { ctx.props(first).map_err(err)? }; + let chroma_format = first_props.hvcc.as_ref().map_or(1, |c| c.chroma_format); + let decode_one = |t: &u32| -> Result { + let tp = ctx.props(*t).map_err(err)?; + let hvcc = tp.hvcc.as_ref().ok_or(heic_rs::Error::MissingBox("hvcC")).map_err(err)?; + let data = ctx.item_data(*t).map_err(err)?; + let nals = hvcc.split_nals(&data).map_err(err)?; + let params = hvcc.parameter_sets(); + if params.is_empty() { + return Err(Error::Malformed("hvcC carries no parameter sets".into())); + } + let frame = heic_rs::hevc::decode_still(¶ms, &nals).map_err(err)?; + frame.validate().map_err(err)?; + heic_rs::image::check_pixels(frame.width, frame.height, limit).map_err(err)?; + Ok(frame) + }; + #[cfg(not(target_arch = "wasm32"))] + let frames: Result, Error> = { + use rayon::prelude::*; + ids.par_iter().map(decode_one).collect() + }; + #[cfg(target_arch = "wasm32")] + let frames: Result, Error> = ids.iter().map(decode_one).collect(); + let frames = frames.map_err(|e| match e { + Error::Malformed(_) if chroma_format >= 2 => Error::Unsupported(CHROMA_422_444), + e => e, + })?; + Ok(Coded { frames, grid, first_props, chroma_format }) + } + + fn planes(&self) -> Result, Error> { + match (&self.grid, self.frames.as_slice()) { + (Some(g), tiles) => heic_rs::grid::Mosaic::new(g, tiles).map(ycc::Planes::Grid).map_err(err), + (None, [frame]) => Ok(ycc::Planes::Picture(frame)), + (None, _) => Err(Error::Malformed(format!("expected one coded picture (chroma format {})", self.chroma_format))), + } + } +} + +/// The colour description in force: `nclx` on the item (or, for a grid without one, on its +/// first tile), else the VUI of its (first) coded picture, else the H.265 defaults. +fn colour(p: &ItemProps<'_>, coded: &ItemProps<'_>) -> Colour { + if let Some(n) = p.nclx.or(coded.nclx) { + return Colour { primaries: n.primaries, transfer: n.transfer, matrix: n.matrix_code, full_range: n.range == Range::Full }; + } + let s = coded.hvcc.as_ref().and_then(|c| vui::signal(&c.parameter_sets())).unwrap_or_default(); + Colour { primaries: s.primaries.into(), transfer: s.transfer.into(), matrix: s.matrix.into(), full_range: s.full_range } +} + +/// The primary image's EXIF (TIFF structure) and XMP packet. Metadata is a courtesy: a broken +/// metadata item never fails a decode whose pixels came out fine. +fn metadata(ctx: &Context<'_>) -> (Option>, Option) { let id = ctx.meta.primary; - let icc = ctx.props(id).ok().and_then(|p| ctx.icc(&p).map(<[u8]>::to_vec)); let exif = ctx.exif(id).ok().flatten().map(<[u8]>::to_vec); let xmp = ctx.xmp_item(id).and_then(|x| ctx.item_data(x).ok()).and_then(|d| match d { Cow::Borrowed(b) => String::from_utf8(b.to_vec()).ok(), Cow::Owned(b) => String::from_utf8(b).ok(), }); - (icc, exif, xmp.map(|x| x.trim_end_matches('\0').to_string())) + (exif, xmp.map(|x| x.trim_end_matches('\0').to_string())) } +#[cfg(test)] +mod tests_synthetic; + #[cfg(test)] mod tests { use super::*; @@ -217,12 +394,13 @@ mod tests { for bytes in [&b""[..], b"\0\0\0\x18ftypheic\0\0\0\0mif1heic", &[0xFF; 64]] { assert!(probe(bytes).is_err()); assert!(decode(bytes, &Options::default()).is_err()); + assert!(decode_thumbnail(bytes, 1, &Options::default()).is_err()); } } #[test] fn errors_display_their_kind() { - assert!(Error::Unsupported("x".into()).to_string().contains("unsupported")); + assert!(Error::Unsupported("x").to_string().contains("unsupported")); assert!(Error::Limit("x".into()).to_string().contains("limit")); assert!(Error::Malformed("x".into()).to_string().contains("malformed")); } diff --git a/crates/heif/src/orient.rs b/crates/heif/src/orient.rs new file mode 100644 index 000000000..487eb7e22 --- /dev/null +++ b/crates/heif/src/orient.rs @@ -0,0 +1,178 @@ +//! The container's transformative properties (`clap` crop, `irot` rotation, `imir` mirror), +//! applied to interleaved 16-bit pixels in the order the file associates them. +//! +//! `imir` follows libheif, which writes and reads `axis = 0` as a flip about the horizontal axis +//! (top and bottom exchanged) and `axis = 1` as a left-right flip. heic-rs 0.1.1 names the coded +//! values the other way round (`Mirror::LeftRight` is `axis = 0`), so they are swapped here. + +use heic_rs::props::Transform; +use heic_rs::props::simple::{Clap, Mirror, Rotation}; + +use crate::Error; + +/// Interleaved samples, `ch` per pixel, row-major. +pub struct Pixels { + pub width: usize, + pub height: usize, + pub ch: usize, + pub data: Vec, +} + +impl Pixels { + fn remap(&self, width: usize, height: usize, src: impl Fn(usize, usize) -> (usize, usize)) -> Pixels { + let ch = self.ch; + let mut data = vec![0u16; width * height * ch]; + for (y, row) in data.chunks_exact_mut(width * ch).enumerate() { + for (x, px) in row.chunks_exact_mut(ch).enumerate() { + let (sx, sy) = src(x, y); + let i = (sy * self.width + sx) * ch; + if let Some(s) = self.data.get(i..i + ch) { + px.copy_from_slice(s); + } + } + } + Pixels { width, height, ch, data } + } +} + +/// The rectangle `(left, top, width, height)` a `clap` selects from a `w` × `h` image. Its centre +/// is `((w - 1) / 2 + horizOff, (h - 1) / 2 + vertOff)`; a corner that falls between two pixels +/// is rounded as libheif does: left down, top up. +pub fn crop_rect(w: usize, h: usize, c: Clap) -> Result<(usize, usize, usize, usize), Error> { + let bad = || Error::Malformed("clap selects an impossible rectangle".into()); + let size = |(n, d): (u32, u32)| (d != 0).then(|| n / d).ok_or_else(bad); + let (cw, ch) = (size(c.width)? as usize, size(c.height)? as usize); + if cw == 0 || ch == 0 || cw > w || ch > h { + return Err(bad()); + } + // Twice the corner, so the half pixels of odd sizes stay exact. + let twice = |full: usize, cropped: usize, (n, d): (i32, u32)| -> Result { + if d == 0 { + return Err(bad()); + } + Ok((full - cropped) as i64 + (2 * i64::from(n)).div_euclid(i64::from(d))) + }; + let left = twice(w, cw, c.horiz_off)?.div_euclid(2); + let top = (twice(h, ch, c.vert_off)? + 1).div_euclid(2); + let left = usize::try_from(left).map_err(|_| bad())?; + let top = usize::try_from(top).map_err(|_| bad())?; + if left + cw > w || top + ch > h { + return Err(bad()); + } + Ok((left, top, cw, ch)) +} + +/// Whether a `clap` among `transforms` (applied to a `w` × `h` image) starts at an odd column +/// or row. libheif crops such images after upsampling their chroma bilinearly instead of with +/// its default nearest neighbour; [`crate::ycc`] follows it so the pixels match. +pub fn odd_crop(mut w: usize, mut h: usize, transforms: &[Transform]) -> Result { + for t in transforms { + match *t { + Transform::Crop(c) => { + let (left, top, cw, ch) = crop_rect(w, h, c)?; + if left % 2 == 1 || top % 2 == 1 { + return Ok(true); + } + (w, h) = (cw, ch); + } + Transform::Rotate(Rotation::Ccw90 | Rotation::Ccw270) => (w, h) = (h, w), + _ => {} + } + } + Ok(false) +} + +pub fn apply(mut img: Pixels, transforms: &[Transform]) -> Result { + for t in transforms { + let (w, h) = (img.width, img.height); + img = match *t { + Transform::Crop(c) => { + let (left, top, cw, ch) = crop_rect(w, h, c)?; + img.remap(cw, ch, |x, y| (x + left, y + top)) + } + // Counter-clockwise: the right column becomes the top row. + Transform::Rotate(Rotation::Ccw90) => img.remap(h, w, |x, y| (w - 1 - y, x)), + Transform::Rotate(Rotation::Ccw180) => img.remap(w, h, |x, y| (w - 1 - x, h - 1 - y)), + Transform::Rotate(Rotation::Ccw270) => img.remap(h, w, |x, y| (y, h - 1 - x)), + Transform::Rotate(Rotation::None) => img, + // `axis = 0` (heic-rs's `LeftRight`): top and bottom exchanged, as libheif reads it. + Transform::Mirror(Mirror::LeftRight) => img.remap(w, h, |x, y| (x, h - 1 - y)), + // `axis = 1`: left and right exchanged. + Transform::Mirror(Mirror::TopBottom) => img.remap(w, h, |x, y| (w - 1 - x, y)), + }; + } + Ok(img) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 3×2, one channel: 1 2 3 / 4 5 6. + fn img() -> Pixels { + Pixels { width: 3, height: 2, ch: 1, data: vec![1, 2, 3, 4, 5, 6] } + } + + fn run(t: Transform) -> (usize, usize, Vec) { + let p = apply(img(), &[t]).unwrap(); + (p.width, p.height, p.data) + } + + #[test] + fn rotations_are_counter_clockwise() { + assert_eq!(run(Transform::Rotate(Rotation::Ccw90)), (2, 3, vec![3, 6, 2, 5, 1, 4])); + assert_eq!(run(Transform::Rotate(Rotation::Ccw180)), (3, 2, vec![6, 5, 4, 3, 2, 1])); + assert_eq!(run(Transform::Rotate(Rotation::Ccw270)), (2, 3, vec![4, 1, 5, 2, 6, 3])); + assert_eq!(run(Transform::Rotate(Rotation::None)), (3, 2, vec![1, 2, 3, 4, 5, 6])); + } + + #[test] + fn mirrors_follow_libheif() { + // axis 0 (heic-rs `LeftRight`) exchanges top and bottom; axis 1 left and right. + assert_eq!(run(Transform::Mirror(Mirror::LeftRight)), (3, 2, vec![4, 5, 6, 1, 2, 3])); + assert_eq!(run(Transform::Mirror(Mirror::TopBottom)), (3, 2, vec![3, 2, 1, 6, 5, 4])); + } + + fn clap(w: u32, h: u32, ho: i32, vo: i32) -> Clap { + Clap { width: (w, 1), height: (h, 1), horiz_off: (ho, 1), vert_off: (vo, 1) } + } + + #[test] + fn clap_corners_round_like_libheif() { + // Measured against libheif 1.23 on a 512 × 512 picture. + assert_eq!(crop_rect(512, 512, clap(301, 201, 0, 0)).unwrap(), (105, 156, 301, 201)); + assert_eq!(crop_rect(512, 512, clap(300, 200, 0, 0)).unwrap(), (106, 156, 300, 200)); + assert_eq!(crop_rect(512, 512, clap(301, 201, 37, -21)).unwrap(), (142, 135, 301, 201)); + assert_eq!(crop_rect(512, 512, clap(301, 201, -3, 5)).unwrap(), (102, 161, 301, 201)); + assert_eq!(crop_rect(512, 512, clap(300, 201, 0, 0)).unwrap(), (106, 156, 300, 201)); + assert_eq!(crop_rect(1500, 1000, clap(1001, 701, 37, -21)).unwrap(), (286, 129, 1001, 701)); + let p = apply(img(), &[Transform::Crop(clap(2, 1, 1, 0))]).unwrap(); + // The half-pixel top rounds down a row. + assert_eq!((p.width, p.height, p.data), (2, 1, vec![5, 6])); + } + + #[test] + fn impossible_claps_are_errors() { + for c in [ + clap(0, 1, 0, 0), + clap(4, 1, 0, 0), + clap(3, 2, 1, 0), + clap(3, 2, 0, -1), + Clap { width: (1, 0), ..clap(1, 1, 0, 0) }, + Clap { horiz_off: (0, 0), ..clap(1, 1, 0, 0) }, + ] { + assert!(crop_rect(3, 2, c).is_err(), "{c:?}"); + } + assert!(apply(img(), &[Transform::Crop(clap(9, 9, 0, 0))]).is_err()); + } + + #[test] + fn odd_crops_are_detected_through_rotations() { + assert!(!odd_crop(512, 512, &[Transform::Crop(clap(300, 200, 0, 0))]).unwrap()); + assert!(odd_crop(512, 512, &[Transform::Crop(clap(301, 200, 0, 0))]).unwrap()); + // After a quarter turn the crop applies to a 200 × 300 image. + let t = [Transform::Crop(clap(300, 200, 0, 0)), Transform::Rotate(Rotation::Ccw90), Transform::Crop(clap(197, 300, 0, 0))]; + assert!(odd_crop(512, 512, &t).unwrap()); + assert!(!odd_crop(512, 512, &[Transform::Rotate(Rotation::Ccw90)]).unwrap()); + } +} diff --git a/crates/heif/src/testdata.rs b/crates/heif/src/testdata.rs new file mode 100644 index 000000000..595cfa33c --- /dev/null +++ b/crates/heif/src/testdata.rs @@ -0,0 +1,542 @@ +//! Synthetic HEIF files for tests (feature `testdata`): lossless, so every decoded sample is known. +//! +//! Each coded picture is one 32 × 32 coding tree unit stored as PCM samples (H.265 7.3.8.7): +//! the bitstream carries the YCbCr values verbatim, so the decoder's HEVC stage reproduces them +//! exactly and a test checks everything after it (grid composition, colour, transforms, +//! metadata) against values it chose. The few CABAC bins around the PCM data are written with a +//! minimal arithmetic encoder (9.3.4.4). Larger images are grids of such pictures, which is how +//! iPhones store photos too (with 512 × 512 tiles). +//! +//! Not a general HEIF or HEVC writer: 4:2:0 or monochrome, 8 or 10 bits, a CTU per picture. + +/// Side of every coded picture. +pub const TILE: u32 = 32; + +/// YCbCr samples at a luma position (chroma is taken at even positions, 4:2:0). +pub type Sampler<'a> = &'a dyn Fn(u32, u32) -> [u16; 3]; + +/// A property of the primary image. +#[derive(Debug, Clone, PartialEq)] +pub enum Prop { + /// `colr` of type `prof`. + Icc(Vec), + /// `colr` of type `nclx`: primaries, transfer, matrix, full range. + Nclx(u16, u16, u16, bool), + /// `irot`: quarter turns counter-clockwise. + Irot(u8), + /// `imir` with this axis. + Imir(u8), + /// `clap`: width, height, horizontal and vertical offset (whole pixels). + Clap(u32, u32, i32, i32), +} + +/// What [`build`] writes. +pub struct Spec<'a> { + /// Displayed (grid output) size, before transforms. + pub width: u32, + pub height: u32, + pub bit_depth: u8, + pub sample: Sampler<'a>, + /// Properties of the primary image, in association order. + pub props: Vec, + /// EXIF as a TIFF structure (written with HEIF's 4-byte offset header). + pub exif: Option>, + /// An alpha auxiliary image (monochrome, full range). + pub alpha: Option<&'a dyn Fn(u32, u32) -> u16>, + /// A thumbnail item of half the size with these samples (in its own coordinates). + pub thumbnail: Option>, + /// Auxiliary images of these types the decoder must ignore (HDR gain map, depth…). + pub ignored_aux: Vec<&'static str>, +} + +impl<'a> Spec<'a> { + pub fn new(width: u32, height: u32, sample: Sampler<'a>) -> Spec<'a> { + Spec { width, height, bit_depth: 8, sample, props: Vec::new(), exif: None, alpha: None, thumbnail: None, ignored_aux: Vec::new() } + } +} + +#[derive(Default)] +struct Bits { + bytes: Vec, + n: usize, +} + +impl Bits { + fn put(&mut self, v: u32, len: u32) { + for i in (0..len).rev() { + if self.n.is_multiple_of(8) { + self.bytes.push(0); + } + if (v >> i) & 1 == 1 + && let Some(b) = self.bytes.last_mut() + { + *b |= 0x80 >> (self.n % 8); + } + self.n += 1; + } + } + + fn ue(&mut self, v: u32) { + let x = u64::from(v) + 1; + let len = 64 - x.leading_zeros(); + self.put(0, len - 1); + self.put((x & 0xffff_ffff) as u32, len); + } + + fn align_zero(&mut self) { + while !self.n.is_multiple_of(8) { + self.put(0, 1); + } + } + + fn trailing(&mut self) { + self.put(1, 1); + self.align_zero(); + } +} + +/// The CABAC arithmetic encoder of H.265 9.3.4.4, for the bins these pictures need. +struct Cabac { + low: u32, + range: u32, + first: bool, + outstanding: u32, +} + +impl Cabac { + fn new() -> Cabac { + Cabac { low: 0, range: 510, first: true, outstanding: 0 } + } + + fn put_bit(&mut self, out: &mut Bits, b: u32) { + if self.first { + self.first = false; + } else { + out.put(b, 1); + } + while self.outstanding > 0 { + out.put(1 - b, 1); + self.outstanding -= 1; + } + } + + fn renorm(&mut self, out: &mut Bits) { + while self.range < 256 { + if self.low < 256 { + self.put_bit(out, 0); + } else if self.low >= 512 { + self.low -= 512; + self.put_bit(out, 1); + } else { + self.low -= 256; + self.outstanding += 1; + } + self.range <<= 1; + self.low <<= 1; + } + } + + /// `part_mode` = PART_2Nx2N: the most probable symbol of a fresh context (initValue 184 at + /// QP 26 gives pStateIdx 0, valMps 1), so the LPS range is `rangeTabLps[0][3]` = 240. + fn part_mode_2nx2n(&mut self, out: &mut Bits) { + let lps = [128, 176, 208, 240][((self.range >> 6) & 3) as usize]; + self.range -= lps; + self.renorm(out); + } + + /// A terminating bin equal to 1 (`pcm_flag`, `end_of_slice_segment_flag`), with the flush. + fn terminate_one(&mut self, out: &mut Bits) { + self.range -= 2; + self.low += self.range; + self.range = 2; + self.renorm(out); + self.put_bit(out, (self.low >> 9) & 1); + out.put(((self.low >> 7) & 3) | 1, 2); + } +} + +fn nal(kind: u8, rbsp: &[u8]) -> Vec { + let mut out = vec![kind << 1, 1]; + let mut zeros = 0; + for &b in rbsp { + if zeros >= 2 && b <= 3 { + out.push(3); + zeros = 0; + } + zeros = if b == 0 { zeros + 1 } else { 0 }; + out.push(b); + } + out +} + +fn profile_idc(bit_depth: u8, mono: bool) -> u32 { + match (bit_depth, mono) { + (_, true) => 4, // range extensions + (8, false) => 1, + _ => 2, + } +} + +fn ptl(w: &mut Bits, idc: u32) { + w.put(0, 2); + w.put(0, 1); + w.put(idc, 5); + w.put(1 << (31 - idc), 32); // compatible with itself + w.put(0b1001, 4); // progressive, frame only + w.put(0, 22); + w.put(0, 21); + w.put(0, 1); + w.put(93, 8); // level 3.1 +} + +fn parameter_sets(bit_depth: u8, mono: bool) -> [Vec; 3] { + let idc = profile_idc(bit_depth, mono); + let bd = u32::from(bit_depth); + let mut v = Bits::default(); + v.put(0, 4); + v.put(3, 2); + v.put(0, 6); + v.put(0, 3); + v.put(1, 1); + v.put(0xffff, 16); + ptl(&mut v, idc); + v.put(0, 1); // vps_sub_layer_ordering_info_present_flag + v.ue(0); + v.ue(0); + v.ue(0); + v.put(0, 6); // vps_max_layer_id + v.ue(0); // vps_num_layer_sets_minus1 + v.put(0, 1); // vps_timing_info_present_flag + v.put(0, 1); // vps_extension_flag + v.trailing(); + + let mut s = Bits::default(); + s.put(0, 4); + s.put(0, 3); + s.put(1, 1); + ptl(&mut s, idc); + s.ue(0); + s.ue(if mono { 0 } else { 1 }); + s.ue(TILE); + s.ue(TILE); + s.put(0, 1); // conformance_window_flag + s.ue(bd - 8); + s.ue(bd - 8); + s.ue(4); + s.put(1, 1); + s.ue(0); + s.ue(0); + s.ue(0); + s.ue(2); // log2_min_luma_coding_block_size_minus3: 32 + s.ue(0); // log2_diff_max_min_luma_coding_block_size: CTB 32 + s.ue(0); // log2_min_luma_transform_block_size_minus2: 4 + s.ue(3); // log2_diff_max_min_luma_transform_block_size: 32 + s.ue(0); + s.ue(0); + s.put(0, 1); // scaling_list_enabled_flag + s.put(0, 1); // amp_enabled_flag + s.put(0, 1); // sample_adaptive_offset_enabled_flag + s.put(1, 1); // pcm_enabled_flag + s.put(bd - 1, 4); // pcm_sample_bit_depth_luma_minus1 + s.put(bd - 1, 4); // pcm_sample_bit_depth_chroma_minus1 + s.ue(2); // log2_min_pcm_luma_coding_block_size_minus3: 32 + s.ue(0); // log2_diff_max_min_pcm_luma_coding_block_size + s.put(1, 1); // pcm_loop_filter_disabled_flag + s.ue(0); // num_short_term_ref_pic_sets + s.put(0, 1); // long_term_ref_pics_present_flag + s.put(0, 1); // sps_temporal_mvp_enabled_flag + s.put(0, 1); // strong_intra_smoothing_enabled_flag + s.put(0, 1); // vui_parameters_present_flag + s.put(0, 1); // sps_extension_present_flag + s.trailing(); + + let mut p = Bits::default(); + p.ue(0); + p.ue(0); + p.put(0, 6); // dependent slices, output flag, 3 extra header bits, sign hiding + p.put(0, 1); // cabac_init_present_flag + p.ue(0); + p.ue(0); + p.ue(0); // init_qp_minus26 (se(0) is the same bits) + p.put(0, 3); // constrained intra, transform skip, cu_qp_delta + p.ue(0); + p.ue(0); + p.put(0, 1); + p.put(0, 2); + p.put(0, 1); // transquant_bypass_enabled_flag + p.put(0, 1); // tiles_enabled_flag + p.put(0, 1); // entropy_coding_sync_enabled_flag + p.put(0, 1); // pps_loop_filter_across_slices_enabled_flag + p.put(1, 1); // deblocking_filter_control_present_flag + p.put(0, 1); // deblocking_filter_override_enabled_flag + p.put(1, 1); // pps_deblocking_filter_disabled_flag + p.put(0, 1); + p.put(0, 1); + p.ue(0); + p.put(0, 1); + p.put(0, 1); + p.trailing(); + [nal(32, &v.bytes), nal(33, &s.bytes), nal(34, &p.bytes)] +} + +/// One coded 32 × 32 picture: the `hvcC` payload and the item data (length-prefixed NAL). +struct Coded { + hvcc: Vec, + data: Vec, +} + +fn picture(bit_depth: u8, mono: bool, sample: &dyn Fn(u32, u32) -> [u16; 3]) -> Coded { + let bd = u32::from(bit_depth); + let mut w = Bits::default(); + w.put(1, 1); // first_slice_segment_in_pic_flag + w.put(0, 1); // no_output_of_prior_pics_flag + w.ue(0); + w.ue(2); // I slice + w.ue(0); // slice_qp_delta, se(0) + w.trailing(); // byte_alignment() + let mut c = Cabac::new(); + c.part_mode_2nx2n(&mut w); + c.terminate_one(&mut w); // pcm_flag + w.align_zero(); // pcm_alignment_zero_bit + for y in 0..TILE { + for x in 0..TILE { + w.put(u32::from(sample(x, y)[0]), bd); + } + } + if !mono { + for k in 1..3 { + for y in 0..TILE / 2 { + for x in 0..TILE / 2 { + w.put(u32::from(sample(2 * x, 2 * y)[k]), bd); + } + } + } + } + let mut c = Cabac::new(); + c.terminate_one(&mut w); // end_of_slice_segment_flag (its flush ends in the stop bit) + w.align_zero(); + let slice = nal(19, &w.bytes); + + let sets = parameter_sets(bit_depth, mono); + let mut h = vec![1u8, profile_idc(bit_depth, mono) as u8]; + h.extend_from_slice(&(1u32 << (31 - profile_idc(bit_depth, mono))).to_be_bytes()); + h.extend_from_slice(&[ + 0x90, + 0, + 0, + 0, + 0, + 0, + 93, + 0xf0, + 0, + 0xfc, + 0xfc | u8::from(!mono), + 0xf8 | (bit_depth - 8), + 0xf8 | (bit_depth - 8), + 0, + 0, + 0x0f, + 3, + ]); + for (kind, set) in [32u8, 33, 34].into_iter().zip(&sets) { + h.push(0x80 | kind); + h.extend_from_slice(&1u16.to_be_bytes()); + h.extend_from_slice(&(set.len() as u16).to_be_bytes()); + h.extend_from_slice(set); + } + let mut data = (slice.len() as u32).to_be_bytes().to_vec(); + data.extend_from_slice(&slice); + Coded { hvcc: h, data } +} + +fn bx(kind: &[u8; 4], payload: &[u8]) -> Vec { + let mut v = ((payload.len() + 8) as u32).to_be_bytes().to_vec(); + v.extend_from_slice(kind); + v.extend_from_slice(payload); + v +} + +fn full_box(kind: &[u8; 4], version: u8, payload: &[u8]) -> Vec { + let mut p = vec![version, 0, 0, 0]; + p.extend_from_slice(payload); + bx(kind, &p) +} + +struct Item { + id: u16, + kind: &'static [u8; 4], + data: Vec, + /// Indices (1-based) into the property list, with the essential flag. + props: Vec<(u8, bool)>, + /// Not an image of its own (a grid tile). + hidden: bool, +} + +#[derive(Default)] +struct File { + items: Vec, + props: Vec>, + refs: Vec<(&'static [u8; 4], u16, Vec)>, +} + +impl File { + fn prop(&mut self, b: Vec) -> u8 { + self.props.push(b); + self.props.len() as u8 + } + + fn add(&mut self, kind: &'static [u8; 4], data: Vec, props: Vec<(u8, bool)>) -> u16 { + let id = self.items.len() as u16 + 1; + self.items.push(Item { id, kind, data, props, hidden: false }); + id + } + + /// A coded image of `w` × `h` (a grid when larger than a tile); returns its item id. + fn image(&mut self, w: u32, h: u32, bit_depth: u8, mono: bool, sample: &dyn Fn(u32, u32) -> [u16; 3]) -> u16 { + let (cols, rows) = (w.div_ceil(TILE), h.div_ceil(TILE)); + let ispe = |w: u32, h: u32| { + let mut p = w.to_be_bytes().to_vec(); + p.extend_from_slice(&h.to_be_bytes()); + full_box(b"ispe", 0, &p) + }; + let mut hvcc_prop = None; + let tile_ispe = self.prop(ispe(TILE, TILE)); + let mut tiles = Vec::new(); + for r in 0..rows { + for c in 0..cols { + let coded = picture(bit_depth, mono, &|x, y| sample((c * TILE + x).min(w - 1), (r * TILE + y).min(h - 1))); + let hv = *hvcc_prop.get_or_insert_with(|| self.prop(bx(b"hvcC", &coded.hvcc))); + tiles.push(self.add(b"hvc1", coded.data, vec![(hv, true), (tile_ispe, false)])); + } + } + if let ([only], true) = (tiles.as_slice(), w == TILE && h == TILE) { + return *only; + } + for it in self.items.iter_mut().filter(|i| tiles.contains(&i.id)) { + it.hidden = true; + } + let mut g = vec![0u8, 0, (rows - 1) as u8, (cols - 1) as u8]; + g.extend_from_slice(&(w as u16).to_be_bytes()); + g.extend_from_slice(&(h as u16).to_be_bytes()); + let grid_ispe = self.prop(ispe(w, h)); + let id = self.add(b"grid", g, vec![(grid_ispe, false)]); + self.refs.push((b"dimg", id, tiles)); + id + } + + fn write(self, primary: u16) -> Vec { + let ftyp = bx(b"ftyp", b"heic\0\0\0\0mif1heic"); + let mut hdlr = vec![0u8; 4]; + hdlr.extend_from_slice(b"pict"); + hdlr.extend_from_slice(&[0; 13]); + let hdlr = full_box(b"hdlr", 0, &hdlr); + let pitm = full_box(b"pitm", 0, &primary.to_be_bytes()); + let mut iinf = (self.items.len() as u16).to_be_bytes().to_vec(); + for it in &self.items { + let mut e = vec![2, 0, 0, u8::from(it.hidden)]; // version 2, flags + e.extend_from_slice(&it.id.to_be_bytes()); + e.extend_from_slice(&[0, 0]); + e.extend_from_slice(it.kind); + e.push(0); + iinf.extend_from_slice(&bx(b"infe", &e)); + } + let iinf = full_box(b"iinf", 0, &iinf); + let mut iref = Vec::new(); + for (kind, from, to) in &self.refs { + let mut r = from.to_be_bytes().to_vec(); + r.extend_from_slice(&(to.len() as u16).to_be_bytes()); + for t in to { + r.extend_from_slice(&t.to_be_bytes()); + } + iref.extend_from_slice(&bx(kind, &r)); + } + let iref = full_box(b"iref", 0, &iref); + let ipco = bx(b"ipco", &self.props.concat()); + let mut ipma = (self.items.iter().filter(|i| !i.props.is_empty()).count() as u32).to_be_bytes().to_vec(); + for it in self.items.iter().filter(|i| !i.props.is_empty()) { + ipma.extend_from_slice(&it.id.to_be_bytes()); + ipma.push(it.props.len() as u8); + ipma.extend(it.props.iter().map(|&(i, essential)| i | if essential { 0x80 } else { 0 })); + } + let iprp = bx(b"iprp", &[ipco, full_box(b"ipma", 0, &ipma)].concat()); + let iloc = |base: u32| { + let mut l = vec![0x44, 0x00]; + l.extend_from_slice(&(self.items.len() as u16).to_be_bytes()); + let mut offset = base; + for it in &self.items { + l.extend_from_slice(&it.id.to_be_bytes()); + l.extend_from_slice(&[0, 0, 0, 0, 0, 1]); // construction method 0, data ref 0, one extent + l.extend_from_slice(&offset.to_be_bytes()); + l.extend_from_slice(&(it.data.len() as u32).to_be_bytes()); + offset += it.data.len() as u32; + } + full_box(b"iloc", 1, &l) + }; + let meta = |base| full_box(b"meta", 0, &[hdlr.clone(), pitm.clone(), iloc(base), iinf.clone(), iref.clone(), iprp.clone()].concat()); + let start = (ftyp.len() + meta(0).len() + 8) as u32; + let mdat = bx(b"mdat", &self.items.iter().flat_map(|i| i.data.iter().copied()).collect::>()); + [ftyp, meta(start), mdat].concat() + } +} + +/// Writes the HEIF file `spec` describes. +pub fn build(spec: &Spec<'_>) -> Vec { + let mut f = File::default(); + let primary = f.image(spec.width, spec.height, spec.bit_depth, false, spec.sample); + let mut assoc = Vec::new(); + for p in &spec.props { + let b = match p { + Prop::Icc(icc) => bx(b"colr", &[b"prof".as_slice(), icc].concat()), + Prop::Nclx(pr, tc, mc, full) => { + let mut c = b"nclx".to_vec(); + for v in [pr, tc, mc] { + c.extend_from_slice(&v.to_be_bytes()); + } + c.push(if *full { 0x80 } else { 0 }); + bx(b"colr", &c) + } + Prop::Irot(a) => bx(b"irot", &[a & 3]), + Prop::Imir(a) => bx(b"imir", &[a & 1]), + Prop::Clap(w, h, ho, vo) => { + let mut c = Vec::new(); + for v in [*w as i32, 1, *h as i32, 1, *ho, 1, *vo, 1] { + c.extend_from_slice(&v.to_be_bytes()); + } + bx(b"clap", &c) + } + }; + assoc.push((f.prop(b), true, matches!(p, Prop::Clap(..)))); + } + if let Some(it) = f.items.iter_mut().find(|i| i.id == primary) { + it.props.extend(assoc.iter().map(|&(i, e, _)| (i, e))); + } + if let Some(exif) = &spec.exif { + let id = f.add(b"Exif", [&[0u8, 0, 0, 0][..], exif].concat(), Vec::new()); + f.refs.push((b"cdsc", id, vec![primary])); + } + let bd = spec.bit_depth; + let aux = |f: &mut File, urn: &str, sample: &dyn Fn(u32, u32) -> [u16; 3]| { + let id = f.image(spec.width, spec.height, bd, true, sample); + let auxc = f.prop(full_box(b"auxC", 0, &[urn.as_bytes(), &[0]].concat())); + if let Some(it) = f.items.iter_mut().find(|i| i.id == id) { + it.props.push((auxc, true)); + } + f.refs.push((b"auxl", id, vec![primary])); + }; + if let Some(alpha) = spec.alpha { + aux(&mut f, "urn:mpeg:hevc:2015:auxid:1", &|x, y| [alpha(x, y), 0, 0]); + } + for urn in &spec.ignored_aux { + aux(&mut f, urn, &|x, y| [((x + y) % 256) as u16, 0, 0]); + } + if let Some(sample) = spec.thumbnail { + // Half the size, with the primary image's colour and orientation (not its crop). + let id = f.image(spec.width / 2, spec.height / 2, spec.bit_depth, false, sample); + if let Some(it) = f.items.iter_mut().find(|i| i.id == id) { + it.props.extend(assoc.iter().filter(|a| !a.2).map(|&(i, e, _)| (i, e))); + } + f.refs.push((b"thmb", id, vec![primary])); + } + f.write(primary) +} diff --git a/crates/heif/src/tests_synthetic.rs b/crates/heif/src/tests_synthetic.rs new file mode 100644 index 000000000..e229af926 --- /dev/null +++ b/crates/heif/src/tests_synthetic.rs @@ -0,0 +1,176 @@ +//! Decoding lossless synthetic HEIF files ([`crate::testdata`]): every expected sample is known. + +use crate::testdata::{Prop, Spec, build}; +use crate::{Error, Options, decode, decode_thumbnail, probe}; + +/// A luma value unique enough to identify its position; chroma neutral. +fn luma(x: u32, y: u32) -> u16 { + ((x * 7 + y * 13) % 220 + 20) as u16 +} + +fn grey(x: u32, y: u32) -> [u16; 3] { + [luma(x, y), 128, 128] +} + +const FULL_601: Prop = Prop::Nclx(1, 13, 6, true); + +/// Decoded red channel at `(x, y)`, back in 8-bit code values. +fn at(d: &crate::Decoded, x: u32, y: u32) -> u16 { + let ch = if d.has_alpha { 4 } else { 3 }; + d.data[((y * d.width + x) as usize) * ch] / 257 +} + +fn decoded(spec: &Spec) -> crate::Decoded { + decode(&build(spec), &Options::default()).unwrap() +} + +#[test] +fn a_grid_reassembles_exactly() { + // 4 × 3 tiles of 32, cropped by the grid to 100 × 70. + let d = decoded(&Spec { props: vec![FULL_601], ..Spec::new(100, 70, &grey) }); + assert_eq!((d.width, d.height, d.bit_depth, d.has_alpha), (100, 70, 8, false)); + for y in 0..70 { + for x in 0..100 { + let px = &d.data[((y * 100 + x) * 3) as usize..][..3]; + assert_eq!(px, [luma(x, y) * 257; 3], "({x}, {y})"); + } + } + let info = probe(&build(&Spec::new(100, 70, &grey))).unwrap(); + assert_eq!((info.width, info.height, info.coded_width, info.coded_height, info.bit_depth), (100, 70, 100, 70, 8)); +} + +#[test] +fn colour_follows_the_matrix_and_range() { + // BT.601 full range, a saturated orange: within one code value of the exact conversion. + let (y, cb, cr) = (150.0f32, 80.0, 190.0); + let s = |_: u32, _: u32| [y as u16, cb as u16, cr as u16]; + let d = decoded(&Spec { props: vec![FULL_601], ..Spec::new(32, 32, &s) }); + let (u, v) = (cb - 128.0, cr - 128.0); + let expected = [y + 1.402 * v, y - 0.344_136 * u - 0.714_136 * v, y + 1.772 * u].map(|c| c.clamp(0.0, 255.0)); + for (got, want) in d.data[..3].iter().zip(expected) { + assert!((f32::from(*got) / 257.0 - want).abs() <= 1.0, "{:?} vs {expected:?}", &d.data[..3]); + } + assert_eq!(d.colour, crate::Colour { primaries: 1, transfer: 13, matrix: 6, full_range: true }); + + // No nclx and no VUI: the H.265 defaults, limited range (16 → black, 235 → white). + let ends = |x: u32, _: u32| [if x < 16 { 16 } else { 235 }, 128, 128]; + let d = decoded(&Spec::new(32, 32, &ends)); + assert!(!d.colour.full_range); + assert_eq!((d.data[0], d.data[20 * 3]), (0, 65535)); +} + +#[test] +fn ten_bit_samples_keep_their_precision() { + let s = |x: u32, y: u32| [(x * 31 + y) as u16 % 1024, 512, 512]; + let d = decoded(&Spec { bit_depth: 10, props: vec![FULL_601], ..Spec::new(64, 32, &s) }); + assert_eq!(d.bit_depth, 10); + for x in [0, 1, 5, 33, 63] { + let want = ((x * 31) % 1024) as f64 / 1023.0 * 65535.0; + assert!((f64::from(d.data[(x * 3) as usize]) - want).abs() <= 1.0, "x {x}"); + } +} + +#[test] +fn container_transforms_are_applied_like_libheif() { + let (w, h) = (70u32, 40u32); + let spec = |props: Vec| Spec { props: [vec![FULL_601], props].concat(), ..Spec::new(w, h, &grey) }; + // irot: counter-clockwise quarter turns. + let d = decoded(&spec(vec![Prop::Irot(1)])); + assert_eq!((d.width, d.height), (h, w)); + assert_eq!(at(&d, 0, 0), luma(w - 1, 0)); // the right column becomes the top row + assert_eq!(at(&d, h - 1, w - 1), luma(0, h - 1)); + let d = decoded(&spec(vec![Prop::Irot(2)])); + assert_eq!(at(&d, 0, 0), luma(w - 1, h - 1)); + let d = decoded(&spec(vec![Prop::Irot(3)])); + assert_eq!(at(&d, 0, 0), luma(0, h - 1)); + // imir: axis 0 exchanges top and bottom, axis 1 left and right (libheif's reading). + let d = decoded(&spec(vec![Prop::Imir(0)])); + assert_eq!((at(&d, 0, 0), at(&d, 5, 0)), (luma(0, h - 1), luma(5, h - 1))); + let d = decoded(&spec(vec![Prop::Imir(1)])); + assert_eq!((at(&d, 0, 0), at(&d, 0, 5)), (luma(w - 1, 0), luma(w - 1, 5))); + // clap: centred crop moved by its offsets; a half-pixel corner goes left and down. + let d = decoded(&spec(vec![Prop::Clap(31, 21, 3, -2)])); + assert_eq!((d.width, d.height), (31, 21)); + // left = (70 - 31 + 6) / 2 = 22.5 → 22, top = (40 - 21 - 4) / 2 = 7.5 → 8 + assert_eq!(at(&d, 0, 0), luma(22, 8)); + // In association order: crop, then turn, then mirror. + let d = decoded(&spec(vec![Prop::Clap(30, 20, 0, 0), Prop::Irot(1), Prop::Imir(1)])); + assert_eq!((d.width, d.height), (20, 30)); + // The crop is (20, 10) to (49, 29); the turn brings its bottom-right corner to the top-right, + // the left-right mirror to the top-left. + assert_eq!(at(&d, 0, 0), luma(49, 29)); + let info = probe(&build(&spec(vec![Prop::Irot(1)]))).unwrap(); + assert_eq!((info.width, info.height, info.coded_width, info.coded_height), (h, w, w, h)); + // Transforms off: as coded. + let d = decode(&build(&spec(vec![Prop::Irot(1)])), &Options { apply_transforms: false, ..Default::default() }).unwrap(); + assert_eq!((d.width, d.height, at(&d, 3, 2)), (w, h, luma(3, 2))); +} + +#[test] +fn metadata_aux_images_and_thumbnails() { + let icc = b"not parsed here: carried verbatim".to_vec(); + let exif = b"MM\0\x2a\0\0\0\x08\0\0".to_vec(); + let spec = Spec { + props: vec![Prop::Icc(icc.clone()), FULL_601], + exif: Some(exif.clone()), + thumbnail: Some(&|_, _| [77, 128, 128]), + ignored_aux: vec!["urn:com:apple:photo:2020:aux:hdrgainmap", "urn:mpeg:hevc:2015:auxid:2"], + ..Spec::new(64, 64, &grey) + }; + let bytes = build(&spec); + let d = decode(&bytes, &Options::default()).unwrap(); + assert_eq!((d.width, d.height, d.has_alpha), (64, 64, false)); + assert_eq!(d.icc.as_deref(), Some(icc.as_slice())); + assert_eq!(d.exif.as_deref(), Some(exif.as_slice())); + let t = decode_thumbnail(&bytes, 32, &Options::default()).unwrap().unwrap(); + assert_eq!((t.width, t.height, t.data[0]), (32, 32, 77 * 257)); + assert_eq!(t.exif.as_deref(), Some(exif.as_slice())); + assert_eq!(t.icc.as_deref(), Some(icc.as_slice())); + assert!(decode_thumbnail(&bytes, 33, &Options::default()).unwrap().is_none()); +} + +#[test] +fn alpha_comes_from_its_auxiliary_image() { + let alpha = |x: u32, _: u32| (x * 4) as u16; + let d = decoded(&Spec { props: vec![FULL_601], alpha: Some(&alpha), ..Spec::new(64, 32, &grey) }); + assert!(d.has_alpha); + assert_eq!(d.data.len(), 64 * 32 * 4); + assert_eq!((d.data[3], d.data[10 * 4 + 3]), (0, 40 * 257)); + assert_eq!(d.data[..3], [luma(0, 0) * 257; 3]); +} + +#[test] +fn limits_are_checked_before_decoding() { + let bytes = build(&Spec::new(96, 64, &grey)); + let r = decode(&bytes, &Options { max_pixels: 96 * 64 - 1, ..Default::default() }); + assert!(matches!(r, Err(Error::Limit(_))), "{r:?}"); +} + +#[test] +fn truncated_and_corrupted_files_are_errors_not_panics() { + let bytes = build(&Spec { props: vec![FULL_601, Prop::Irot(1)], thumbnail: Some(&grey), ..Spec::new(64, 64, &grey) }); + let full = decode(&bytes, &Options::default()).unwrap(); + for cut in (0..bytes.len()).step_by(7) { + let b = &bytes[..cut]; + // Cut inside the photo: an error. Cut in the trailing thumbnail: the whole photo. + if let (Ok(_), Ok(d)) = (probe(b), decode(b, &Options::default())) { + assert_eq!(d.data, full.data, "cut at {cut} of {}", bytes.len()); + } + let _ = decode_thumbnail(b, 1, &Options::default()); + } + // Flip bits all over the file: any outcome but a panic. + let mut state = 0x2545_f491_u32; + for _ in 0..400 { + let mut b = bytes.clone(); + for _ in 0..3 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let i = state as usize % b.len(); + b[i] ^= 1 << (state % 8); + } + let _ = probe(&b); + let _ = decode(&b, &Options::default()); + let _ = decode_thumbnail(&b, 1, &Options::default()); + } +} diff --git a/crates/heif/src/vui.rs b/crates/heif/src/vui.rs new file mode 100644 index 000000000..d732f2863 --- /dev/null +++ b/crates/heif/src/vui.rs @@ -0,0 +1,427 @@ +//! The colour description in an HEVC sequence parameter set's VUI (Rec. ITU-T H.265, 7.3.2.2 and +//! Annex E): the fallback a HEIF reader uses when the item carries no `colr` box of type `nclx`. +//! iPhone photos are the common case: they label their colour with an ICC profile only and say in +//! the VUI that their YCbCr is full range. +//! +//! Only the syntax up to `matrix_coeffs` is read. Anything this parser can't follow returns `None` +//! and the caller falls back to the H.265 defaults; it never fails a decode. + +/// What the VUI's `video_signal_type` says (H.265 E.3.1). Absent fields keep the defaults the +/// standard infers: limited range, every code point 2 ("unspecified"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Signal { + pub full_range: bool, + pub primaries: u8, + pub transfer: u8, + pub matrix: u8, +} + +impl Default for Signal { + fn default() -> Self { + Signal { full_range: false, primaries: 2, transfer: 2, matrix: 2 } + } +} + +/// NAL unit type of a sequence parameter set. +const SPS_NUT: u8 = 33; + +/// Reads the signal description from the SPS among `parameter_sets` (NAL units with their +/// two-byte header, as an `hvcC` record stores them). `None`: no SPS, or one this reader can't +/// follow; `Some(Signal::default())`: an SPS without VUI colour fields. +pub fn signal(parameter_sets: &[&[u8]]) -> Option { + let sps = parameter_sets.iter().find(|n| n.first().is_some_and(|b| (b >> 1) & 0x3f == SPS_NUT))?; + parse_sps(&unescape(sps.get(2..)?)) +} + +/// Removes emulation prevention bytes (`00 00 03` → `00 00`). +fn unescape(nal: &[u8]) -> Vec { + let mut out = Vec::with_capacity(nal.len()); + let mut zeros = 0usize; + for &b in nal { + if zeros >= 2 && b == 3 { + zeros = 0; + continue; + } + zeros = if b == 0 { zeros + 1 } else { 0 }; + out.push(b); + } + out +} + +struct Bits<'a> { + data: &'a [u8], + pos: usize, +} + +impl Bits<'_> { + fn u(&mut self, n: u32) -> Option { + let mut v = 0u32; + for _ in 0..n { + let byte = *self.data.get(self.pos / 8)?; + v = (v << 1) | u32::from((byte >> (7 - self.pos % 8)) & 1); + self.pos += 1; + } + Some(v) + } + + fn flag(&mut self) -> Option { + self.u(1).map(|b| b == 1) + } + + fn skip(&mut self, n: usize) -> Option<()> { + self.pos = self.pos.checked_add(n)?; + (self.pos <= self.data.len().saturating_mul(8)).then_some(()) + } + + /// `ue(v)`; values past 32 bits are refused (no valid field here needs them). + fn ue(&mut self) -> Option { + let mut zeros = 0u32; + while !self.flag()? { + zeros += 1; + if zeros > 31 { + return None; + } + } + let rest = self.u(zeros)?; + ((1u64 << zeros) - 1 + u64::from(rest)).try_into().ok() + } + + fn se(&mut self) -> Option<()> { + self.ue().map(|_| ()) + } +} + +/// `profile_tier_level(1, max_sub_layers_minus1)` (7.3.3): only skipped. +fn profile_tier_level(r: &mut Bits, max_sub_layers_minus1: u32) -> Option<()> { + r.skip(88)?; // general profile space .. general_inbld/reserved flag + r.skip(8)?; // general_level_idc + let mut present = [(false, false); 8]; + for p in present.iter_mut().take(max_sub_layers_minus1 as usize) { + *p = (r.flag()?, r.flag()?); + } + if max_sub_layers_minus1 > 0 { + r.skip(2 * (8 - max_sub_layers_minus1 as usize))?; // reserved_zero_2bits + } + for &(profile, level) in present.iter().take(max_sub_layers_minus1 as usize) { + if profile { + r.skip(88)?; + } + if level { + r.skip(8)?; + } + } + Some(()) +} + +/// `scaling_list_data()` (7.3.4): only skipped. +fn scaling_list_data(r: &mut Bits) -> Option<()> { + for size_id in 0..4u32 { + let step = if size_id == 3 { 3 } else { 1 }; + for _ in (0..6).step_by(step) { + if !r.flag()? { + r.ue()?; // scaling_list_pred_matrix_id_delta + } else { + let coefs = 64.min(1 << (4 + (size_id << 1))); + if size_id > 1 { + r.se()?; // scaling_list_dc_coef_minus8 + } + for _ in 0..coefs { + r.se()?; // scaling_list_delta_coef + } + } + } + } + Some(()) +} + +/// `st_ref_pic_set(idx)` (7.3.7) as it appears in an SPS; returns `NumDeltaPocs[idx]`. +fn st_ref_pic_set(r: &mut Bits, idx: usize, num_delta_pocs: &[u32]) -> Option { + if idx != 0 && r.flag()? { + // inter_ref_pic_set_prediction_flag: predicted from the previous set (in an SPS, + // delta_idx_minus1 is absent and RefRpsIdx = idx - 1). + r.skip(1)?; // delta_rps_sign + r.ue()?; // abs_delta_rps_minus1 + let reference = *num_delta_pocs.get(idx - 1)?; + let mut count = 0u32; + for _ in 0..=reference { + let used = r.flag()?; + let kept = used || r.flag()?; // use_delta_flag when not used_by_curr_pic_flag + count += u32::from(kept); + } + return Some(count); + } + let negative = r.ue()?; + let positive = r.ue()?; + let total = negative.checked_add(positive)?; + if total > 32 { + return None; // at most 16 + 16 for any level + } + for _ in 0..total { + r.ue()?; // delta_poc_s*_minus1 + r.skip(1)?; // used_by_curr_pic_s*_flag + } + Some(total) +} + +fn parse_sps(rbsp: &[u8]) -> Option { + let r = &mut Bits { data: rbsp, pos: 0 }; + r.skip(4)?; // sps_video_parameter_set_id + let max_sub_layers_minus1 = r.u(3)?; + if max_sub_layers_minus1 > 6 { + return None; + } + r.skip(1)?; // sps_temporal_id_nesting_flag + profile_tier_level(r, max_sub_layers_minus1)?; + r.ue()?; // sps_seq_parameter_set_id + if r.ue()? == 3 { + r.skip(1)?; // separate_colour_plane_flag + } + r.ue()?; // pic_width_in_luma_samples + r.ue()?; // pic_height_in_luma_samples + if r.flag()? { + for _ in 0..4 { + r.ue()?; // conf_win_*_offset + } + } + r.ue()?; // bit_depth_luma_minus8 + r.ue()?; // bit_depth_chroma_minus8 + let log2_max_poc_lsb = r.ue()?.checked_add(4)?; + if log2_max_poc_lsb > 16 { + return None; + } + let first = if r.flag()? { 0 } else { max_sub_layers_minus1 }; + for _ in first..=max_sub_layers_minus1 { + r.ue()?; // sps_max_dec_pic_buffering_minus1 + r.ue()?; // sps_max_num_reorder_pics + r.ue()?; // sps_max_latency_increase_plus1 + } + for _ in 0..6 { + r.ue()?; // coding/transform block sizes, transform hierarchy depths + } + if r.flag()? && r.flag()? { + // scaling_list_enabled_flag, sps_scaling_list_data_present_flag + scaling_list_data(r)?; + } + r.skip(2)?; // amp_enabled_flag, sample_adaptive_offset_enabled_flag + if r.flag()? { + // pcm_enabled_flag + r.skip(8)?; // pcm sample bit depths + r.ue()?; + r.ue()?; + r.skip(1)?; // pcm_loop_filter_disabled_flag + } + let sets = r.ue()? as usize; + if sets > 64 { + return None; + } + let mut num_delta_pocs = Vec::with_capacity(sets); + for idx in 0..sets { + let n = st_ref_pic_set(r, idx, &num_delta_pocs)?; + num_delta_pocs.push(n); + } + if r.flag()? { + // long_term_ref_pics_present_flag + let n = r.ue()?; + if n > 32 { + return None; + } + for _ in 0..n { + r.skip(log2_max_poc_lsb as usize + 1)?; // lt_ref_pic_poc_lsb_sps, used_by_curr_pic_lt_sps_flag + } + } + r.skip(2)?; // sps_temporal_mvp_enabled_flag, strong_intra_smoothing_enabled_flag + let mut signal = Signal::default(); + if !r.flag()? { + return Some(signal); // no VUI + } + if r.flag()? && r.u(8)? == 255 { + // aspect_ratio_info_present_flag, aspect_ratio_idc == EXTENDED_SAR + r.skip(32)?; + } + if r.flag()? { + r.skip(1)?; // overscan_appropriate_flag + } + if r.flag()? { + // video_signal_type_present_flag + r.skip(3)?; // video_format + signal.full_range = r.flag()?; + if r.flag()? { + signal.primaries = r.u(8)? as u8; + signal.transfer = r.u(8)? as u8; + signal.matrix = r.u(8)? as u8; + } + } + Some(signal) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A bit writer for building parameter sets in tests. + #[derive(Default)] + pub(crate) struct Writer { + pub bits: Vec, + } + + impl Writer { + pub fn u(&mut self, v: u32, n: u32) { + for i in (0..n).rev() { + self.bits.push(i < 32 && (v >> i) & 1 == 1); + } + } + pub fn ue(&mut self, v: u32) { + let x = u64::from(v) + 1; + let len = 64 - x.leading_zeros(); + self.u(0, len - 1); + for i in (0..len).rev() { + self.bits.push((x >> i) & 1 == 1); + } + } + pub fn bytes(mut self) -> Vec { + self.bits.push(true); // rbsp_stop_one_bit + while !self.bits.len().is_multiple_of(8) { + self.bits.push(false); + } + self.bits.chunks(8).map(|c| c.iter().fold(0u8, |a, &b| (a << 1) | u8::from(b))).collect() + } + } + + fn sps(vui: impl FnOnce(&mut Writer), st_sets: impl FnOnce(&mut Writer)) -> Vec { + let mut w = Writer::default(); + w.u(0, 4); + w.u(1, 3); // one sub-layer above the base + w.u(1, 1); + w.u(0, 88); + w.u(93, 8); + w.u(0b11, 2); // sub-layer profile and level present + w.u(0, 2 * 7); + w.u(0, 88); + w.u(0, 8); + w.ue(0); + w.ue(1); // 4:2:0 + w.ue(4032); + w.ue(3024); + w.u(1, 1); // conformance window + for _ in 0..4 { + w.ue(3); + } + w.ue(0); + w.ue(0); + w.ue(4); + w.u(1, 1); + for _ in 0..2 { + w.ue(1); + w.ue(0); + w.ue(0); + } + for v in [0, 3, 0, 3, 0, 0] { + w.ue(v); + } + w.u(1, 1); // scaling lists, with data + w.u(1, 1); + for size_id in 0..4u32 { + for _ in (0..6).step_by(if size_id == 3 { 3 } else { 1 }) { + w.u(1, 1); + if size_id > 1 { + w.ue(0); + } + for _ in 0..64.min(1 << (4 + (size_id << 1))) { + w.ue(0); + } + } + } + w.u(0, 2); + w.u(1, 1); // pcm + w.u(0x77, 8); + w.ue(0); + w.ue(1); + w.u(1, 1); + st_sets(&mut w); + w.u(1, 1); // long-term pictures + w.ue(1); + w.u(0, 8 + 1); + w.u(0, 2); + vui(&mut w); + let mut nal = vec![SPS_NUT << 1, 1]; + // escape like a real NAL so `signal` has to unescape + let mut zeros = 0; + for b in w.bytes() { + if zeros >= 2 && b <= 3 { + nal.push(3); + zeros = 0; + } + zeros = if b == 0 { zeros + 1 } else { 0 }; + nal.push(b); + } + nal + } + + fn two_rps(w: &mut Writer) { + w.ue(2); + w.ue(1); // set 0: one negative picture + w.ue(0); + w.ue(0); + w.u(1, 1); + w.u(1, 1); // set 1: predicted from set 0 + w.u(0, 1); + w.ue(0); + w.u(0, 1); // not used ... + w.u(1, 1); // ... but kept + w.u(1, 1); + } + + #[test] + fn reads_full_range_and_matrix_from_the_vui() { + let nal = sps( + |w| { + w.u(1, 1); // vui present + w.u(1, 1); + w.u(255, 8); + w.u(0x0001_0001, 32); + w.u(1, 1); + w.u(0, 1); + w.u(1, 1); // video_signal_type_present_flag + w.u(5, 3); + w.u(1, 1); // full range + w.u(1, 1); + w.u(12, 8); + w.u(13, 8); + w.u(6, 8); + }, + two_rps, + ); + let s = signal(&[&[0x40, 1, 0xc], &nal, &[0x44, 1]]); + assert_eq!(s, Some(Signal { full_range: true, primaries: 12, transfer: 13, matrix: 6 })); + } + + #[test] + fn absent_fields_take_the_standard_defaults() { + let no_vui = sps(|w| w.u(0, 1), |w| w.ue(0)); + assert_eq!(signal(&[&no_vui]), Some(Signal::default())); + let range_only = sps( + |w| { + w.u(1, 1); + w.u(0, 2); + w.u(1, 1); + w.u(5, 3); + w.u(1, 1); + w.u(0, 1); // no colour description + }, + |w| w.ue(0), + ); + assert_eq!(signal(&[&range_only]), Some(Signal { full_range: true, ..Signal::default() })); + } + + #[test] + fn missing_or_truncated_sps_is_none() { + assert_eq!(signal(&[]), None); + assert_eq!(signal(&[&[0x40, 1, 2, 3]]), None); + let nal = sps(|w| w.u(0, 1), |w| w.ue(0)); + for cut in 2..nal.len() - 2 { + // never panics; most cuts are None + let _ = signal(&[nal.get(..cut).unwrap_or_default()]); + } + assert_eq!(signal(&[&[SPS_NUT << 1, 1, 0xff, 0xff, 0xff]]), None); + } +} diff --git a/crates/heif/src/ycc.rs b/crates/heif/src/ycc.rs new file mode 100644 index 000000000..a0429b453 --- /dev/null +++ b/crates/heif/src/ycc.rs @@ -0,0 +1,326 @@ +//! YCbCr → RGB for decoded HEVC pictures, done here rather than in heic-rs so the result matches +//! libheif (the reference every HEIF tool is checked against) to within one code value: +//! +//! - **Which matrix and range.** The item's `colr` `nclx` box when there is one; otherwise the +//! HEVC VUI ([`crate::vui`]); otherwise the H.265 defaults (limited range). An unspecified matrix +//! is BT.601, as libheif reads it. heic-rs 0.1.1 ignores the VUI and assumes BT.709 limited +//! range for every file without `nclx`, which shifts every iPhone photo (full range, ICC only) +//! by about 8 code values. +//! - **Chroma upsampling.** Nearest neighbour (each chroma sample covers its 2×2 or 2×1 luma +//! block), as libheif does by default. libheif switches to a centred bilinear filter (weights +//! 9/3/3/1 sixteenths, in integers) when a `clap` crop starts at an odd column or row; so does +//! [`Upsampling::Bilinear`]. +//! +//! The arithmetic is floating point and the result 16-bit, whatever the coded depth, so the +//! develop pipeline gets every bit the conversion produces. + +use heic_rs::grid::Mosaic; +use heic_rs::hevc::{ChromaFormat, Frame}; + +/// The coded pictures one image is made of. +pub enum Planes<'a> { + Picture(&'a Frame), + Grid(Mosaic<'a>), +} + +impl Planes<'_> { + pub fn size(&self) -> (usize, usize) { + match self { + Planes::Picture(f) => (f.width as usize, f.height as usize), + Planes::Grid(m) => (m.width() as usize, m.height() as usize), + } + } + + pub fn bit_depth(&self) -> u8 { + match self { + Planes::Picture(f) => f.bit_depth, + Planes::Grid(m) => m.bit_depth(), + } + } + + pub fn chroma(&self) -> ChromaFormat { + match self { + Planes::Picture(f) => f.chroma, + Planes::Grid(m) => m.chroma(), + } + } + + fn chroma_width(&self) -> usize { + let (w, h) = self.size(); + self.chroma().chroma_size(w as u32, h as u32).0 as usize + } + + fn luma_row(&self, y: usize, dst: &mut [u16]) -> Option<()> { + match self { + Planes::Picture(f) => copy_row(&f.y, f.y_stride as usize, y, dst), + Planes::Grid(m) => m.luma_row(y, dst), + } + } + + fn chroma_row(&self, cr: bool, r: usize, dst: &mut [u16]) -> Option<()> { + match self { + Planes::Picture(f) => copy_row(if cr { &f.cr } else { &f.cb }, f.c_stride as usize, r, dst), + Planes::Grid(m) => m.chroma_row(cr, r, dst), + } + } +} + +fn copy_row(plane: &[u16], stride: usize, y: usize, dst: &mut [u16]) -> Option<()> { + let start = y.checked_mul(stride)?; + dst.copy_from_slice(plane.get(start..start.checked_add(dst.len())?)?); + Some(()) +} + +/// How the three coded channels map to R, G, B (H.273 `MatrixCoefficients`). +#[derive(Debug, Clone, Copy, PartialEq)] +enum Matrix { + /// 0: the channels are G, B, R. + Identity, + /// 8: YCgCo. + YCgCo, + /// Every Kr/Kb matrix: R = Y + `cr_r`·Cr, G = Y − `cb_g`·Cb − `cr_g`·Cr, B = Y + `cb_b`·Cb. + Kr { cr_r: f32, cb_g: f32, cr_g: f32, cb_b: f32 }, +} + +impl Matrix { + /// Unknown, reserved and unspecified codes are BT.601, like libheif. + fn from_code(code: u16) -> Matrix { + let (kr, kb) = match code { + 0 => return Matrix::Identity, + 8 => return Matrix::YCgCo, + 1 => (0.2126, 0.0722), + 4 => (0.30, 0.11), + 7 => (0.212, 0.087), + // 10 (BT.2020 constant luminance) is approximated by its non-constant matrix. + 9 | 10 => (0.2627, 0.0593), + _ => (0.299, 0.114), + }; + let kg = 1.0 - kr - kb; + Matrix::Kr { cr_r: 2.0 * (1.0 - kr), cb_g: 2.0 * kb * (1.0 - kb) / kg, cr_g: 2.0 * kr * (1.0 - kr) / kg, cb_b: 2.0 * (1.0 - kb) } + } +} + +/// How the samples are to be read: the matrix and the range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Signal { + pub matrix: u16, + pub full_range: bool, +} + +/// How subsampled chroma is brought to the luma resolution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Upsampling { + Nearest, + /// Centred bilinear: chroma sits between its two luma columns (and rows for 4:2:0). + Bilinear, +} + +/// One chroma row brought to the full width `w` of luma row `y`. +fn chroma_at(planes: &Planes<'_>, cr: bool, y: usize, w: usize, mode: Upsampling, buf: &mut [u16], next: &mut [u16]) -> Option> { + let chroma = planes.chroma(); + let (xs, ys) = (chroma.x_shift(), chroma.y_shift()); + let (wf, hf) = planes.size(); + let rows = chroma.chroma_size(wf as u32, hf as u32).1 as usize; + let cy = y >> ys; + planes.chroma_row(cr, cy, buf)?; + let last = buf.len().checked_sub(1)?; + if mode == Upsampling::Nearest || xs == 0 { + return (0..w).map(|x| buf.get(x >> xs).copied()).collect(); + } + // The nearer neighbour on each axis takes a quarter of the weight. + let ny = if ys == 0 { + cy + } else if y.is_multiple_of(2) { + cy.saturating_sub(1) + } else { + (cy + 1).min(rows.checked_sub(1)?) + }; + planes.chroma_row(cr, ny, next)?; + (0..w) + .map(|x| { + let cx = x >> 1; + let nx = if x % 2 == 0 { cx.saturating_sub(1) } else { (cx + 1).min(last) }; + let (a, an) = (u32::from(*buf.get(cx)?), u32::from(*buf.get(nx)?)); + let v = if ys == 0 { + (3 * a + an + 2) >> 2 + } else { + let (b, bn) = (u32::from(*next.get(cx)?), u32::from(*next.get(nx)?)); + (9 * a + 3 * b + 3 * an + bn + 8) >> 4 + }; + u16::try_from(v).ok() + }) + .collect() +} + +/// Converts `planes` to interleaved 16-bit RGB, or RGBA with room left for alpha (filled with +/// 65535 here). `None` when the planes can't supply a row they promised (a malformed grid). +pub fn to_rgb16(planes: &Planes<'_>, signal: Signal, alpha: bool, mode: Upsampling) -> Option> { + let (w, h) = planes.size(); + let ch = if alpha { 4 } else { 3 }; + let bd = u32::from(planes.bit_depth()); + if !(8..=16).contains(&bd) || w == 0 || h == 0 { + return None; + } + let max = ((1u32 << bd) - 1) as f32; + let half = (1u32 << (bd - 1)) as f32; + let scale = (1u32 << (bd - 8)) as f32; + // Normalised Y = (v − y_off)·y_mul, Cb/Cr = (v − half)·c_mul. + let (y_off, y_mul, c_mul) = + if signal.full_range { (0.0, 1.0 / max, 1.0 / max) } else { (16.0 * scale, 1.0 / (219.0 * scale), 1.0 / (224.0 * scale)) }; + let matrix = Matrix::from_code(signal.matrix); + let mono = planes.chroma() == ChromaFormat::Monochrome; + let cw = planes.chroma_width(); + + let mut out = vec![0u16; w.checked_mul(h)?.checked_mul(ch)?]; + let row = |y: usize, dst: &mut [u16]| -> Option<()> { + let mut luma = vec![0u16; w]; + planes.luma_row(y, &mut luma)?; + let (cb, cr) = if mono { + (Vec::new(), Vec::new()) + } else { + let (mut buf, mut next) = (vec![0u16; cw], vec![0u16; cw]); + (chroma_at(planes, false, y, w, mode, &mut buf, &mut next)?, chroma_at(planes, true, y, w, mode, &mut buf, &mut next)?) + }; + for (x, (px, &yv)) in dst.chunks_exact_mut(ch).zip(&luma).enumerate() { + let rgb = if mono { + let v = (f32::from(yv) - y_off) * y_mul; + [v, v, v] + } else { + let (u, v) = (f32::from(*cb.get(x)?), f32::from(*cr.get(x)?)); + match matrix { + // G, B, R all read like luma. + Matrix::Identity => [(v - y_off) * y_mul, (f32::from(yv) - y_off) * y_mul, (u - y_off) * y_mul], + Matrix::YCgCo => { + let (yn, cg, co) = ((f32::from(yv) - y_off) * y_mul, (u - half) * c_mul, (v - half) * c_mul); + let t = yn - cg; + [t + co, yn + cg, t - co] + } + Matrix::Kr { cr_r, cb_g, cr_g, cb_b } => { + let (yn, u, v) = ((f32::from(yv) - y_off) * y_mul, (u - half) * c_mul, (v - half) * c_mul); + [yn + cr_r * v, yn - cb_g * u - cr_g * v, yn + cb_b * u] + } + } + }; + for (o, c) in px.iter_mut().zip(rgb) { + *o = (c.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16; + } + if alpha && let Some(a) = px.get_mut(3) { + *a = u16::MAX; + } + } + Some(()) + }; + let row_len = w * ch; + #[cfg(not(target_arch = "wasm32"))] + let ok = { + use rayon::prelude::*; + out.par_chunks_mut(row_len).enumerate().all(|(y, dst)| row(y, dst).is_some()) + }; + #[cfg(target_arch = "wasm32")] + let ok = out.chunks_mut(row_len).enumerate().all(|(y, dst)| row(y, dst).is_some()); + ok.then_some(out) +} + +/// Writes an alpha picture's luma, full range, into channel 3 of `rgba`. +pub fn put_alpha(planes: &Planes<'_>, rgba: &mut [u16]) -> Option<()> { + let (w, h) = planes.size(); + let max = ((1u32 << u32::from(planes.bit_depth()).clamp(1, 16)) - 1) as f32; + let mut luma = vec![0u16; w]; + for (y, row) in rgba.chunks_exact_mut(w * 4).enumerate().take(h) { + planes.luma_row(y, &mut luma)?; + for (px, &a) in row.as_chunks_mut::<4>().0.iter_mut().zip(&luma) { + let v = (f32::from(a) / max).clamp(0.0, 1.0); + px[3] = (v * 65535.0 + 0.5) as u16; + } + } + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(w: u32, h: u32, bd: u8, chroma: ChromaFormat, y: u16, cb: u16, cr: u16) -> Frame { + let (cw, chh) = chroma.chroma_size(w, h); + let n = (cw * chh) as usize; + Frame { + width: w, + height: h, + bit_depth: bd, + chroma, + y: vec![y; (w * h) as usize], + cb: vec![cb; n], + cr: vec![cr; n], + y_stride: w, + c_stride: cw, + } + } + + fn rgb(f: &Frame, matrix: u16, full_range: bool) -> [u16; 3] { + let v = to_rgb16(&Planes::Picture(f), Signal { matrix, full_range }, false, Upsampling::Nearest).unwrap(); + [v[0], v[1], v[2]] + } + + #[test] + fn neutral_and_primaries_convert_exactly() { + // Full-range grey stays grey; limited-range black and white reach the ends. + assert_eq!(rgb(&frame(4, 4, 8, ChromaFormat::Yuv420, 128, 128, 128), 6, true), [128 * 257; 3]); + assert_eq!(rgb(&frame(4, 4, 8, ChromaFormat::Yuv420, 16, 128, 128), 1, false), [0; 3]); + assert_eq!(rgb(&frame(4, 4, 8, ChromaFormat::Yuv420, 235, 128, 128), 1, false), [65535; 3]); + assert_eq!(rgb(&frame(4, 4, 10, ChromaFormat::Yuv420, 940, 512, 512), 9, false), [65535; 3]); + // BT.601 full-range pure red: Y = 76.245, Cb = 84.97, Cr = 255.5 → (255, 0, 0) within rounding. + let red = rgb(&frame(2, 2, 8, ChromaFormat::Yuv444, 76, 85, 255), 6, true); + assert!(red[0] > 65000 && red[1] < 300 && red[2] < 300, "{red:?}"); + // Identity: the channels are G, B, R. + assert_eq!(rgb(&frame(2, 2, 8, ChromaFormat::Yuv444, 10, 20, 30), 0, true), [30 * 257, 10 * 257, 20 * 257]); + // Monochrome. + assert_eq!(rgb(&frame(2, 2, 8, ChromaFormat::Monochrome, 51, 0, 0), 6, true), [51 * 257; 3]); + } + + #[test] + fn chroma_is_nearest_neighbour() { + // 4:2:0, 4×2 luma, chroma 2×1: the left chroma sample covers columns 0–1 only. + let mut f = frame(4, 2, 8, ChromaFormat::Yuv420, 128, 128, 128); + f.cr = vec![200, 128]; + let v = to_rgb16(&Planes::Picture(&f), Signal { matrix: 6, full_range: true }, false, Upsampling::Nearest).unwrap(); + let red: Vec = v.chunks(3).map(|p| p[0]).collect(); + assert!(red[0] == red[1] && red[1] == red[4] && red[0] > red[2] && red[2] == 128 * 257, "{red:?}"); + } + + #[test] + fn bilinear_chroma_is_centred_9_3_3_1() { + // 4:2:0, 4×4 luma, chroma 2×2 with one bright corner sample. + let mut f = frame(4, 4, 8, ChromaFormat::Yuv420, 128, 128, 128); + f.cr = vec![200, 128, 128, 128]; + let p = Planes::Picture(&f); + let (mut buf, mut next) = (vec![0; 2], vec![0; 2]); + let rows: Vec> = (0..4).map(|y| chroma_at(&p, true, y, 4, Upsampling::Bilinear, &mut buf, &mut next).unwrap()).collect(); + // Row 0 sits nearest chroma row 0 (and clamps upward); row 1 blends a quarter of row 1. + assert_eq!(rows[0], [200, 182, 146, 128]); + assert_eq!(rows[1], [182, 169, 142, 128]); + assert_eq!(rows[2], [146, 142, 133, 128]); + assert_eq!(rows[3], [128; 4]); + assert_eq!(chroma_at(&p, true, 1, 4, Upsampling::Nearest, &mut buf, &mut next).unwrap(), [200, 200, 128, 128]); + } + + #[test] + fn alpha_and_bad_planes() { + let f = frame(2, 2, 8, ChromaFormat::Yuv420, 128, 128, 128); + let mut v = to_rgb16(&Planes::Picture(&f), Signal { matrix: 6, full_range: true }, true, Upsampling::Nearest).unwrap(); + assert_eq!(v[3], 65535); + put_alpha(&Planes::Picture(&frame(2, 2, 8, ChromaFormat::Monochrome, 51, 0, 0)), &mut v).unwrap(); + assert_eq!(v[3], 51 * 257); + let mut short = frame(4, 4, 8, ChromaFormat::Yuv420, 128, 128, 128); + short.y.truncate(5); + assert!(to_rgb16(&Planes::Picture(&short), Signal { matrix: 6, full_range: true }, false, Upsampling::Bilinear).is_none()); + assert!( + to_rgb16( + &Planes::Picture(&frame(2, 2, 7, ChromaFormat::Yuv420, 0, 0, 0)), + Signal { matrix: 6, full_range: true }, + false, + Upsampling::Nearest + ) + .is_none() + ); + } +} diff --git a/docs/parity.md b/docs/parity.md index 4c27755b5..a4d2cd0b3 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -142,7 +142,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported (fields that only describe the preset or photo, e.g. `Cluster`, `SortName`, `SupportsAmount2`, the as-shot white, empty Point Color slots, are not); profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | -| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs`, `crates/raw/src/jxl.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG and DNG 1.7 JPEG XL tiles/strips: lossless ones decode bit-exactly on synthetic CFA and LinearRaw files, lossy ones are restored to the declared linear primaries without clipping or gamut mapping, checked on one real file with 108 lossy tiles, and any tile that fails to decode makes the file preview-only; a single-chunk JPEG XL preview IFD is offered as the embedded preview), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`), Hasselblad 3FR with uncompressed raw data (X1D, X1D II, X2D, CFV 100C: a TIFF without a DNG version tag that describes its CFA IFD, levels, crop and white balance with DNG's tags; `RawFormat::CfaTiff`, read by `dng::decode_as`, RGGB when the file states no `CFAPattern`; the lossless-JPEG 3FR and FFF files use scan selection value 8 and stay preview-only), Samsung SRW without compression (NX5, NX10, NX11 and the NX20-generation bodies NX20, NX200, NX210, NX1000 and NX1100 as 12-bit packed samples in either bit order, EX1 and WB2000 as 16-bit words; layout, camera-JPEG framing, levels and white balance from the maker note, `crates/raw/src/vendor/srw.rs`; median ΔE vs the camera JPEG 2.2–7.6 on 11 of the 12 files from 10 bodies checked, 10.7 (5.7 registered) on the backlit WB2000 sample). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini: preview only with the reason), NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", AVIF decode (HEIC decodes only behind the non-default `heif` feature: `crates/heif`, heic-rs, same policy as PhotoCraft; 8/10-bit, alpha, irot/imir/clap, ICC/EXIF/XMP), palette (indexed-colour) TIFF (refused by the `tiff` crate) | +| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs`, `crates/raw/src/jxl.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG and DNG 1.7 JPEG XL tiles/strips: lossless ones decode bit-exactly on synthetic CFA and LinearRaw files, lossy ones are restored to the declared linear primaries without clipping or gamut mapping, checked on one real file with 108 lossy tiles, and any tile that fails to decode makes the file preview-only; a single-chunk JPEG XL preview IFD is offered as the embedded preview), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`), Hasselblad 3FR with uncompressed raw data (X1D, X1D II, X2D, CFV 100C: a TIFF without a DNG version tag that describes its CFA IFD, levels, crop and white balance with DNG's tags; `RawFormat::CfaTiff`, read by `dng::decode_as`, RGGB when the file states no `CFAPattern`; the lossless-JPEG 3FR and FFF files use scan selection value 8 and stay preview-only), Samsung SRW without compression (NX5, NX10, NX11 and the NX20-generation bodies NX20, NX200, NX210, NX1000 and NX1100 as 12-bit packed samples in either bit order, EX1 and WB2000 as 16-bit words; layout, camera-JPEG framing, levels and white balance from the maker note, `crates/raw/src/vendor/srw.rs`; median ΔE vs the camera JPEG 2.2–7.6 on 11 of the 12 files from 10 bodies checked, 10.7 (5.7 registered) on the backlit WB2000 sample). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini: preview only with the reason), NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", AVIF decode (HEIC decodes only behind the non-default `heif` feature, which the release packages enable: `crates/heif`, heic-rs for the container and HEVC (bit-exact against libde265 on iPhone tiles) plus our own colour conversion, matching libheif within 1 code value on iPhone-style grids (VUI full range, Display P3 ICC, nclx, 10-bit, irot/imir/clap, gain map/depth ignored); 4:2:0/monochrome only, 4:2:2/4:4:4 Canon/Sony HIF refused; same policy as PhotoCraft), palette (indexed-colour) TIFF (refused by the `tiff` crate) | | LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | diff --git a/docs/whats-new.md b/docs/whats-new.md index 33557867d..4e9c8fb91 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -83,6 +83,12 @@ - HEIC / HEIF photos (iPhone and Mac) open now: the optional `lightcraft-heif` crate (heic-rs, pure Rust) behind codecs' `heif` feature — 8- and 10-bit, alpha, grid tiles, the container's rotation/mirror/crop, ICC, EXIF and XMP. Off by default (HEVC patents are the distributor's call, same as PhotoCraft); official builds pass `--features heif`. +- HEIC colours now match libheif to within one code value (they were up to ~10 off): the HEVC stream's own + full-range/matrix signalling is honoured (iPhone photos are full range), chroma is upsampled like libheif, grids + take their tiles' ICC profile, and `imir` mirrors the way libheif writes it. Importing a HEIC reads its size from + the container instead of decoding it; small previews use the file's embedded thumbnail. The release packages + (macOS, Windows, Linux, FreeBSD, Nix) now really are built with HEIC support; a build without it reports + "HEIC/HEIF support isn't included in this build" for each `.heic`/`.heif` at import. ### Presets and profiles - Import presets from other editors: XMP presets, classic `.lrtemplate` files, "DNG presets" from mobile apps and `.zip` diff --git a/nix/package.nix b/nix/package.nix index b364b5c47..9e769602e 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -75,6 +75,12 @@ let "lightcraft" "lightcraft-cli" ]; + + # HEIC/HEIF decoding (opt-in upstream: HEVC patents are the distributor's call), as the release builds. + buildFeatures = [ + "lightcraft/heif" + "lightcraft-cli/heif" + ]; in rustPlatform.buildRustPackage { pname = "lightcraft"; diff --git a/packaging/freebsd/package.sh b/packaging/freebsd/package.sh index 0f3b928ab..665444f80 100755 --- a/packaging/freebsd/package.sh +++ b/packaging/freebsd/package.sh @@ -47,7 +47,7 @@ echo "==> LightCraft $VERSION for FreeBSD $ARCH" # The release VM has 12 GB; full parallelism on the biggest crates runs it out of memory. export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-4}" if [ "$SKIP_BUILD" = 0 ]; then - (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli) + (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli --features lightcraft/heif,lightcraft-cli/heif) fi WORK="$CARGO_TARGET_DIR/freebsd-package" STAGE="$WORK/$BASENAME" diff --git a/packaging/linux/package.sh b/packaging/linux/package.sh index e03cab5cd..441cd69c6 100755 --- a/packaging/linux/package.sh +++ b/packaging/linux/package.sh @@ -42,7 +42,7 @@ BASENAME="lightcraft-$VERSION-linux-$ARCH" echo "==> LightCraft $VERSION for Linux $ARCH ($FORMATS)" if [ "$SKIP_BUILD" = 0 ]; then - (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli) + (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli --features lightcraft/heif,lightcraft-cli/heif) fi BIN="$CARGO_TARGET_DIR/release" WORK="$CARGO_TARGET_DIR/linux-package" diff --git a/packaging/macos/package.sh b/packaging/macos/package.sh index 5416e0dbf..b43166f1e 100755 --- a/packaging/macos/package.sh +++ b/packaging/macos/package.sh @@ -58,7 +58,7 @@ echo "==> LightCraft $VERSION for macOS ($ARCH), identity: $IDENTITY, notarize: if [ "$SKIP_BUILD" = 0 ]; then args=() for t in "${TARGETS[@]}"; do args+=(--target "$t"); done - (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli "${args[@]}") + (cd "$ROOT" && cargo build --release --locked -p lightcraft -p lightcraft-cli --features lightcraft/heif,lightcraft-cli/heif "${args[@]}") fi rm -rf "$WORK" diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index dae317570..78ec9fc93 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -62,7 +62,7 @@ if (-not $SkipBuild) { [Environment]::SetEnvironmentVariable($flagVar, '-C target-feature=+crt-static') # Fail the build (rather than warn) if the icon/VERSIONINFO can't be embedded. $env:LIGHTCRAFT_REQUIRE_WINRES = '1' - Invoke-Native "cargo build ($Target)" { cargo build --release --locked -p lightcraft -p lightcraft-cli --target $Target } + Invoke-Native "cargo build ($Target)" { cargo build --release --locked -p lightcraft -p lightcraft-cli --features lightcraft/heif,lightcraft-cli/heif --target $Target } } $Bin = Join-Path $TargetDir "$Target\release" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index cfdec92a3..acb98a6b5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -268,7 +268,11 @@ fn cmd_ci() -> Result<(), String> { run(c, "cargo clippy -p lightcraft-codecs --features heif --all-targets -- -D warnings")?; let mut c = cargo(); c.args(["test", "-p", "lightcraft-codecs", "-p", "lightcraft-heif", "--features", "lightcraft-codecs/heif"]); - run(c, "cargo test -p lightcraft-codecs -p lightcraft-heif --features lightcraft-codecs/heif") + run(c, "cargo test -p lightcraft-codecs -p lightcraft-heif --features lightcraft-codecs/heif")?; + // the engine's HEIC import test, with the decoder (the workspace run checks the error) + let mut c = cargo(); + c.args(["test", "-p", "lightcraft-engine", "--lib", "--features", "lightcraft-codecs/heif", "heic"]); + run(c, "cargo test -p lightcraft-engine --lib --features lightcraft-codecs/heif heic") }), ), (