DNG: lossy compression 34892 (baseline JPEG tiles) (#2021)

* DNG: lossy compression 34892 (baseline JPEG tiles)

Adobe's lossy DNG (compression 34892) decodes: each tile is a complete
baseline JPEG (8-bit luma at tile size), decoded by zune-jpeg into the
tile geometry (a mismatch is an error), widened 0xFF -> 0xFFFF like
Photoshop's read of 8-bit CFA data. Predictors and the rest of the
pipeline are unchanged.

testgen grows LossyJpegTiles (image-rs encodes the Q60 fixtures behind
the testgen feature), so CI runs a decode oracle: a smooth ramp stays
within a few quantisation steps of the 8-bit source, and a tile cut
mid-stream fails cleanly.

* Lossy DNG: check the tile JPEG header before decoding; zune deps go workspace

decode_headers() runs and the (width, height) == (s.w, rows) check fires
before decode(), so a hostile SOF cannot allocate the decoder's buffers
(up to 16384x16364) before the size mismatch is caught. The mismatch has
a regression test that swaps a 32x32 tile JPEG for a 16x16 one. zune-core
and zune-jpeg move to [workspace.dependencies] next to image and flate2.

* chore(ui): format prefs_ui (the workspace fmt gate is red on main until #2417 lands)

* Lossy DNG: 3-sample LinearRaw (YCbCr) decodes to RGB; tiles size-check

Real lossy DNGs are 3-sample YCbCr LinearRaw, not single-channel CFA:
the tile now decodes through zune-jpeg's YCbCr-to-RGB output when
SamplesPerPixel is 3 (gray stays for CFA), the decoded length is checked
against the tile's sample count, and other sample counts say unsupported.
The generator writes RGB tiles for 3-sample specs. A LinearizationTable
maps the decoded samples in the develop stage as before - nothing to add
in the sensor. Test: a 3-sample ramp with 64-step channel separation
(surviving Q60 chroma quantisation) decodes within the quality band.

Cargo.lock

@@ -4504,11 +4504,13 @@ name = "photocraft-raw"
 version = "0.6.0"
 dependencies = [
  "flate2",
+ "image",
  "jxl-oxide",
  "photocraft-raw",
  "png 0.18.1",
  "rayon",
  "zune-core 0.5.3",
+ "zune-jpeg 0.5.15",
  "zune-jpegxl",
 ]
 

Cargo.toml

@@ -44,6 +44,9 @@ arc-swap = "1"
 bytemuck = { version = "1", features = ["derive"] }
 flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 image = { version = "0.25", default-features = false }
+# Baseline-JPEG decode of lossy DNG tiles (compression 34892), shared with zune-jpeg.
+zune-core = "0.5"
+zune-jpeg = "0.5"
 proptest = "1"
 eframe = { version = "0.36", default-features = false }
 egui = "0.36"

crates/raw/Cargo.toml

@@ -10,13 +10,18 @@ rust-version.workspace = true
 default = []
 # Synthetic raw-file generators (DNG / CR2 / TIFF-EP writers and a lossless-JPEG
 # encoder) for tests and fuzz seeds.
-testgen = []
+testgen = ["dep:image"]
 # Real-file corpus tests (opt-in; `cargo xtask test-corpus`). Fail when the corpus is missing.
 corpus = []
 
 [dependencies]
 # Adobe DNG compression 8: zlib (RFC 1950) with the TIFF predictor (pure-Rust backend).
 flate2 = { workspace = true }
+# Adobe DNG compression 34892: baseline JPEG tiles (lossy), decoded by zune-jpeg.
+zune-jpeg = { workspace = true }
+# Encodes the lossy-JPEG tiles of the testgen fixtures (feature `testgen`).
+image = { workspace = true, features = ["jpeg"], optional = true }
+zune-core = { workspace = true }
 
 # Parallel bands on native; wasm builds stay single-threaded.
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]

crates/raw/src/sensor.rs

@@ -212,8 +212,7 @@ pub(crate) fn read_plane(t: &Tiff, ifd: &Ifd, limits: &Limits, layout: JpegLayou
         return Err(RawError::unsupported("planar raw data"));
     }
     match compression {
-        1 | 7 | 8 | JPEG_XL => {}
-        34892 => return Err(RawError::unsupported("lossy-compressed DNG")),
+        1 | 7 | 8 | 34892 | JPEG_XL => {}
         34713 => return Err(RawError::unsupported("Nikon compressed NEF")),
         32767 => return Err(RawError::unsupported("Sony compressed ARW")),
         65535 => return Err(RawError::unsupported("Pentax compressed PEF")),
@@ -255,6 +254,32 @@ pub(crate) fn read_plane(t: &Tiff, ifd: &Ifd, limits: &Limits, layout: JpegLayou
                 v.resize(s.w * s.h * samples, 0);
                 Ok(v)
             }
+            34892 => {
+                // Adobe DNG compression 34892: baseline JPEG tiles (lossy, 8-bit). The tile is
+                // a complete JPEG stream: a CFA part decodes to gray samples, a 3-sample
+                // LinearRaw part holds YCbCr, which zune-jpeg converts to RGB. The header is
+                // decoded and size-checked before any pixel allocation, so a hostile tile
+                // cannot size the decoder's buffers (up to 16384x16384) with a fake SOF.
+                let out = match samples {
+                    1 => zune_core::colorspace::ColorSpace::Luma,
+                    3 => zune_core::colorspace::ColorSpace::RGB,
+                    n => return Err(RawError::unsupported(format!("lossy DNG with {n} samples per pixel (1 or 3 are supported)"))),
+                };
+                let options = zune_core::options::DecoderOptions::default().jpeg_set_out_colorspace(out);
+                let mut dec = zune_jpeg::JpegDecoder::new_with_options(zune_core::bytestream::ZCursor::new(src), options);
+                dec.decode_headers().map_err(|e| RawError::malformed(format!("lossy JPEG tile header: {e:?}")))?;
+                let f = dec.info().ok_or_else(|| RawError::malformed("lossy JPEG tile has no header"))?;
+                if (f.width as usize, f.height as usize) != (s.w, rows) {
+                    return Err(RawError::malformed(format!("lossy JPEG tile is {}x{}, expected {}x{}", f.width, f.height, s.w, rows)));
+                }
+                let decoded = dec.decode().map_err(|e| RawError::malformed(format!("lossy JPEG tile: {e:?}")))?;
+                let need = s.w * rows * samples;
+                if decoded.len() != need {
+                    return Err(RawError::malformed(format!("lossy JPEG tile holds {} samples, expected {need}", decoded.len())));
+                }
+                // The 8-bit samples widen to the document's 16-bit range (0xFF -> 0xFFFF).
+                Ok(decoded.into_iter().map(|b| u16::from(b) * 257).collect())
+            }
             _ => {
                 let need = s.w * rows * samples;
                 let max = s.w.saturating_mul(s.h).saturating_mul(samples).saturating_mul(4);

crates/raw/src/testgen.rs

@@ -368,6 +368,9 @@ fn urat(v: f64) -> (u32, u32) {
 pub enum DngStorage {
     /// One strip per `rows` rows, uncompressed at `bits` per sample (8, 12, 16…).
     Strips { rows: usize },
+    /// Adobe DNG compression 34892: baseline JPEG tiles, 8-bit luma at tile size (the reader
+    /// decodes and widens to 16 bits). Quality 60 leaves visible-but-mild artifacts.
+    LossyJpegTiles { width: usize, height: usize },
     /// Adobe DNG compression 8: zlib streams with the TIFF predictor (1 = none, 2 = horizontal
     /// differencing) applied before compressing.
     DeflateStrips { rows: usize, predictor: u16 },
@@ -560,6 +563,39 @@ impl DngSpec {
                 raw.push((273, Val::Blobs(offs)));
                 raw.push((279, Val::Long(lens)));
             }
+            DngStorage::LossyJpegTiles { width: tw, height: th } => {
+                let mut offs = Vec::new();
+                let mut lens = Vec::new();
+                let s = self.samples;
+                for ty in (0..self.height).step_by(th) {
+                    for tx in (0..self.width).step_by(tw) {
+                        // Edge tiles are padded by repeating the last row / column. A 3-sample
+                        // (LinearRaw) spec writes RGB tiles, a CFA spec gray ones.
+                        let mut tile = vec![0u8; tw * th * s];
+                        for y in 0..th {
+                            for x in 0..tw {
+                                let sy = (ty + y).min(self.height - 1);
+                                let sx = (tx + x).min(self.width - 1);
+                                for c in 0..s {
+                                    tile[(y * tw + x) * s + c] = (self.data[(sy * self.width + sx) * s + c] >> 8) as u8;
+                                }
+                            }
+                        }
+                        let colour = if s == 3 { image::ExtendedColorType::Rgb8 } else { image::ExtendedColorType::L8 };
+                        let mut j = Vec::new();
+                        let enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut j, 60);
+                        use image::ImageEncoder as _;
+                        enc.write_image(&tile, tw as u32, th as u32, colour).expect("jpeg tile");
+                        lens.push(j.len() as u32);
+                        offs.push(t.blob(j));
+                    }
+                }
+                raw.push((259, Val::Short(vec![34892])));
+                raw.push((322, Val::Long(vec![tw as u32])));
+                raw.push((323, Val::Long(vec![th as u32])));
+                raw.push((324, Val::Blobs(offs)));
+                raw.push((325, Val::Long(lens)));
+            }
             DngStorage::Lj92Strips { rows } => {
                 let mut offs = Vec::new();
                 let mut lens = Vec::new();

crates/raw/tests/dng.rs

@@ -560,3 +560,88 @@ fn profile_gain_table_map_is_reported() {
     spec.profile_gain_table_map = Some(vec![0; 64]);
     assert!(gain_table(&sensor(&spec.build())));
 }
+
+/// Adobe DNG compression 34892 (lossy baseline JPEG, 8-bit luma tiles): decodes to the tile
+/// samples widened to 16 bits, within the JPEG quality band (Q60 artifacts are mild but real).
+#[test]
+fn lossy_jpeg_tiles_decode() {
+    let (w, h) = (32, 32);
+    // A smooth ramp: baseline JPEG at Q60 keeps it within a few quantisation steps.
+    let data: Vec<u16> = (0..w * h).map(|i| 4096 + (i / w) as u16 * 16).collect();
+    let mut spec = DngSpec::cfa(w, h, data.clone());
+    spec.storage = DngStorage::LossyJpegTiles { width: 16, height: 16 };
+    let s = sensor(&spec.build());
+    assert_eq!((s.width, s.height, s.samples), (w, h, 1));
+    let mut worst = 0u32;
+    for (got, want) in s.data.iter().zip(&data) {
+        // The tile was written 8-bit (>>8), so the quantised expectation is (want>>8)<<8.
+        let quant = u32::from(*want >> 8) << 8;
+        worst = worst.max(u32::from(got.abs_diff(quant as u16)));
+    }
+    assert!(worst <= 4 * 256, "worst deviation from the 8-bit quantisation: {worst}");
+}
+
+/// A lossy JPEG tile cut mid-stream fails cleanly.
+#[test]
+fn lossy_jpeg_tiles_hostile() {
+    let (w, h) = (32, 32);
+    let data: Vec<u16> = (0..w * h).map(|_i| 8192u16).collect();
+    let mut spec = DngSpec::cfa(w, h, data);
+    spec.storage = DngStorage::LossyJpegTiles { width: 32, height: 32 };
+    let bytes = spec.build();
+    let at = bytes.windows(2).position(|w| w == [0xFF, 0xD8]).expect("jpeg SOI") + 2;
+    let cut = &bytes[..at + (bytes.len() - at) / 2];
+    assert!(decode(cut, &Limits::default()).is_err(), "a cut JPEG tile must not decode");
+}
+
+/// A tile JPEG whose header lies about its size fails the header check, before any pixel
+/// decode (a hostile SOF must not be able to size the decoder's buffers first).
+#[test]
+fn lossy_jpeg_tile_dimension_mismatch_errors() {
+    use image::ImageEncoder as _;
+    let (w, h) = (32, 32);
+    let data: Vec<u16> = (0..w * h).map(|i| 4096 + (i / w) as u16 * 16).collect();
+    let mut spec = DngSpec::cfa(w, h, data);
+    spec.storage = DngStorage::LossyJpegTiles { width: 32, height: 32 };
+    let mut bytes = spec.build();
+    // Swap the 32x32 tile JPEG for a 16x16 one, inside the same segment (the trailing bytes
+    // after its EOI stay; the JPEG decoder stops at EOI).
+    let small = {
+        let img = image::GrayImage::from_fn(16, 16, |x, _| image::Luma([(x as u16 * 16) as u8]));
+        let mut j = Vec::new();
+        let enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut j, 60);
+        enc.write_image(img.as_raw(), 16, 16, image::ExtendedColorType::L8).unwrap();
+        j
+    };
+    let soi = bytes.windows(3).position(|w| w == [0xFF, 0xD8, 0xFF]).expect("SOI");
+    let eoi = soi + bytes[soi..].windows(2).position(|w| w == [0xFF, 0xD9]).expect("EOI") + 2;
+    bytes.splice(soi..eoi, small);
+    let err = decode(&bytes, &Limits::default()).unwrap_err().to_string();
+    assert!(err.contains("16x16") && err.contains("32x32"), "{err}");
+}
+
+/// A 3-sample LinearRaw lossy DNG (YCbCr on disk, as real camera files are) decodes to RGB
+/// tiles widened to 16 bits, within the JPEG quality band; a LinearizationTable maps the
+/// decoded samples in the develop stage, so the sensor sees the raw widened values.
+#[test]
+fn lossy_jpeg_rgb_linearraw_decodes() {
+    let (w, h) = (32, 32);
+    // Channel offsets of 64 8-bit steps survive JPEG chroma quantisation at Q60.
+    let data: Vec<u16> = (0..w * h * 3).map(|i| 4096 + ((i / (w * 3)) as u16 * 16) + (i % 3) as u16 * 64 * 256).collect();
+    let mut spec = DngSpec::cfa(w, h, data.clone());
+    spec.samples = 3;
+    spec.storage = DngStorage::LossyJpegTiles { width: 16, height: 16 };
+    let s = sensor(&spec.build());
+    assert_eq!((s.width, s.height, s.samples), (w, h, 3));
+    let mut worst = 0u32;
+    for (got, want) in s.data.iter().zip(&data) {
+        let quant = u32::from(*want >> 8) << 8;
+        worst = worst.max(u32::from(got.abs_diff(quant as u16)));
+    }
+    assert!(worst <= 8 * 256, "worst deviation from the 8-bit quantisation: {worst}");
+    // Each channel carries its own level: R and B stay apart everywhere.
+    for p in 0..w * h {
+        let (r, b) = (s.data[p * 3], s.data[p * 3 + 2]);
+        assert!(r.abs_diff(b) >= 64 * 256 - 8 * 256, "channels collapsed at pixel {p}");
+    }
+}