Fix hostile PSD pattern geometry (#743)

* Fix hostile PSD pattern geometry

Use checked rectangle extents and reject channel rectangles that differ from their parent tile, preventing panic-prone subtraction and huge zero-filled plane allocations. Bound aggregate decoded pattern data and PackBits expansion using parser limits and payload size. Add regression coverage for .pat, PSD global pattern blocks, and .abr embedded patterns.

Fixes #682
Fixes #683

Co-authored-by: Copilot <[email protected]>
Copilot-Session: e77cd34a-ce3e-4981-996f-4d37d49df90d

* Preserve bounded PSD pattern channel placement

Co-authored-by: Copilot <[email protected]>

Copilot-Session: e85c024b-ace5-4415-9208-b8d4e4684505

---------

Co-authored-by: Copilot <[email protected]>
Copilot-Session: e77cd34a-ce3e-4981-996f-4d37d49df90d
Copilot-Session: e85c024b-ace5-4415-9208-b8d4e4684505

crates/psd/README.md

@@ -107,6 +107,7 @@ testgen::{all_cases, merged_only, layered, small, sample_descriptor, pattern_pla
 | Layer tree (groups) | ✓ | ✓ | ✓ | open/closed folders plus bounding dividers; tolerant of malformed nesting |
 | RGBA8 extraction | ✓ | – | – | RGB / gray / CMYK layers; merged image also supports indexed / bitmap / duotone |
 | Builder | – | ✓ | – | RGB / gray / CMYK at 8/16 bit, masks, groups, blend modes, opacity, fill, visibility, clipping |
+| Patterns | ✓ | ✓ | ✓ | `Patt` / `Pat2` / `Pat3`, `.pat`, and `.abr` embedded patterns; non-empty tiles allow in-bounds channel subrectangles, decoded size is capped and PackBits expansion is input-bounded |
 
 ### Preserved raw, not yet typed (parsers planned for M8)
 

crates/psd/src/patterns.rs

@@ -20,12 +20,14 @@
 //! ag-psd (MIT) we take the first written arrays as the colour channels and the next one as
 //! transparency.
 
-use crate::compression::{Compression, PlaneLayout, decode_planes, encode_planes};
+use crate::compression::{Compression, MAX_DECODED_BYTES, PlaneLayout, decode_planes, encode_planes};
 use crate::error::{PsdError, Result};
 use crate::header::Version;
 
 /// Largest pattern edge accepted (Photoshop's own limit is far lower).
 const MAX_EDGE: u32 = 30_000;
+// A PackBits run packet emits at most 128 bytes from a 2-byte packet.
+const MAX_RLE_EXPANSION: u64 = 64;
 
 /// One pattern tile with planar, big-endian samples.
 #[derive(Debug, Clone, PartialEq)]
@@ -127,16 +129,19 @@ fn read_pattern(r: &mut Rd) -> Result<PsdPattern> {
     let body = r.take(len)?;
     let mut v = Rd { b: body, p: 0 };
     let [top, left, bottom, right] = v.rect()?;
-    let (w, h) = ((right - left).max(0) as u32, (bottom - top).max(0) as u32);
-    if w > MAX_EDGE || h > MAX_EDGE {
+    let w = right.checked_sub(left).ok_or(PsdError::LimitExceeded("pattern width overflow"))?;
+    let h = bottom.checked_sub(top).ok_or(PsdError::LimitExceeded("pattern height overflow"))?;
+    if w <= 0 || h <= 0 || w as u32 > MAX_EDGE || h as u32 > MAX_EDGE {
         return Err(PsdError::LimitExceeded("pattern size"));
     }
+    let (w, h) = (w as u32, h as u32);
     let count = v.u32()?;
     if count > 64 {
         return Err(PsdError::LimitExceeded("pattern channel count"));
     }
     let mut planes = Vec::new();
     let mut depth = 8u16;
+    let mut decoded_bytes = 0u64;
     for _ in 0..count + 2 {
         if v.p >= body.len() {
             break;
@@ -161,8 +166,22 @@ fn read_pattern(r: &mut Rd) -> Result<PsdPattern> {
             return Err(PsdError::invalid(format!("pattern depth {d}")));
         }
         depth = d;
-        let (cw, ch) = ((cr - cl).max(0) as usize, (cb - ct).max(0) as usize);
-        let layout = PlaneLayout { planes: 1, width: cw, height: ch, depth: d, version: Version::Psd };
+        if ct < top || cl < left || cb > bottom || cr > right || cb < ct || cr < cl {
+            return Err(PsdError::invalid("pattern channel rectangle falls outside pattern rectangle"));
+        }
+        let cw = usize::try_from(cr.checked_sub(cl).ok_or(PsdError::LimitExceeded("pattern channel width overflow"))?)
+            .map_err(|_| PsdError::invalid("negative pattern channel width"))?;
+        let ch = usize::try_from(cb.checked_sub(ct).ok_or(PsdError::LimitExceeded("pattern channel height overflow"))?)
+            .map_err(|_| PsdError::invalid("negative pattern channel height"))?;
+        let channel_layout = PlaneLayout { planes: 1, width: cw, height: ch, depth: d, version: Version::Psd };
+        let channel_bytes = channel_layout.total_bytes()?;
+        let layout = PlaneLayout { planes: 1, width: w as usize, height: h as usize, depth: d, version: Version::Psd };
+        let plane_bytes = layout.total_bytes()?;
+        decoded_bytes = decoded_bytes.checked_add(plane_bytes).ok_or(PsdError::LimitExceeded("pattern decoded size overflow"))?;
+        if decoded_bytes > MAX_DECODED_BYTES {
+            return Err(PsdError::LimitExceeded("pattern data exceeds MAX_DECODED_BYTES"));
+        }
+        let _ = layout.decoded_len()?;
         let compression = if comp == 1 {
             Compression::Rle
         } else if comp == 0 {
@@ -170,31 +189,26 @@ fn read_pattern(r: &mut Rd) -> Result<PsdPattern> {
         } else {
             Compression::Unknown(u16::from(comp))
         };
-        let plane = decode_planes(compression, data, &layout)?;
-        // Place a channel rect smaller than the pattern rect into a full plane.
-        let bpp = (usize::from(d) / 8).max(1);
-        let full = if d == 1 || (cw, ch) == (w as usize, h as usize) {
-            plane
-        } else {
-            let mut out = vec![0u8; w as usize * h as usize * bpp];
-            for y in 0..ch {
-                let ty = y as i32 + ct - top;
-                if ty < 0 || ty >= h as i32 {
-                    continue;
-                }
-                for x in 0..cw {
-                    let tx = x as i32 + cl - left;
-                    if tx < 0 || tx >= w as i32 {
-                        continue;
-                    }
-                    let s = (y * cw + x) * bpp;
-                    let o = (ty as usize * w as usize + tx as usize) * bpp;
-                    out[o..o + bpp].copy_from_slice(&plane[s..s + bpp]);
-                }
-            }
-            out
+        let max_expansion = match compression {
+            Compression::Raw => Some(1),
+            Compression::Rle => Some(MAX_RLE_EXPANSION),
+            Compression::Zip | Compression::ZipPrediction | Compression::Unknown(_) => None,
         };
-        planes.push(full);
+        if let Some(max_expansion) = max_expansion {
+            let expansion_limit = u64::try_from(data.len())
+                .map_err(|_| PsdError::LimitExceeded("pattern channel input exceeds address space"))?
+                .checked_mul(max_expansion)
+                .ok_or(PsdError::LimitExceeded("pattern channel input size overflow"))?;
+            if channel_bytes > expansion_limit {
+                return Err(PsdError::LimitExceeded("pattern channel exceeds its input-derived size limit"));
+            }
+        }
+        let plane = decode_planes(compression, data, &channel_layout)?;
+        let x = usize::try_from(cl.checked_sub(left).ok_or(PsdError::LimitExceeded("pattern channel x offset overflow"))?)
+            .map_err(|_| PsdError::invalid("negative pattern channel x offset"))?;
+        let y = usize::try_from(ct.checked_sub(top).ok_or(PsdError::LimitExceeded("pattern channel y offset overflow"))?)
+            .map_err(|_| PsdError::invalid("negative pattern channel y offset"))?;
+        planes.push(place_channel_plane(&plane, &channel_layout, x, y, &layout)?);
     }
     let nc = mode_channels(mode);
     if planes.len() < nc {
@@ -205,6 +219,67 @@ fn read_pattern(r: &mut Rd) -> Result<PsdPattern> {
     Ok(PsdPattern { mode, width: w, height: h, name, id, palette, depth, channels: planes, alpha })
 }
 
+fn place_channel_plane(plane: &[u8], source: &PlaneLayout, x: usize, y: usize, destination: &PlaneLayout) -> Result<Vec<u8>> {
+    if source.planes != 1 || destination.planes != 1 || source.depth != destination.depth {
+        return Err(PsdError::invalid("incompatible pattern channel layouts"));
+    }
+    let x_end = x.checked_add(source.width).ok_or(PsdError::LimitExceeded("pattern channel x offset overflow"))?;
+    let y_end = y.checked_add(source.height).ok_or(PsdError::LimitExceeded("pattern channel y offset overflow"))?;
+    if x_end > destination.width || y_end > destination.height {
+        return Err(PsdError::invalid("pattern channel rectangle falls outside pattern rectangle"));
+    }
+    let source_len = source.decoded_len()?;
+    if plane.len() != source_len {
+        return Err(PsdError::invalid("pattern channel payload does not match its rectangle"));
+    }
+    let mut out = vec![0; destination.decoded_len()?];
+    if source.depth == 1 {
+        let source_row_bytes = source.row_bytes();
+        let destination_row_bytes = destination.row_bytes();
+        for row in 0..source.height {
+            for col in 0..source.width {
+                let source_pos = row
+                    .checked_mul(source_row_bytes)
+                    .and_then(|v| v.checked_add(col / 8))
+                    .ok_or(PsdError::LimitExceeded("pattern channel source offset overflow"))?;
+                let destination_col = x.checked_add(col).ok_or(PsdError::LimitExceeded("pattern channel destination offset overflow"))?;
+                let destination_pos = y
+                    .checked_add(row)
+                    .and_then(|v| v.checked_mul(destination_row_bytes))
+                    .and_then(|v| v.checked_add(destination_col / 8))
+                    .ok_or(PsdError::LimitExceeded("pattern channel destination offset overflow"))?;
+                let source_bit = 7 - col % 8;
+                let destination_bit = 7 - destination_col % 8;
+                let source_byte = plane.get(source_pos).ok_or(PsdError::invalid("pattern channel payload does not match its rectangle"))?;
+                let value = (*source_byte >> source_bit) & 1;
+                let dst = out.get_mut(destination_pos).ok_or(PsdError::invalid("pattern channel placement exceeds its output plane"))?;
+                *dst |= value << destination_bit;
+            }
+        }
+    } else {
+        let bytes_per_sample = usize::from(source.depth / 8);
+        let source_row_bytes = source.row_bytes();
+        let destination_row_bytes = destination.row_bytes();
+        let row_bytes = source.width.checked_mul(bytes_per_sample).ok_or(PsdError::LimitExceeded("pattern channel row size overflow"))?;
+        let x_bytes = x.checked_mul(bytes_per_sample).ok_or(PsdError::LimitExceeded("pattern channel x offset overflow"))?;
+        for row in 0..source.height {
+            let source_start = row.checked_mul(source_row_bytes).ok_or(PsdError::LimitExceeded("pattern channel source offset overflow"))?;
+            let destination_start = y
+                .checked_add(row)
+                .and_then(|v| v.checked_mul(destination_row_bytes))
+                .and_then(|v| v.checked_add(x_bytes))
+                .ok_or(PsdError::LimitExceeded("pattern channel destination offset overflow"))?;
+            let source_end = source_start.checked_add(row_bytes).ok_or(PsdError::LimitExceeded("pattern channel source offset overflow"))?;
+            let destination_end = destination_start.checked_add(row_bytes).ok_or(PsdError::LimitExceeded("pattern channel destination offset overflow"))?;
+            let source_row = plane.get(source_start..source_end).ok_or(PsdError::invalid("pattern channel payload does not match its rectangle"))?;
+            let destination_row =
+                out.get_mut(destination_start..destination_end).ok_or(PsdError::invalid("pattern channel placement exceeds its output plane"))?;
+            destination_row.copy_from_slice(source_row);
+        }
+    }
+    Ok(out)
+}
+
 fn write_pattern(p: &PsdPattern, out: &mut Vec<u8>) -> Result<()> {
     if p.width > MAX_EDGE || p.height > MAX_EDGE || p.width > i16::MAX as u32 || p.height > i16::MAX as u32 {
         return Err(PsdError::LimitExceeded("pattern size"));
@@ -381,4 +456,128 @@ mod tests {
         assert!(parse_pat_file(b"8BPS\0\x01").is_err());
         assert!(parse_pattern_block(&[]).unwrap().is_empty());
     }
+
+    fn pattern_vma_rect_offset(pattern_start: usize, p: &PsdPattern) -> usize {
+        let name_units = p.name.encode_utf16().count() + 1;
+        pattern_start + 16 + name_units * 2 + 1 + p.id.len() + if p.mode == 2 { 768 } else { 0 } + 8
+    }
+
+    fn set_i32(bytes: &mut [u8], offset: usize, value: i32) {
+        bytes[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
+    }
+
+    #[test]
+    fn hostile_pattern_rectangles_are_rejected_through_all_import_routes() {
+        let p = pat(8, false);
+
+        // A subtraction of the hostile endpoints overflowed before rectangle validation.
+        let mut global = write_pattern_block(std::slice::from_ref(&p)).unwrap();
+        let vma = pattern_vma_rect_offset(4, &p);
+        set_i32(&mut global, vma + 4, i32::MIN);
+        set_i32(&mut global, vma + 12, i32::MAX);
+        assert!(parse_pattern_block(&global).is_err());
+
+        let mut standalone = write_pat_file(std::slice::from_ref(&p)).unwrap();
+        let vma = pattern_vma_rect_offset(8, &p);
+        set_i32(&mut standalone, vma + 4, i32::MIN);
+        set_i32(&mut standalone, vma + 12, i32::MAX);
+        assert!(parse_pat_file(&standalone).is_err());
+
+        let mut abr = crate::abr::write_v6(
+            1,
+            &[crate::abr::AbrSample { id: String::new(), width: 1, height: 1, depth: 8, data: vec![0] }],
+            std::slice::from_ref(&p),
+            &[],
+            true,
+        )
+        .unwrap();
+        let section = abr.windows(4).position(|w| w == b"patt").unwrap();
+        let vma = pattern_vma_rect_offset(section + 8, &p) + 4;
+        set_i32(&mut abr, vma + 4, i32::MIN);
+        set_i32(&mut abr, vma + 12, i32::MAX);
+        let parsed = crate::abr::parse(&abr).unwrap();
+        assert!(parsed.patterns.is_empty());
+        assert!(parsed.warnings.iter().any(|warning| warning.contains("embedded patterns unreadable")));
+    }
+
+    #[test]
+    fn huge_pattern_rect_with_tiny_channel_payload_is_rejected() {
+        let p = pat(32, false);
+        let mut global = write_pattern_block(std::slice::from_ref(&p)).unwrap();
+        let vma = pattern_vma_rect_offset(4, &p);
+        set_i32(&mut global, vma + 8, 30_000);
+        set_i32(&mut global, vma + 12, 30_000);
+        assert!(parse_pattern_block(&global).is_err());
+
+        let mut standalone = write_pat_file(std::slice::from_ref(&p)).unwrap();
+        let vma = pattern_vma_rect_offset(8, &p);
+        set_i32(&mut standalone, vma + 8, 30_000);
+        set_i32(&mut standalone, vma + 12, 30_000);
+        assert!(parse_pat_file(&standalone).is_err());
+
+        let mut abr = crate::abr::write_v6(
+            1,
+            &[crate::abr::AbrSample { id: String::new(), width: 1, height: 1, depth: 8, data: vec![0] }],
+            std::slice::from_ref(&p),
+            &[],
+            true,
+        )
+        .unwrap();
+        let section = abr.windows(4).position(|w| w == b"patt").unwrap();
+        let vma = pattern_vma_rect_offset(section + 8, &p) + 4;
+        set_i32(&mut abr, vma + 8, 30_000);
+        set_i32(&mut abr, vma + 12, 30_000);
+        let parsed = crate::abr::parse(&abr).unwrap();
+        assert!(parsed.patterns.is_empty());
+        assert!(parsed.warnings.iter().any(|warning| warning.contains("embedded patterns unreadable")));
+    }
+
+    #[test]
+    fn offset_channel_rectangles_are_placed_inside_the_pattern() {
+        for depth in [8, 16, 32] {
+            let original = pat(depth, true);
+            let mut block = write_pattern_block(std::slice::from_ref(&original)).unwrap();
+            let vma = pattern_vma_rect_offset(4, &original);
+            set_i32(&mut block, vma, -2);
+            set_i32(&mut block, vma + 4, -3);
+            set_i32(&mut block, vma + 8, 4);
+            set_i32(&mut block, vma + 12, 5);
+
+            let parsed = parse_pattern_block(&block).unwrap();
+            let parsed = parsed.first().unwrap();
+            assert_eq!((parsed.width, parsed.height), (8, 6));
+            let bytes_per_sample = usize::from(depth / 8);
+            for (actual, expected) in parsed.channels.iter().zip(&original.channels).chain(parsed.alpha.iter().zip(original.alpha.iter())) {
+                for y in 0..6usize {
+                    for x in 0..8usize {
+                        let at = (y * 8 + x) * bytes_per_sample;
+                        let actual_sample = actual.get(at..at + bytes_per_sample).unwrap();
+                        if (2..5).contains(&y) && (3..8).contains(&x) {
+                            let source_at = ((y - 2) * 5 + (x - 3)) * bytes_per_sample;
+                            assert_eq!(actual_sample, expected.get(source_at..source_at + bytes_per_sample).unwrap());
+                        } else {
+                            assert!(actual_sample.iter().all(|&sample| sample == 0));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn one_bit_offset_channel_placement_preserves_msb_first_pixels() {
+        let source = PlaneLayout { planes: 1, width: 5, height: 2, depth: 1, version: Version::Psd };
+        let destination = PlaneLayout { planes: 1, width: 8, height: 4, depth: 1, version: Version::Psd };
+        let placed = place_channel_plane(&[0b1010_1000, 0b0101_0000], &source, 2, 1, &destination).unwrap();
+        assert_eq!(placed, [0, 0b0010_1010, 0b0001_0100, 0]);
+    }
+
+    #[test]
+    fn zero_size_pattern_rectangles_are_rejected() {
+        let p = pat(8, false);
+        let mut block = write_pattern_block(std::slice::from_ref(&p)).unwrap();
+        let vma = pattern_vma_rect_offset(4, &p);
+        set_i32(&mut block, vma + 8, 0);
+        assert!(parse_pattern_block(&block).is_err());
+    }
 }