Never crash: predictor rows and stroke widths from the input can't exhaust memory

The second full nightly fuzz run on main found two more:
- hayro-syntax computed a Flate/LZW predictor row as columns × bits in
  wrapping arithmetic: /Columns 9223372036854775807 asked for 2^61 bytes and
  aborted on any machine. Row widths are now saturating usize (colors × bits
  also overflowed u8), a zero width is refused (it divided by zero), and a
  row longer than the data is not allocated (empty output, as before).
- hayro stroked /LW 9223372036854775807 as given, and stroke expansion
  allocated 10 GB. Widths are clamped to 4 × (canvas width + height) device
  pixels, which covers the same pixels.
Both findings now run in 8 MB. Synthetic regression test in render.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/render/src/raster.rs

@@ -821,6 +821,53 @@ trailer << /Root 1 0 R >>
         page(&format!("q 20 0 0 20 5 5 cm /Im1 Do Q {red}"), fax);
     }
 
+    /// From the nightly `cargo xtask fuzz`: a FlateDecode predictor with `/Columns
+    /// 9223372036854775807` wrapped to a 2^61-byte row allocation (an abort on any machine), and
+    /// a line width of 9223372036854775807 made stroke expansion allocate 10 GB. Vendored hayro
+    /// patches: saturating predictor rows refused when longer than the data, and stroke widths
+    /// clamped to a few canvases. The page still renders, with the huge stroke covering it.
+    #[test]
+    fn absurd_predictor_columns_and_line_widths_render() {
+        let render = |streams: &[&str]| {
+            let mut objs = String::new();
+            let mut refs = Vec::new();
+            for (i, s) in streams.iter().enumerate() {
+                let n = 4 + i;
+                refs.push(format!("{n} 0 R"));
+                objs.push_str(&format!("{n} 0 obj {s} endobj\n"));
+            }
+            let pdf = format!(
+                "%PDF-1.7
+1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj
+2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj
+3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 40 40] /Contents [{}] >> endobj
+{objs}trailer << /Root 1 0 R >>
+%%EOF",
+                refs.join(" ")
+            );
+            let (tx, rx) = std::sync::mpsc::channel();
+            std::thread::spawn(move || {
+                let mut r = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default());
+                let _ = tx.send(r.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 1.0, tag: 0 }));
+            });
+            let page = rx.recv_timeout(std::time::Duration::from_secs(20)).expect("must not stall the renderer");
+            assert!(page.error.is_none(), "{:?}", page.error);
+            page
+        };
+        let plain = |content: &str| format!("<< /Length {} >> stream\n{content}\nendstream", content.len());
+        // Stored (uncompressed) zlib data, so the stream needs no encoder: header, one final
+        // stored block of two bytes, checksum.
+        let predicted = "<< /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns 9223372036854775807 >> /Length 13 >> stream\nx\u{1}\u{1}\u{2}\u{0}\u{fd}\u{ff}\u{0}\u{0}\u{0}\u{1}\u{0}\u{1}\nendstream";
+        let page = render(&[predicted, &plain("1 0 0 rg 0 0 4 4 re f")]);
+        assert_eq!(&page.rgba[((38 * 40 + 1) * 4)..][..4], &[255, 0, 0, 255], "the other content stream draws");
+        let page = render(&[&plain("1 0 0 RG 9223372036854775807 w 10 20 m 30 20 l S")]);
+        // Butt caps: the stroke covers the band over the segment (x 10 to 30) top to bottom.
+        for (x, y) in [(20, 0), (20, 39), (12, 0), (28, 39)] {
+            assert_eq!(&page.rgba[((y * 40 + x) * 4)..][..4], &[255, 0, 0, 255], "({x}, {y}) under the huge stroke");
+        }
+        assert_eq!(&page.rgba[((20 * 40 + 2) * 4)..][..4], &[255, 255, 255, 255], "beyond the butt cap");
+    }
+
     /// From `cargo xtask fuzz`: an inline image claiming /W 4294967295 over four bytes of data
     /// hung in resampling (vendored hayro patch: `MAX_IMAGE_PIXELS`).
     #[test]

vendor/README.md

@@ -9,8 +9,8 @@ the fix. Vendoring copyleft code is never allowed (plan/adr/0001).
 |---|---|---|---|---|
 | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding` |
 
-| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB) | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step` |
+| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion) | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render` |
 | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows); (2) predictor row widths use saturating arithmetic | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating` |
-| hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB) | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding` |
+| hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB); (3) Flate/LZW predictor row widths are saturating `usize`, a zero width is refused and a row longer than the data is not allocated (a fuzzed `/Columns 9223372036854775807` wrapped to a 2^61-byte allocation and aborted) | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (3) `raster::tests::absurd_predictor_columns_and_line_widths_render` |
 
 Temporary: hayro is the bootstrap renderer and lopdf the bootstrap inspector (ADR-0004); M2.6 replaces them with our own devices and `cos`/`model`.

vendor/hayro-syntax/src/filter/lzw_flate.rs

@@ -716,12 +716,12 @@ struct PredictorParams {
 }
 
 impl PredictorParams {
-    fn bits_per_pixel(&self) -> u8 {
-        self.bits_per_component * self.colors
-    }
-
     fn row_length_in_bytes(&self) -> usize {
-        (self.columns * self.bits_per_pixel() as usize).div_ceil(8)
+        // PrintCraft patch: saturating, and in usize (colors × bits overflowed u8): a fuzzed
+        // `/Columns 9223372036854775807` wrapped to a 2^61-byte row.
+        self.columns
+            .saturating_mul(usize::from(self.bits_per_component) * usize::from(self.colors))
+            .div_ceil(8)
     }
 }
 
@@ -759,11 +759,21 @@ fn apply_predictor(data: Vec<u8>, params: &PredictorParams) -> Option<Vec<u8>> {
 
             let total_row_len = if is_png_predictor {
                 // + 1 Because each row must start with the predictor that is used for PNG predictors.
-                row_len + 1
+                row_len.saturating_add(1)
             } else {
                 row_len
             };
 
+            // PrintCraft patch: a zero-width row divided by zero below, and a row longer than the
+            // data was allocated before finding there is nothing to read (no rows: empty output,
+            // as before).
+            if row_len == 0 {
+                return None;
+            }
+            if total_row_len > data.len() {
+                return Some(Vec::new());
+            }
+
             let num_rows = data.len() / total_row_len;
 
             if !matches!(params.bits_per_component, 1 | 2 | 4 | 8 | 16) {
@@ -775,7 +785,8 @@ fn apply_predictor(data: Vec<u8>, params: &PredictorParams) -> Option<Vec<u8>> {
             let (bit_size, chunk_len) = if is_png_predictor {
                 (
                     8,
-                    (params.colors * params.bits_per_component).div_ceil(8) as usize,
+                    // PrintCraft patch: in usize (the u8 product overflowed).
+                    (usize::from(params.colors) * usize::from(params.bits_per_component)).div_ceil(8),
                 )
             } else {
                 (params.bits_per_component, params.colors as usize)

vendor/hayro/src/renderer.rs

@@ -77,6 +77,14 @@ impl Renderer {
             line_width *= threshold;
         }
 
+        // PrintCraft patch: a stroke far wider than the canvas looks the same as one a few canvases
+        // wide, but its geometry grows with the width: a fuzzed `/LW 9223372036854775807`
+        // allocated 10 GB in stroke expansion.
+        let widest = 4.0 * (f32::from(self.ctx.width()) + f32::from(self.ctx.height()));
+        if min_factor.is_finite() && min_factor > 0.0 && line_width * min_factor > widest {
+            line_width = widest / min_factor;
+        }
+
         let stroke = kurbo::Stroke {
             width: line_width as f64,
             join: stroke_props.line_join,