M2: kanji that Adobe-Japan1 maps to a variation sequence draw in non-embedded fonts (#662)

Adobe-Japan1-UCS2 maps the JIS X 0208 forms of the kanji that
JIS X 0213:2004 redrew to a variation sequence (CID 1247 is 噂 U+E0100).
hayro only looked up single characters in a substitute font, so a
non-embedded Japanese font drew 噂, 逢, 溢, 祇, 箸 and over 200 other
common kanji as unrelated glyphs. A character with a variation selector,
from Adobe-Japan1-UCS2 or a /ToUnicode, now draws the substitute's glyph
for that form when its cmap has one, else its glyph for the character.

Co-authored-by: Claude Opus 5.5 <[email protected]>
Co-authored-by: Brandon Thomas <[email protected]>

crates/render/src/raster.rs

@@ -3512,6 +3512,62 @@ trailer << /Root 1 0 R >>
         }
     }
 
+    /// Adobe-Japan1 maps the JIS X 0208 forms of the kanji that JIS X 0213:2004 redrew (噂, 逢,
+    /// 溢, …) to a variation sequence: CID 1247 is 噂 U+E0100. A non-embedded Japanese font drew
+    /// such a CID as an unrelated glyph, since the substitute has no glyph for a sequence. It
+    /// draws the substitute's glyph for that form now, and its 噂 for a form it doesn't have.
+    #[test]
+    fn variation_sequences_draw_their_kanji_in_a_substitute() {
+        if pdfcraft_fonts::document_japanese_font().is_none() {
+            eprintln!("built without craft-fonts (CRAFT_FONTS_DIR unset): no Japanese face to check");
+            return;
+        }
+        let render = |base_font: &str, encoding: &str, to_unicode: &str, code: &str| {
+            let content = format!("BT /F1 40 Tf 5 15 Td <{code}> Tj ET");
+            let cmap = format!(
+                "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CMapName /Test def /CMapType 2 def 1 begincodespacerange <0000> <FFFF> endcodespacerange 1 beginbfchar <0001> <{to_unicode}> endbfchar endcmap CMapName currentdict /CMap defineresource pop end end"
+            );
+            // Code <0001> through `/ToUnicode` when the encoding is Identity-H; otherwise no
+            // `/ToUnicode`, so the CID goes through Adobe-Japan1-UCS2.
+            let to_unicode = if encoding == "Identity-H" { "/ToUnicode 8 0 R" } else { "" };
+            let pdf = format!(
+                "%PDF-1.7
+1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj
+2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj
+3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 60 60] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj
+4 0 obj << /Length {} >> stream
+{content}
+endstream endobj
+5 0 obj << /Type /Font /Subtype /Type0 /BaseFont /{base_font} /Encoding /{encoding} /DescendantFonts [6 0 R] {to_unicode} >> endobj
+6 0 obj << /Type /Font /Subtype /CIDFontType0 /BaseFont /{base_font} /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >>
+  /FontDescriptor 7 0 R /DW 1000 >> endobj
+7 0 obj << /Type /FontDescriptor /FontName /{base_font} /Flags 6 /FontBBox [0 -141 1000 859] /ItalicAngle 0 /Ascent 859 /Descent -141 /CapHeight 700 /StemV 80 >> endobj
+8 0 obj << /Length {} >> stream
+{cmap}
+endstream endobj
+trailer << /Root 1 0 R >>
+%%EOF",
+                content.len(),
+                cmap.len()
+            );
+            let mut r = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default());
+            let p = r.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 1.0, tag: 0 });
+            assert!(p.error.is_none(), "{base_font} <{code}>: {:?}", p.error);
+            p.rgba
+        };
+        let inked = |rgba: &[u8]| rgba.as_chunks::<4>().0.iter().filter(|c| c[0] < 128).count();
+        for base_font in ["HeiseiMin-W3", "HeiseiKakuGo-W5"] {
+            let collection = render(base_font, "UniJIS-UCS2-H", "", "5642");
+            let form = render(base_font, "Identity-H", "5642DB40DD00", "0001");
+            let character = render(base_font, "Identity-H", "5642", "0001");
+            let unknown_form = render(base_font, "Identity-H", "5642DB40DD05", "0001");
+            assert!(inked(&form) > 300, "{base_font}: 噂 U+E0100 is drawn ({} dark pixels)", inked(&form));
+            assert!(collection == form, "{base_font}: CID 1247 draws 噂 U+E0100");
+            assert!(form != character, "{base_font}: the form differs from the face's default 噂");
+            assert!(unknown_form == character, "{base_font}: a form the face lacks draws its 噂");
+        }
+    }
+
     /// An alpha soft mask whose transparency group has no /CS (as Chrome writes gradient text)
     /// must still mask. Regression test for the vendored hayro-interpret patch.
     #[test]

vendor/README.md

@@ -7,7 +7,7 @@ the fix. Vendoring copyleft code is never allowed (plan/adr/0001).
 
 | Crate | Version | Licence | Patches | Test |
 |---|---|---|---|---|
-| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index oLine truncated
+| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index oLine truncated
 
 | hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement` |
 | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap, page bitmap, halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one); (2) at most 2^26 symbol pixels per symbol dictionary (a fuzzed dictionary decoded for minutes); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer) | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`; (2) the nightly `cargo xtask fuzz` finding, which now finishes in about a second; (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate` |

vendor/hayro-interpret/src/font/cid.rs

@@ -14,6 +14,7 @@ use hayro_syntax::object::dict::keys::*;
 use hayro_syntax::object::{Array, Object};
 use kurbo::{BezPath, Vec2};
 use skrifa::attribute::Style;
+use skrifa::charmap::MapVariant;
 use skrifa::raw::collections::int_set::Domain;
 use skrifa::{FontRef, GlyphId, MetadataProvider};
 use std::collections::HashMap;
@@ -225,7 +226,7 @@ impl Type0Font {
     fn map_via_unicode(&self, key: u32) -> Option<GlyphId> {
         let to_unicode = self.to_unicode.as_ref()?;
 
-        let character = to_unicode
+        let (character, selector) = to_unicode
             .lookup_bf_string(key)
             .or_else(|| {
                 for len in 0..4 {
@@ -239,12 +240,32 @@ impl Type0Font {
                 None
             })
             .and_then(|bf| match bf {
-                BfString::Char(c) => Some(c),
-                BfString::String(_) => None,
+                BfString::Char(c) => Some((c, None)),
+                // PdfCraft patch: a character and a variation selector is that character in a
+                // given form, as Adobe-Japan1 maps the JIS X 0208 forms of the kanji that
+                // JIS X 0213:2004 redrew (噂 is U+5642 U+E0100). Other strings (ligatures) still
+                // have no glyph.
+                BfString::String(s) => {
+                    let mut chars = s.chars();
+                    let character = chars.next()?;
+                    let selector = chars.next().filter(|c| is_variation_selector(*c))?;
+                    chars.next().is_none().then_some((character, Some(selector)))
+                }
             })?;
 
         match &self.font_type {
-            FontType::OpenType(t) => t.font_ref().charmap().map(character),
+            FontType::OpenType(t) => {
+                let charmap = t.font_ref().charmap();
+                // PdfCraft patch: the face's glyph for that form when its `cmap` has one, else its
+                // glyph for the character, as when a form isn't supported (Unicode FAQ on
+                // variation sequences).
+                if let Some(selector) = selector
+                    && let Some(MapVariant::Variant(glyph)) = charmap.map_variant(character, selector)
+                {
+                    return Some(glyph);
+                }
+                charmap.map(character)
+            }
             FontType::Cff(c) => {
                 // Map codepoint to glyph name via AFL, and then look it up.
                 if let Some(name) = glyph_names::get_reverse(character)
@@ -527,6 +548,11 @@ impl CidToGIdMap {
     }
 }
 
+/// PdfCraft patch: Variation Selectors (U+FE00–U+FE0F) and their supplement (U+E0100–U+E01EF).
+fn is_variation_selector(c: char) -> bool {
+    matches!(c, '\u{FE00}'..='\u{FE0F}' | '\u{E0100}'..='\u{E01EF}')
+}
+
 /// PdfCraft patch: the largest CID (ISO 32000-2 §9.7.2; CIDs are at most two bytes).
 const MAX_CID: u32 = 65_535;