Refs #347: encode the full WinAnsi 0x80-0x9F range (Š š Ž ž Œ œ Ÿ ...) (#546)

pdfcraft_fonts::win_ansi, edit's is_win_ansi_char and the xtask demo
encoder each carried a hand-written subset of WinAnsiEncoding's 0x80-0x9F
row (quotes, dashes, euro, trademark) and left out Š š Ž ž Œ œ Ÿ ƒ † ‡ ˆ
˜ ‰ ‹ ›. Text editing therefore treated š/ž as non-WinAnsi and sent them
to the Type3 fallback font (or refused them without craft-fonts), and
every other generator (forms, annotations, OCR layer, redaction overlay,
signatures, create) wrote them as '?'.

win_ansi now goes through a new win_ansi_byte, which reads the 0x80-0x9F
codes from the same ISO 32000-2 Annex D table the text extractor decodes
with, and is_win_ansi_char reuses it. xtask doesn't depend on
pdfcraft-fonts, so its demo encoder gets the full row with a pointer to
the shared table. č/ć are not in WinAnsi and still use the fallback.

Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/edit/src/tests.rs

@@ -718,6 +718,22 @@ fn editing_text_keeps_the_tokens_a_stream_shares_with_its_neighbours() {
     assert_split_tokens_kept(&reopen(&doc), "Rewrapped");
 }
 
+#[test]
+fn win_ansi_high_glyphs_use_helvetica_not_the_fallback_font() {
+    // š ž Š Ž Œ Ÿ ƒ † ‰ (WinAnsi 0x80–0x9F) are standard-font glyphs (#347): no Type3 fallback.
+    let text = "Šumava žaba œuvre Ÿ ƒ † ‰ €";
+    let mut doc = text_page("BT /F2 10 Tf 72 600 Td (ab) Tj ET");
+    let r = text::replace_line(&mut doc, 0, 0, text).unwrap();
+    assert_eq!(r.substituted.as_deref(), Some("Helvetica"));
+    let lines = text::text_lines(&reopen(&doc), 0).unwrap();
+    assert_eq!((lines[0].text.as_str(), lines[0].base_font.as_str()), (text, "Helvetica"));
+    let mut doc = text_page("BT /F2 10 Tf 72 600 Td (ab) Tj ET");
+    let r = text::replace_block(&mut doc, 0, 0, text).unwrap();
+    assert!(r.substituted.as_deref().is_none_or(|s| !s.contains("Type3")), "{:?}", r.substituted);
+    let lines = text::text_lines(&reopen(&doc), 0).unwrap();
+    assert_eq!(lines.iter().map(|l| l.text.as_str()).collect::<Vec<_>>().join(" "), text);
+}
+
 #[test]
 fn missing_glyphs_substitute_helvetica_and_impossible_text_is_refused() {
     let mut doc = text_page("BT /F2 10 Tf 72 600 Td (ab) Tj ET");

crates/edit/src/text.rs

@@ -499,7 +499,7 @@ pub fn text_lines(doc: &Document, page: usize) -> Result<Vec<TextLine>, EditErro
 type Encoder = Box<dyn Fn(&str) -> Option<Vec<u8>>>;
 
 fn is_win_ansi_char(c: char) -> bool {
-    matches!(c, '\u{20}'..='\u{7e}' | '\u{a0}'..='\u{ff}' | '€' | '‚' | '„' | '…' | '‘' | '’' | '“' | '”' | '•' | '–' | '—' | '™' | '\t')
+    c == '\t' || pdfcraft_fonts::win_ansi_byte(c).is_some()
 }
 
 fn needs_type3(text: &str) -> bool {

crates/fonts/src/lib.rs

@@ -66,28 +66,25 @@ pub fn wrap(text: &str, size: f64, width: f64) -> Vec<String> {
     lines
 }
 
-/// Encode text in WinAnsiEncoding (ISO 32000-2 Annex D); unmappable characters become `?`.
+/// The WinAnsiEncoding (ISO 32000-2 Annex D) code for `c`, if the encoding has one.
+///
+/// Codes 0x80–0x9F come from the same table the text extractor decodes with, so encoding and
+/// decoding agree (Š š Ž ž Œ œ Ÿ ƒ † ‡ ˆ ˜ ‰ ‹ › as well as quotes, dashes, € and ™).
+pub fn win_ansi_byte(c: char) -> Option<u8> {
+    match c {
+        '\u{20}'..='\u{7e}' | '\u{a0}'..='\u{ff}' => u8::try_from(u32::from(c)).ok(),
+        _ => {
+            let high = encodings::WIN_ANSI.get(0x80..0xa0)?;
+            let i = high.iter().position(|&u| u != 0 && u == u32::from(c))?;
+            u8::try_from(0x80 + i).ok()
+        }
+    }
+}
+
+/// Encode text in WinAnsiEncoding (ISO 32000-2 Annex D); tabs become spaces and unmappable
+/// characters become `?`.
 pub fn win_ansi(s: &str) -> Vec<u8> {
-    s.chars()
-        .map(|c| match c {
-            '\u{20}'..='\u{7e}' => c as u8,
-            '\u{a0}'..='\u{ff}' => c as u32 as u8,
-            '€' => 0x80,
-            '‚' => 0x82,
-            '„' => 0x84,
-            '…' => 0x85,
-            '‘' => 0x91,
-            '’' => 0x92,
-            '“' => 0x93,
-            '”' => 0x94,
-            '•' => 0x95,
-            '–' => 0x96,
-            '—' => 0x97,
-            '™' => 0x99,
-            '\t' => b' ',
-            _ => b'?',
-        })
-        .collect()
+    s.chars().map(|c| if c == '\t' { b' ' } else { win_ansi_byte(c).unwrap_or(b'?') }).collect()
 }
 
 /// Bytes as a PDF literal string, `(` … `)`, with delimiters escaped.
@@ -122,4 +119,21 @@ mod tests {
         assert_eq!(win_ansi("Café — 5€ ☃"), b"Caf\xe9 \x97 5\x80 ?");
         assert_eq!(literal(b"a(b)\\c"), b"(a\\(b\\)\\\\c)");
     }
+
+    #[test]
+    fn win_ansi_covers_the_0x80_to_0x9f_glyphs() {
+        // Every character in the WinAnsiEncoding table encodes to its code and back (#347).
+        for (code, &u) in encodings::WIN_ANSI.iter().enumerate() {
+            let Some(c) = char::from_u32(u).filter(|_| u != 0) else { continue };
+            assert_eq!(win_ansi_byte(c), Some(code as u8), "{c:?} (U+{u:04X})");
+            assert_eq!(win_ansi(&c.to_string()), [code as u8]);
+        }
+        assert_eq!(
+            win_ansi("Šta? žaba, œuvre — „ok“ ‰ ƒ † ‡ ˆ ˜ ‹›Ÿ"),
+            b"\x8ata? \x9eaba, \x9cuvre \x97 \x84ok\x93 \x89 \x83 \x86 \x87 \x88 \x98 \x8b\x9b\x9f"
+        );
+        // Not in WinAnsi: č ć still need the fallback font; undefined codes are never produced.
+        assert_eq!(win_ansi("čć\u{81}\u{8d}\t"), b"???? ");
+        assert_eq!(win_ansi_byte('\0'), None);
+    }
 }

xtask/src/demo_pdf/pdf.rs

@@ -351,20 +351,41 @@ pub fn pdf_string(s: &str) -> String {
 }
 
 /// Map text to WinAnsiEncoding bytes (unmappable characters become `?`).
+///
+/// Mirrors `pdfcraft_fonts::win_ansi` (ISO 32000-2 Annex D), which xtask doesn't depend on;
+/// keep the 0x80–0x9F rows in step with `crates/fonts/src/encodings.rs`.
 pub fn winansi(s: &str) -> Vec<u8> {
     s.chars()
         .map(|c| match c {
             '\u{20}'..='\u{7e}' => c as u8,
             '\u{a0}'..='\u{ff}' => c as u32 as u8,
-            '\u{2022}' => 0x95,
-            '\u{2013}' => 0x96,
-            '\u{2014}' => 0x97,
+            '\u{20ac}' => 0x80,
+            '\u{201a}' => 0x82,
+            '\u{0192}' => 0x83,
+            '\u{201e}' => 0x84,
+            '\u{2026}' => 0x85,
+            '\u{2020}' => 0x86,
+            '\u{2021}' => 0x87,
+            '\u{02c6}' => 0x88,
+            '\u{2030}' => 0x89,
+            '\u{0160}' => 0x8a,
+            '\u{2039}' => 0x8b,
+            '\u{0152}' => 0x8c,
+            '\u{017d}' => 0x8e,
             '\u{2018}' => 0x91,
             '\u{2019}' => 0x92,
             '\u{201c}' => 0x93,
             '\u{201d}' => 0x94,
-            '\u{2026}' => 0x85,
-            '\u{20ac}' => 0x80,
+            '\u{2022}' => 0x95,
+            '\u{2013}' => 0x96,
+            '\u{2014}' => 0x97,
+            '\u{02dc}' => 0x98,
+            '\u{2122}' => 0x99,
+            '\u{0161}' => 0x9a,
+            '\u{203a}' => 0x9b,
+            '\u{0153}' => 0x9c,
+            '\u{017e}' => 0x9e,
+            '\u{0178}' => 0x9f,
             _ => b'?',
         })
         .collect()
@@ -415,4 +436,13 @@ mod tests {
         assert_eq!(pdf_string("a(b)\\"), "(a\\(b\\)\\\\)");
         assert_eq!(pdf_string("—"), "(\\227)");
     }
+
+    #[test]
+    fn winansi_covers_the_0x80_to_0x9f_glyphs() {
+        // The 27 defined codes of ISO 32000-2 Annex D WinAnsiEncoding between 0x80 and 0x9F.
+        let high = "€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ";
+        let want: Vec<u8> = (0x80..=0x9f).filter(|b| ![0x81, 0x8d, 0x8f, 0x90, 0x9d].contains(b)).collect();
+        assert_eq!(winansi(high), want);
+        assert_eq!(winansi("č"), b"?");
+    }
 }