diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rs index 8dc50ea7..22f79c51 100644 --- a/crates/textimport/src/lib.rs +++ b/crates/textimport/src/lib.rs @@ -28,6 +28,33 @@ pub fn xlsx_records(bytes: &[u8], sheet: Option<&str>) -> Result use designcraft_doc::{CharAttrs, ParaAttrs, Story}; +/// Joins UTF-16 code units that arrive one at a time (RTF `\uN`, Tagged Text `<0xNNNN>`) into +/// characters. A character past U+FFFF comes as two units, a surrogate pair: the first half +/// waits for the second. +#[derive(Default)] +pub(crate) struct Utf16Units { + high: Option, +} + +impl Utf16Units { + /// The character that `unit` is or completes. Half a pair without its partner is no character. + pub(crate) fn push(&mut self, unit: u32) -> Option { + match (self.high.take(), unit) { + (Some(high), 0xDC00..=0xDFFF) => char::from_u32(0x10000 + ((high - 0xD800) << 10) + (unit - 0xDC00)), + (_, 0xD800..=0xDBFF) => { + self.high = Some(unit); + None + } + _ => char::from_u32(unit), + } + } + + /// Other text came in between: a waiting first half has no partner. + pub(crate) fn reset(&mut self) { + self.high = None; + } +} + /// A paragraph or character style found in the source. #[derive(Clone, Debug, Default, PartialEq)] pub struct ImportedStyle { diff --git a/crates/textimport/src/rtf.rs b/crates/textimport/src/rtf.rs index c02e5e55..9189b52d 100644 --- a/crates/textimport/src/rtf.rs +++ b/crates/textimport/src/rtf.rs @@ -70,6 +70,7 @@ pub fn import(bytes: &[u8]) -> Result { let mut st = Story::new(StoryId(0)); let mut stack: Vec = vec![State { uc: 1, ..Default::default() }]; let mut pending = 0usize; // characters still to skip after \uN + let mut units = crate::Utf16Units::default(); let mut buf = String::new(); let mut fmt = stack[0].format(); let mut warnings = Vec::new(); @@ -124,6 +125,7 @@ pub fn import(bytes: &[u8]) -> Result { if let Ok(b) = u8::from_str_radix(&hex, 16) && !stack.last().is_some_and(|s| s.skip) { + units.reset(); buf.push(cp1252(b)); } continue; @@ -225,8 +227,8 @@ pub fn import(bytes: &[u8]) -> Result { } "u" => { if let Some(n) = param { - let cp = if n < 0 { (n + 65536) as u32 } else { n as u32 }; - if let Some(ch) = char::from_u32(cp) { + let unit = if n < 0 { (n + 65536) as u32 } else { n as u32 }; + if let Some(ch) = units.push(unit) { buf.push(ch); } pending = stack.last().map_or(1, |s| s.uc); @@ -243,6 +245,7 @@ pub fn import(bytes: &[u8]) -> Result { continue; } if !stack.last().is_some_and(|s| s.skip) { + units.reset(); buf.push(c); } } @@ -287,4 +290,19 @@ Caf\'e9 \u8364? price\tab tab\line next\par assert_eq!(st.format_after(0).over.size, Some(12.0)); assert!(!st.text.contains("Times"), "font table skipped"); } + + /// Characters past U+FFFF are written as two `\uN` words, a UTF-16 surrogate pair (the form + /// our own exporter and Word write). Each half was converted alone and dropped. + #[test] + fn surrogate_pairs_become_one_character() { + let i = import(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}").unwrap(); + assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}"); + // What the exporter writes comes back. + let mut st = Story::new(StoryId(1)); + st.insert(0, "x \u{1F600} \u{20BB7}\u{7530} y"); + let doc = designcraft_doc::Document::new(&designcraft_doc::build::NewDocument::default()); + assert_eq!(import(crate::export::rtf(&doc, &st).as_bytes()).unwrap().story.text, st.text); + // Half a pair is not a character: it is still left out. + assert_eq!(import(br"{\rtf1 A\u-10179?B\u-8704?C}").unwrap().story.text, "ABC"); + } } diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rs index ca78bc4b..cfd08eea 100644 --- a/crates/textimport/src/tagged.rs +++ b/crates/textimport/src/tagged.rs @@ -110,12 +110,14 @@ pub fn import(bytes: &[u8]) -> Result { _ => runs.push((c.to_string(), fmt.clone())), } }; + let mut units = crate::Utf16Units::default(); let mut i = 0; let mut skipped = 0; while i < chars.len() { let c = chars[i]; match c { '\\' if i + 1 < chars.len() => { + units.reset(); push(&mut paras, chars[i + 1], &fmt); i += 2; } @@ -143,7 +145,7 @@ pub fn import(bytes: &[u8]) -> Result { match name { n if n.starts_with("0x") => { if let Ok(u) = u32::from_str_radix(&n[2..], 16) - && let Some(ch) = char::from_u32(u) + && let Some(ch) = units.push(u) { push(&mut paras, ch, &fmt); } @@ -193,6 +195,7 @@ pub fn import(bytes: &[u8]) -> Result { i += 1; } c => { + units.reset(); push(&mut paras, c, &fmt); i += 1; } @@ -226,6 +229,20 @@ pub fn import(bytes: &[u8]) -> Result { mod tests { use super::*; + /// Characters past U+FFFF are written as two tags, a UTF-16 surrogate pair. Each half was + /// converted alone and dropped. + #[test] + fn surrogate_pairs_become_one_character() { + let i = import("\r\n\r\nA<0xD83D><0xDE00>B <0xD842><0xDFB7><0x7530>".as_bytes()).unwrap(); + assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}"); + // What the exporter writes comes back. + let doc = Document::new(&designcraft_doc::build::NewDocument::default()); + let back = import(tagged_text(&doc, &i.story).as_bytes()).unwrap(); + assert_eq!(back.story.text, i.story.text); + // Half a pair is not a character: it is still left out. + assert_eq!(import("\r\nA<0xD83D>B<0xDE00>C".as_bytes()).unwrap().story.text, "ABC"); + } + #[test] fn tagged_text_round_trips() { let src = "\r\n\r\nCaf<0x00E9> \\<1\\>\r\nA word and big text";