Import: RTF and Tagged Text keep characters past U+FFFF (#149)

Placing an RTF or Tagged Text file dropped every character outside the
Basic Multilingual Plane: emoji, mathematical letters and the rarer CJK
ideographs such as 𠮷 (U+20BB7, used in Japanese names). The text
around them came in, so the loss was silent. It also broke our own round
trip: Export Text writes these characters correctly and Place lost them.

Both formats carry such a character as a UTF-16 surrogate pair, two
units in a row: \u-10179?\u-8704? in RTF, <0xD83D><0xDE00> in Tagged
Text. Both readers passed each unit to char::from_u32 on its own. Half a
pair is not a character, so each half was dropped.

A small shared Utf16Units joins the units: a first half waits for the
second, and the pair becomes one character. Half a pair with no partner
is still left out, as before.

Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>

crates/textimport/src/lib.rs

@@ -28,6 +28,33 @@ pub fn xlsx_records(bytes: &[u8], sheet: Option<&str>) -> Result<Vec<Vec<String>
 
 use designcraft_doc::{CharAttrs, ParaAttrs, Story};
 
+/// Joins UTF-16 code units that arrive one at a time (RTF `\uN`, Tagged Text `<0xNNNN>`) into
+/// characters. A character past U+FFFF comes as two units, a surrogate pair: the first half
+/// waits for the second.
+#[derive(Default)]
+pub(crate) struct Utf16Units {
+    high: Option<u32>,
+}
+
+impl Utf16Units {
+    /// The character that `unit` is or completes. Half a pair without its partner is no character.
+    pub(crate) fn push(&mut self, unit: u32) -> Option<char> {
+        match (self.high.take(), unit) {
+            (Some(high), 0xDC00..=0xDFFF) => char::from_u32(0x10000 + ((high - 0xD800) << 10) + (unit - 0xDC00)),
+            (_, 0xD800..=0xDBFF) => {
+                self.high = Some(unit);
+                None
+            }
+            _ => char::from_u32(unit),
+        }
+    }
+
+    /// Other text came in between: a waiting first half has no partner.
+    pub(crate) fn reset(&mut self) {
+        self.high = None;
+    }
+}
+
 /// A paragraph or character style found in the source.
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct ImportedStyle {

crates/textimport/src/rtf.rs

@@ -70,6 +70,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
     let mut st = Story::new(StoryId(0));
     let mut stack: Vec<State> = vec![State { uc: 1, ..Default::default() }];
     let mut pending = 0usize; // characters still to skip after \uN
+    let mut units = crate::Utf16Units::default();
     let mut buf = String::new();
     let mut fmt = stack[0].format();
     let mut warnings = Vec::new();
@@ -124,6 +125,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
                     if let Ok(b) = u8::from_str_radix(&hex, 16)
                         && !stack.last().is_some_and(|s| s.skip)
                     {
+                        units.reset();
                         buf.push(cp1252(b));
                     }
                     continue;
@@ -225,8 +227,8 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
                     }
                     "u" => {
                         if let Some(n) = param {
-                            let cp = if n < 0 { (n + 65536) as u32 } else { n as u32 };
-                            if let Some(ch) = char::from_u32(cp) {
+                            let unit = if n < 0 { (n + 65536) as u32 } else { n as u32 };
+                            if let Some(ch) = units.push(unit) {
                                 buf.push(ch);
                             }
                             pending = stack.last().map_or(1, |s| s.uc);
@@ -243,6 +245,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
                     continue;
                 }
                 if !stack.last().is_some_and(|s| s.skip) {
+                    units.reset();
                     buf.push(c);
                 }
             }
@@ -287,4 +290,19 @@ Caf\'e9 \u8364? price\tab tab\line next\par
         assert_eq!(st.format_after(0).over.size, Some(12.0));
         assert!(!st.text.contains("Times"), "font table skipped");
     }
+
+    /// Characters past U+FFFF are written as two `\uN` words, a UTF-16 surrogate pair (the form
+    /// our own exporter and Word write). Each half was converted alone and dropped.
+    #[test]
+    fn surrogate_pairs_become_one_character() {
+        let i = import(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}").unwrap();
+        assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");
+        // What the exporter writes comes back.
+        let mut st = Story::new(StoryId(1));
+        st.insert(0, "x \u{1F600} \u{20BB7}\u{7530} y");
+        let doc = designcraft_doc::Document::new(&designcraft_doc::build::NewDocument::default());
+        assert_eq!(import(crate::export::rtf(&doc, &st).as_bytes()).unwrap().story.text, st.text);
+        // Half a pair is not a character: it is still left out.
+        assert_eq!(import(br"{\rtf1 A\u-10179?B\u-8704?C}").unwrap().story.text, "ABC");
+    }
 }

crates/textimport/src/tagged.rs

@@ -110,12 +110,14 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
             _ => runs.push((c.to_string(), fmt.clone())),
         }
     };
+    let mut units = crate::Utf16Units::default();
     let mut i = 0;
     let mut skipped = 0;
     while i < chars.len() {
         let c = chars[i];
         match c {
             '\\' if i + 1 < chars.len() => {
+                units.reset();
                 push(&mut paras, chars[i + 1], &fmt);
                 i += 2;
             }
@@ -143,7 +145,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
                 match name {
                     n if n.starts_with("0x") => {
                         if let Ok(u) = u32::from_str_radix(&n[2..], 16)
-                            && let Some(ch) = char::from_u32(u)
+                            && let Some(ch) = units.push(u)
                         {
                             push(&mut paras, ch, &fmt);
                         }
@@ -193,6 +195,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
                 i += 1;
             }
             c => {
+                units.reset();
                 push(&mut paras, c, &fmt);
                 i += 1;
             }
@@ -226,6 +229,20 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
 mod tests {
     use super::*;
 
+    /// Characters past U+FFFF are written as two tags, a UTF-16 surrogate pair. Each half was
+    /// converted alone and dropped.
+    #[test]
+    fn surrogate_pairs_become_one_character() {
+        let i = import("<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Body>A<0xD83D><0xDE00>B <0xD842><0xDFB7><0x7530>".as_bytes()).unwrap();
+        assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");
+        // What the exporter writes comes back.
+        let doc = Document::new(&designcraft_doc::build::NewDocument::default());
+        let back = import(tagged_text(&doc, &i.story).as_bytes()).unwrap();
+        assert_eq!(back.story.text, i.story.text);
+        // Half a pair is not a character: it is still left out.
+        assert_eq!(import("<ASCII-WIN>\r\n<ParaStyle:Body>A<0xD83D>B<0xDE00>C".as_bytes()).unwrap().story.text, "ABC");
+    }
+
     #[test]
     fn tagged_text_round_trips() {
         let src = "<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Head>Caf<0x00E9> \\<1\\>\r\n<ParaStyle:Body>A <CharStyle:Em>word<CharStyle:> and <cSize:18.000000>big<cSize:> text";