CJK 1.1: the character language reaches the shaper

The character's language (as IDML names it) maps to a BCP 47 tag
(designcraft_doc::language_tag) and is the shaping context's language
on the harfrust buffer, so OpenType locl applies for every language in
the list (Japanese, Chinese, Korean, Turkish, ...), not only Arabic,
Persian, Urdu, Hebrew and English: other names used to reach the
shaper as they were written, which it doesn't know, so their locl never
applied. designcraft_fonts no longer maps names itself. The word-shaping
cache is keyed by the tag. Chinese: Simplified and Chinese: Traditional
join the language list (plain Chinese stays, tagged zh); Simplified
Chinese takes curly quotes, Traditional Chinese corner brackets.

ROADMAP.md

@@ -87,7 +87,7 @@ DesignCraft aims at full Adobe InDesign parity — and to be better: faster, ope
 - Edit ▸ Menus (`window.hideMenuItem`): hide menu items; menus with hidden items end in Show All Menu Items.
 - View ▸ Separations Preview (`view.separations`): one process plate as ink density, or areas over an ink limit in red (a GCR preview of the rendered view; spot plates and the flattener preview aren't there).
 - Separations Preview plates now come from each object's own colour (CMYK as authored; RGB and images through the colour settings, images via a cached 17³ table), so pure black shows only on the black plate.
-- Language (Character panel ▸ Language, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, and typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, CJK 「…」). Hyphenation dictionaries for other languages aren't bundled yet.
+- Language (Character panel ▸ Language, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, Japanese and Traditional Chinese 「…」, Simplified Chinese “…”), and the font's localized forms (OpenType `locl`: Turkish i, Bulgarian Cyrillic, Japanese / Chinese / Korean glyph variants) follow it too. Hyphenation dictionaries for other languages aren't bundled yet.
 - Books (`book.new/open/add/remove/list/styleSource/paginate/syncStyles/exportPdf`, Book panel): `.dcbook` files, continuous page numbering, style sync from a style source, one merged PDF.
 - Track Changes (`changes.track/list/acceptAll/rejectAll`, Type ▸ Track Changes): typing is marked added (highlighted on screen), deletions are kept but hidden from layout and exports until accepted or rejected. Per-change accept/reject and authors aren't there yet.
 - Track Changes: per-change Accept / Reject (`changes.accept`, `changes.reject`) and a Track Changes panel listing every change.

crates/compose/src/shape.rs

@@ -614,10 +614,10 @@ fn shape_run_raw(
 }
 
 type WordMap = HashMap<Box<str>, Arc<[ShapedGlyph]>>;
-type WordKey = (u32, bool, String);
+type WordKey = (u32, bool, String, Option<&'static str>);
 
-/// Shaped words (font units) by (face, caps, features), then text. Lookups take read locks and
-/// misses are shaped outside any lock, so parallel composition doesn't serialize here.
+/// Shaped words (font units) by (face, caps, features, language), then text. Lookups take read
+/// locks and misses are shaped outside any lock, so parallel composition doesn't serialize here.
 static WORD_CACHE: RwLock<Vec<(WordKey, Arc<RwLock<WordMap>>)>> = RwLock::new(Vec::new());
 const WORD_CACHE_MAX: usize = 50_000;
 
@@ -640,12 +640,13 @@ fn word_map(key: WordKey) -> Arc<RwLock<WordMap>> {
 /// Shape `src` word by word through a cache (like a browser's word cache): the text is split
 /// after each U+0020 so repeated words are shaped once. Shaping does not cross word spaces in the
 /// scripts we lay out, and clusters stay byte offsets into `src`.
-fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], caps: bool, context: designcraft_fonts::ShapeContext<'_>) -> Vec<ShapedGlyph> {
+fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], lang: Option<&'static str>, caps: bool, around: (&str, &str)) -> Vec<ShapedGlyph> {
     let map = |c: char| if caps { c.to_uppercase().next().unwrap_or(c) } else { c };
     if src.len() < 2 || !src.contains(' ') || src.chars().any(designcraft_fonts::is_rtl) {
-        return designcraft_fonts::shape_with_context(face, src, feats, map, context);
+        let (before, after) = around;
+        return designcraft_fonts::shape_with_context(face, src, feats, map, designcraft_fonts::ShapeContext { before, after, language: lang });
     }
-    let words = word_map((face.id(), caps, format!("{feats:?}/{}", context.language)));
+    let words = word_map((face.id(), caps, format!("{feats:?}"), lang));
     let pieces: Vec<&str> = src.split_inclusive(' ').collect();
     let mut found: Vec<Option<Arc<[ShapedGlyph]>>> = {
         let r = words.read().unwrap_or_else(|e| e.into_inner());
@@ -659,7 +660,7 @@ fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], caps: bool, conte
                 pieces[k],
                 feats,
                 map,
-                designcraft_fonts::ShapeContext { language: context.language, ..Default::default() },
+                designcraft_fonts::ShapeContext { language: lang, ..Default::default() },
             )
             .into();
             fresh.push((pieces[k].into(), g.clone()));
@@ -780,12 +781,11 @@ fn shape_segment(
         feats.extend(["vert", "vrt2"].iter().filter_map(|t| designcraft_fonts::feature(t)));
     }
     let space = face.advance(face.glyph_for(' ')) * k * hs;
-    let context = designcraft_fonts::ShapeContext {
-        before: if replacement.is_none() { text.get(..range.start).unwrap_or("") } else { "" },
-        after: if replacement.is_none() { text.get(range.end..).unwrap_or("") } else { "" },
-        language: &p.language,
-    };
-    let shaped: Vec<ShapedGlyph> = shape_cached(face, src, &feats, caps, context);
+    // The language picks the font's localized forms (`locl`); the text around the run lets
+    // letters join across a change of style.
+    let lang = designcraft_doc::language_tag(&p.language);
+    let around = if replacement.is_none() { (text.get(..range.start).unwrap_or(""), text.get(range.end..).unwrap_or("")) } else { ("", "") };
+    let shaped: Vec<ShapedGlyph> = shape_cached(face, src, &feats, lang, caps, around);
     let n = shaped.len();
     let fref = FaceRef::of(face);
     for (gi, sg) in shaped.iter().enumerate() {

crates/compose/src/tests.rs

@@ -1428,3 +1428,43 @@ fn arabic_character_overrides_mirror_brackets_using_final_levels() {
         }
     }
 }
+
+#[test]
+fn language_picks_localized_forms_through_the_word_cache() {
+    // Source Serif 4 has a Turkish `locl` form of `i`; the same words in English and Turkish
+    // (both shaped word by word through the cache) keep their own forms.
+    let text = "in in in in";
+    let (mut d, sid, _) = doc_with(text, Rect::new(36.0, 36.0, 500.0, 200.0), ParaAttrs::default());
+    d.story_mut(sid).unwrap().format_chars(6..text.len(), |f| f.over.language = Some("Turkish".into()));
+    let cs = compose_story(&d, sid, &ComposeOptions::default());
+    let gid_at = |byte: usize| all_lines(&cs).iter().flat_map(|l| l.glyphs.iter()).find(|g| g.byte == byte).map(|g| g.gid).unwrap();
+    assert_eq!(gid_at(0), gid_at(3), "English i");
+    assert_eq!(gid_at(6), gid_at(9), "Turkish i");
+    assert_ne!(gid_at(0), gid_at(6), "Turkish i takes its localized form");
+    // A single word is shaped directly, in its language too.
+    let (mut d, sid, _) = doc_with("i", Rect::new(36.0, 36.0, 500.0, 200.0), ParaAttrs::default());
+    d.story_mut(sid).unwrap().format_chars(0..1, |f| f.over.language = Some("Turkish".into()));
+    let cs = compose_story(&d, sid, &ComposeOptions::default());
+    assert_eq!(all_lines(&cs)[0].glyphs[0].gid, gid_at(6), "Turkish i, shaped on its own");
+}
+
+#[test]
+fn chinese_and_japanese_take_their_localized_forms() {
+    // Noto Sans CJK SC (craft-fonts) draws 直 differently for Japanese (`locl`).
+    let Some(noto) = designcraft_fonts::CRAFT_FONTS.iter().find(|f| f.family == "Noto Sans CJK SC") else { return };
+    FontDb::global().add_font(noto.bytes.to_vec());
+    let gid = |language: &str| {
+        let (mut d, sid, _) = doc_with("直", Rect::new(36.0, 36.0, 300.0, 100.0), ParaAttrs::default());
+        d.story_mut(sid).unwrap().format_chars(0.."直".len(), |f| {
+            f.over.font_family = Some(noto.family.into());
+            f.over.language = Some(language.into());
+        });
+        let cs = compose_story(&d, sid, &ComposeOptions::default());
+        let g = &all_lines(&cs)[0].glyphs[0];
+        assert_eq!(g.face.family, noto.family);
+        g.gid
+    };
+    let chinese = gid("Chinese: Simplified");
+    assert_eq!(chinese, FontDb::global().face(noto.family, noto.style).glyph_for('直'));
+    assert_ne!(gid("Japanese"), chinese, "Japanese 直");
+}

crates/doc/src/attrs.rs

@@ -190,6 +190,65 @@ impl Digits {
     }
 }
 
+/// The BCP 47 tag of a language name as InDesign and IDML write it ("English: USA", "Chinese:
+/// Simplified"), for the shaper's localized forms (`locl`). A variant we don't know falls back to
+/// its language ("English: Australian" → `en`); an unknown language (or "[No Language]") has none.
+pub fn language_tag(language: &str) -> Option<&'static str> {
+    let l = language.trim().to_ascii_lowercase();
+    let variant = match l.as_str() {
+        "english: usa" => Some("en-US"),
+        "english: uk" => Some("en-GB"),
+        "english: canadian" => Some("en-CA"),
+        "german: swiss 2006 reform" | "german: swiss" => Some("de-CH"),
+        "french: canadian" => Some("fr-CA"),
+        "portuguese: brazilian" => Some("pt-BR"),
+        "norwegian: bokmål" | "norwegian: bokmal" => Some("nb"),
+        "norwegian: nynorsk" => Some("nn"),
+        "chinese: simplified" => Some("zh-Hans"),
+        "chinese: traditional" => Some("zh-Hant"),
+        _ => None,
+    };
+    if variant.is_some() {
+        return variant;
+    }
+    Some(match l.split(':').next().unwrap_or("").trim() {
+        "english" => "en",
+        "german" => "de",
+        "french" => "fr",
+        "spanish" => "es",
+        "catalan" => "ca",
+        "italian" => "it",
+        "portuguese" => "pt",
+        "dutch" => "nl",
+        "danish" => "da",
+        "swedish" => "sv",
+        "norwegian" => "no",
+        "finnish" => "fi",
+        "polish" => "pl",
+        "czech" => "cs",
+        "slovak" => "sk",
+        "hungarian" => "hu",
+        "romanian" => "ro",
+        "croatian" => "hr",
+        "lithuanian" => "lt",
+        "greek" => "el",
+        "russian" => "ru",
+        "ukrainian" => "uk",
+        "bulgarian" => "bg",
+        "serbian" => "sr",
+        "macedonian" => "mk",
+        "turkish" => "tr",
+        "japanese" => "ja",
+        "korean" => "ko",
+        "chinese" => "zh",
+        "arabic" => "ar",
+        "persian" | "farsi" => "fa",
+        "urdu" => "ur",
+        "hebrew" => "he",
+        _ => return None,
+    })
+}
+
 /// A tracked change on text.
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -701,6 +760,26 @@ mod tests {
         assert_eq!(Digits::Hindi.map('x', ""), 'x');
     }
 
+    #[test]
+    fn language_names_map_to_bcp47_tags() {
+        assert_eq!(language_tag("English: USA"), Some("en-US"));
+        assert_eq!(language_tag("English: Australian"), Some("en"));
+        assert_eq!(language_tag("Turkish"), Some("tr"));
+        assert_eq!(language_tag("Japanese"), Some("ja"));
+        assert_eq!(language_tag("Korean"), Some("ko"));
+        assert_eq!(language_tag("Chinese"), Some("zh"));
+        assert_eq!(language_tag("Chinese: Simplified"), Some("zh-Hans"));
+        assert_eq!(language_tag("Chinese: Traditional"), Some("zh-Hant"));
+        assert_eq!(language_tag("Arabic"), Some("ar"));
+        assert_eq!(language_tag("Persian"), Some("fa"));
+        assert_eq!(language_tag("Farsi"), Some("fa"));
+        assert_eq!(language_tag("Urdu"), Some("ur"));
+        assert_eq!(language_tag("Hebrew"), Some("he"));
+        assert_eq!(language_tag("[No Language]"), None);
+        assert_eq!(language_tag("Klingon"), None);
+        assert_eq!(language_tag(""), None);
+    }
+
     #[test]
     fn merge_and_resolve() {
         let mut a = CharAttrs { size: Some(10.0), ..Default::default() };

crates/engine/src/cmd/text.rs

@@ -499,6 +499,8 @@ pub fn quote_marks(language: &str) -> [char; 4] {
         ['\u{201E}', '\u{201D}', '\u{201A}', '\u{2019}']
     } else if l.starts_with("swedish") || l.starts_with("finnish") {
         ['\u{201D}', '\u{201D}', '\u{2019}', '\u{2019}']
+    } else if l.starts_with("chinese: simplified") {
+        ['\u{201C}', '\u{201D}', '\u{2018}', '\u{2019}']
     } else if l.starts_with("japanese") || l.starts_with("chinese") {
         ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']
     } else if l.starts_with("danish") {
@@ -1275,6 +1277,18 @@ mod language_tests {
         let st = s.execute("spelling.check", &json!({"story": sid})).unwrap();
         assert!(st.as_array().unwrap().is_empty(), "{st}");
     }
+
+    #[test]
+    fn cjk_languages_have_their_quotes_and_no_english_rules() {
+        use super::quote_marks;
+        assert_eq!(quote_marks("Chinese: Simplified"), ['\u{201C}', '\u{201D}', '\u{2018}', '\u{2019}']);
+        assert_eq!(quote_marks("Chinese: Traditional"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);
+        assert_eq!(quote_marks("Chinese"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);
+        assert_eq!(quote_marks("Japanese"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);
+        for l in ["Japanese", "Korean", "Chinese", "Chinese: Simplified", "Chinese: Traditional"] {
+            assert!(!designcraft_compose::is_english(l), "{l}: no English hyphenation or spelling");
+        }
+    }
 }
 
 #[cfg(test)]

crates/fonts/src/lib.rs

@@ -11,7 +11,7 @@ mod fontdb;
 
 pub use fontdb::{FALLBACK_FAMILY, FaceRef, FontDb, FontFace, base_style, bundled, system_font_dirs};
 pub use harfrust::Feature;
-use harfrust::{Direction, ShapeOptions, Tag, UnicodeBuffer};
+use harfrust::{Direction, Language, ShapeOptions, Tag, UnicodeBuffer};
 pub use kurbo::BezPath;
 use skrifa::MetadataProvider;
 use skrifa::instance::Size;
@@ -113,23 +113,14 @@ fn direction_runs(text: &str) -> Vec<(std::ops::Range<usize>, bool, unicode_scri
     runs
 }
 
+/// What surrounds a run being shaped: the text before and after it (so Arabic letters join across a
+/// change of style) and its language, a BCP 47 tag such as `tr` or `zh-Hant` that picks the font's
+/// localized forms (OpenType `locl`; `None` or a tag the shaper can't read: the font's defaults).
 #[derive(Clone, Copy, Default)]
 pub struct ShapeContext<'a> {
     pub before: &'a str,
     pub after: &'a str,
-    pub language: &'a str,
-}
-
-/// Normalize the supported application language names while also accepting BCP 47 tags.
-pub fn language_tag(language: &str) -> &str {
-    match language.split(':').next().unwrap_or(language).trim() {
-        "Arabic" => "ar",
-        "Persian" | "Farsi" => "fa",
-        "Urdu" => "ur",
-        "Hebrew" => "he",
-        "English" => "en",
-        _ => language,
-    }
+    pub language: Option<&'a str>,
 }
 
 /// Shape `text` with `face`. `chars` lets callers substitute characters (e.g. uppercase for All
@@ -140,6 +131,7 @@ pub fn shape(face: &FontFace, text: &str, features: &[Feature], map: impl Fn(cha
     shape_with_context(face, text, features, map, ShapeContext::default())
 }
 
+/// [`shape`] `text` in its `context`: its language and the text around it.
 pub fn shape_with_context(
     face: &FontFace,
     text: &str,
@@ -196,7 +188,7 @@ fn shape_dir(
         if let Ok(script) = script.short_name().parse() {
             buf.set_script(script);
         }
-        if let Ok(language) = language_tag(context.language).parse() {
+        if let Some(language) = context.language.and_then(|t| t.parse::<Language>().ok()) {
             buf.set_language(language);
         }
         buf.set_flags(harfrust::BufferFlags::PRODUCE_SAFE_TO_INSERT_TATWEEL);
@@ -254,10 +246,6 @@ mod tests {
         assert_eq!(runs.len(), 2);
         assert_eq!(runs[0], (0..5, true, Script::Hebrew));
         assert_eq!(runs[1], (5..text.len(), true, Script::Arabic));
-        assert_eq!(language_tag("Arabic"), "ar");
-        assert_eq!(language_tag("Farsi"), "fa");
-        assert_eq!(language_tag("Urdu"), "ur");
-        assert_eq!(language_tag("ar-SA"), "ar-SA");
     }
 
     #[test]
@@ -297,6 +285,17 @@ mod tests {
         assert_eq!(off.len(), 6);
     }
 
+    #[test]
+    fn language_picks_localized_forms() {
+        // Source Serif 4's `locl` has a Turkish `i`.
+        let face = FontDb::global().face(DEFAULT_FAMILY, "Regular");
+        let gid = |language| shape_with_context(&face, "i", &[], |c| c, ShapeContext { language, ..Default::default() })[0].gid;
+        assert_eq!(gid(None), face.glyph_for('i'));
+        assert_eq!(gid(Some("en-US")), face.glyph_for('i'));
+        assert_ne!(gid(Some("tr")), face.glyph_for('i'));
+        assert_eq!(gid(Some("not a tag")), face.glyph_for('i'));
+    }
+
     #[test]
     fn mapping_keeps_clusters() {
         let face = FontDb::global().face(DEFAULT_FAMILY, "Regular");

crates/ui-egui/src/panels/properties.rs

@@ -1206,6 +1206,8 @@ const LANGUAGES: &[&str] = &[
     "Turkish",
     "Japanese",
     "Chinese",
+    "Chinese: Simplified",
+    "Chinese: Traditional",
     "Korean",
     "Arabic",
     "Hebrew",

docs/cjk.md

@@ -47,9 +47,8 @@ presets or tables are copied. Our default mojikumi and kinsoku tables are built
 Known bugs to fix first:
 
 - Korean breaks between every syllable (it should break at spaces by default).
-- Only Arabic, Persian, Urdu, Hebrew and English language names reach the shaper as languages
-  it knows, so `locl` never applies for the others (Japanese, Chinese, Korean, Turkish…); Chinese
-  and Korean text get Japanese glyph forms and the Japanese fallback font.
+- Fallback is language-blind: Chinese and Korean text without a covering font get the Japanese
+  fallback font (and so Japanese glyph forms, whatever their language's `locl` asks for).
 - Vertical advances are the horizontal ones and upright glyphs turn about a guessed centre
   (`adv × 0.38`), so proportional and non-square glyphs sit wrong in vertical text.
 - A justified CJK line has no stretch except Single Word Justification.