feat(emoji): add Unicode 17 emoji and fix mixed skin tones (#2883)

packages/markdown_parser/rust/src/inline.rs

@@ -2,6 +2,7 @@
 
 use crate::ast::{EmojiKind, Node, ParserFlags, ParserResult};
 use crate::constants::{CODE_FENCE_LENGTH, MAX_INLINE_DEPTH, MAX_LINE_LENGTH};
+use crate::emoji::{EmojiContext, StandardEmoji};
 use crate::links;
 use crate::normalize::{
     combine_adjacent_text, compact_empty_text_nodes, flatten_top_level_formatting,
@@ -168,7 +169,13 @@ fn parse_inline_with_context(
 
         if let Some(result) = parse_regional_indicator_flag(remaining) {
             flush_accumulated_text(&mut nodes, &mut accumulated);
-            nodes.push(result.node);
+            push_regional_indicator_pair(
+                &mut nodes,
+                parser.emoji_context(),
+                base_offset + position,
+                &remaining[..result.advance],
+                result.node,
+            );
             position += result.advance;
             continue;
         }
@@ -349,6 +356,49 @@ fn parse_regional_indicator_flag(text: &str) -> Option<ParserResult> {
     })
 }
 
+fn push_regional_indicator_pair(
+    nodes: &mut Vec<Node>,
+    emoji_context: &EmojiContext,
+    offset: usize,
+    pair: &str,
+    flag: Node,
+) {
+    let (first, second) = pair.split_at(pair.len() / 2);
+    match emoji_context.standard_at(offset) {
+        Some(emoji) if emoji.raw == pair => nodes.push(standard_emoji_node(emoji)),
+        Some(emoji) if emoji.raw == first => {
+            nodes.push(standard_emoji_node(emoji));
+            match emoji_context.standard_at(offset + first.len()) {
+                Some(emoji) if emoji.raw == second => nodes.push(standard_emoji_node(emoji)),
+                _ => nodes.extend(regional_indicator_emoji(second)),
+            }
+        }
+        _ => nodes.push(flag),
+    }
+}
+
+fn standard_emoji_node(emoji: &StandardEmoji) -> Node {
+    Node::Emoji {
+        kind: EmojiKind::Standard {
+            raw: emoji.raw.clone(),
+            codepoints: emoji.codepoints.clone(),
+            name: emoji.name.clone(),
+        },
+    }
+}
+
+fn regional_indicator_emoji(text: &str) -> Option<Node> {
+    let ch = text.chars().next()?;
+    let letter = regional_indicator_letter(ch)?;
+    Some(Node::Emoji {
+        kind: EmojiKind::Standard {
+            raw: ch.to_string(),
+            codepoints: format!("{:x}", ch as u32),
+            name: format!("regional_indicator_{letter}"),
+        },
+    })
+}
+
 fn regional_indicator_letter(ch: char) -> Option<char> {
     let codepoint = ch as u32;
     if !(0x1f1e6..=0x1f1ff).contains(&codepoint) {

packages/markdown_parser/rust/tests/fixtures.rs

@@ -151,6 +151,67 @@ fn native_parser_fixtures_cover_typescript_suite_surface() {
     );
 }
 
+#[test]
+fn native_parser_pairs_regional_indicators_with_emoji_context() {
+    assert_eq!(
+        parse(
+            "🇦🇧🇸🇪",
+            0,
+            "S\t0\t4\t🇦\tregional_indicator_a\t1f1e6\nS\t4\t4\t🇧\tregional_indicator_b\t1f1e7\nS\t8\t8\t🇸🇪\tflag_se\t1f1f8-1f1ea\n"
+        ),
+        json!({"nodes":[
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦","codepoints":"1f1e6","name":"regional_indicator_a"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇧","codepoints":"1f1e7","name":"regional_indicator_b"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇸🇪","codepoints":"1f1f8-1f1ea","name":"flag_se"}}
+        ]})
+    );
+    assert_eq!(
+        parse(
+            "🇦🇧🇦🇸",
+            0,
+            "S\t0\t4\t🇦\tregional_indicator_a\t1f1e6\nS\t4\t4\t🇧\tregional_indicator_b\t1f1e7\nS\t8\t8\t🇦🇸\tflag_as\t1f1e6-1f1f8\n"
+        ),
+        json!({"nodes":[
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦","codepoints":"1f1e6","name":"regional_indicator_a"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇧","codepoints":"1f1e7","name":"regional_indicator_b"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦🇸","codepoints":"1f1e6-1f1f8","name":"flag_as"}}
+        ]})
+    );
+    assert_eq!(
+        parse("Sark 🇨🇶", 0, "S\t5\t8\t🇨🇶\tflag_sark\t1f1e8-1f1f6\n"),
+        json!({"nodes":[
+            {"type":"Text","content":"Sark "},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇨🇶","codepoints":"1f1e8-1f1f6","name":"flag_sark"}}
+        ]})
+    );
+    assert_eq!(
+        parse("x 🇦", 0, "S\t2\t4\t🇦\tregional_indicator_a\t1f1e6\n"),
+        json!({"nodes":[
+            {"type":"Text","content":"x "},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦","codepoints":"1f1e6","name":"regional_indicator_a"}}
+        ]})
+    );
+    assert_eq!(
+        parse(
+            "🇦🇧🇸🇪",
+            0,
+            "S\t0\t4\t🇦\tregional_indicator_a\t1f1e6\nS\t4\t8\t🇧🇸\tflag_bs\t1f1e7-1f1f8\nS\t12\t4\t🇪\tregional_indicator_e\t1f1ea\n"
+        ),
+        json!({"nodes":[
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦","codepoints":"1f1e6","name":"regional_indicator_a"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇧","codepoints":"1f1e7","name":"regional_indicator_b"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇸🇪","codepoints":"1f1f8-1f1ea","name":"flag_se"}}
+        ]})
+    );
+    assert_eq!(
+        parse("🇦🇧🇸🇪", 0, ""),
+        json!({"nodes":[
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇦🇧","codepoints":"1f1e6-1f1e7","name":"flag_ab"}},
+            {"type":"Emoji","kind":{"kind":"Standard","raw":"🇸🇪","codepoints":"1f1f8-1f1ea","name":"flag_se"}}
+        ]})
+    );
+}
+
 #[test]
 fn native_parser_allows_apostrophe_in_masked_link_destination() {
     let url = "https://docs.example.test/help/faq/#why-can't-this-link-parse%3F";