feat(emoji): add Unicode 17 emoji and fix mixed skin tones (#2883)

fluxer_app/src/features/expressions/utils/EmojiCodepointUtils.ts

@@ -1,8 +1,11 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+const EYE_IN_SPEECH_BUBBLE = '\u{1F441}\u200D\u{1F5E8}';
+
 export function convertToCodePoints(emoji: string): string {
-	const containsZWJ = emoji.includes('\u200D');
-	const processedEmoji = containsZWJ ? emoji : emoji.replace(/\uFE0F/g, '');
+	const emojiWithoutFE0F = emoji.replace(/\uFE0F/g, '');
+	const keepsFE0F = emoji.includes('\u200D') && emojiWithoutFE0F !== EYE_IN_SPEECH_BUBBLE;
+	const processedEmoji = keepsFE0F ? emoji : emojiWithoutFE0F;
 	return Array.from(processedEmoji)
 		.map((char) => char.codePointAt(0)?.toString(16).replace(/^0+/, '') || '')
 		.join('-');

fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts

@@ -2,6 +2,7 @@
 
 import type {UnicodeEmoji} from '@app/features/emoji/types/EmojiTypes';
 import * as EmojiUtils from '@app/features/expressions/utils/EmojiUtils';
+import type {EmojiSurrogateMatch} from '@app/features/messaging/utils/markdown/parser/EmojiParsers';
 import * as RegexUtils from '@app/features/messaging/utils/RegexUtils';
 import emojiData from '@app/media/data/emojis.json';
 import {SKIN_TONE_SURROGATES} from '@fluxer/constants/src/EmojiConstants';
@@ -60,6 +61,15 @@ const categories = Object.freeze(Object.keys(emojiData.categories));
 
 const toCanonicalSurrogate = (surrogate: string): string => surrogate.replace(/️/g, '');
 
+const EYE_IN_SPEECH_BUBBLE_FULLY_QUALIFIED = '\u{1F441}\uFE0F\u200D\u{1F5E8}\uFE0F';
+const HAIR_COMPONENT_NAMES: Record<string, string> = {
+	'\u{1F9B0}': 'red_hair',
+	'\u{1F9B1}': 'curly_hair',
+	'\u{1F9B3}': 'white_hair',
+	'\u{1F9B2}': 'bald',
+};
+const REGIONAL_INDICATOR_PAIR_PATTERN = '\\uD83C[\\uDDE6-\\uDDFF]\\uD83C[\\uDDE6-\\uDDFF]';
+
 let defaultSkinTone: string = '';
 
 class UnicodeEmojiClass {
@@ -216,6 +226,11 @@ function buildEmojiIndex(): EmojiIndex {
 				surrogateToName[skinToneEntry.surrogatePair] = skinToneEntry.name;
 				canonicalSurrogateToName[toCanonicalSurrogate(skinToneEntry.surrogatePair)] = skinToneEntry.name;
 			});
+			const skins = (emojiObject as {skins?: ReadonlyArray<{names: ReadonlyArray<string>; surrogates: string}>}).skins;
+			skins?.slice(SKIN_TONE_SURROGATES.length).forEach((skin) => {
+				surrogateToName[skin.surrogates] = skin.names[0];
+				canonicalSurrogateToName[toCanonicalSurrogate(skin.surrogates)] = skin.names[0];
+			});
 			categoryByEmojiName[emoji.uniqueName] = category;
 			emojis.push(emojiJson);
 			return emojiJson;
@@ -228,6 +243,20 @@ function buildEmojiIndex(): EmojiIndex {
 		canonicalSurrogateToName[toCanonicalSurrogate(surrogatePair)] = `skin-tone-${index + 1}`;
 	});
 
+	Object.entries(HAIR_COMPONENT_NAMES).forEach(([surrogate, name]) => {
+		surrogateToName[surrogate] = name;
+		canonicalSurrogateToName[surrogate] = name;
+	});
+
+	Object.keys(surrogateToName)
+		.filter((surrogate) => surrogate.includes('\u20E3'))
+		.map(toCanonicalSurrogate)
+		.concat(EYE_IN_SPEECH_BUBBLE_FULLY_QUALIFIED)
+		.forEach((alias) => {
+			const name = canonicalSurrogateToName[toCanonicalSurrogate(alias)];
+			if (name) surrogateToName[alias] = name;
+		});
+
 	const keywordOwner: Record<string, string> = {};
 	const keywordCount: Record<string, number> = {};
 
@@ -275,7 +304,7 @@ function buildEmojiIndex(): EmojiIndex {
 		emojis,
 		skinToneSpriteCount,
 		baseSpriteCount,
-		emojiSurrogateRegex: new RegExp(`(${surrogateAlternation})`, 'g'),
+		emojiSurrogateRegex: new RegExp(`(${REGIONAL_INDICATOR_PAIR_PATTERN}|${surrogateAlternation})`, 'g'),
 		emojiShortcutRegex: new RegExp(`^(${shortcutAlternation})`),
 	};
 }
@@ -288,6 +317,50 @@ function getEmojiIndex(): EmojiIndex {
 const lookupSurrogateName = (surrogate: string): string | null =>
 	getEmojiIndex().canonicalSurrogateToName[toCanonicalSurrogate(surrogate)] ?? null;
 
+const isRegionalIndicatorAt = (text: string, index: number): boolean => {
+	const lowSurrogate = text.charCodeAt(index + 1);
+	return text.charCodeAt(index) === 0xd83c && lowSurrogate >= 0xdde6 && lowSurrogate <= 0xddff;
+};
+
+const isInsideRegionalIndicatorPair = (text: string, index: number): boolean => {
+	if (!isRegionalIndicatorAt(text, index)) return false;
+	let runStart = index;
+	while (isRegionalIndicatorAt(text, runStart - 2)) runStart -= 2;
+	return (index - runStart) % 4 === 2;
+};
+
+const toEmojiSurrogateMatch = (text: string, start: number, end: number): EmojiSurrogateMatch => ({
+	start,
+	end,
+	name: lookupSurrogateName(text.slice(start, end)),
+});
+
+function* matchEmojiSurrogates(text: string, startIndex = 0): Generator<EmojiSurrogateMatch, void> {
+	const regex = getEmojiIndex().emojiSurrogateRegex;
+	let index = startIndex;
+	if (isInsideRegionalIndicatorPair(text, index)) {
+		yield toEmojiSurrogateMatch(text, index, index + 2);
+		index += 2;
+	}
+	while (true) {
+		regex.lastIndex = index;
+		const match = regex.exec(text);
+		if (match == null) return;
+		index = match.index + match[0].length;
+		const emojiMatch = toEmojiSurrogateMatch(text, match.index, index);
+		const isUnknownRegionalIndicatorPair =
+			emojiMatch.name == null &&
+			isRegionalIndicatorAt(text, match.index) &&
+			isRegionalIndicatorAt(text, match.index + 2);
+		if (isUnknownRegionalIndicatorPair) {
+			yield toEmojiSurrogateMatch(text, match.index, match.index + 2);
+			yield toEmojiSurrogateMatch(text, match.index + 2, index);
+		} else {
+			yield emojiMatch;
+		}
+	}
+}
+
 const EMOJI_SHORTCODE_RE = /^:([^\s:]+(?:::skin-tone-[0-9])?):/;
 const categoryIcons = {
 	people: SmileyIcon,
@@ -377,6 +450,8 @@ export default {
 	getSurrogateName: (surrogate: string): string | null => {
 		return lookupSurrogateName(surrogate);
 	},
+	matchEmojiSurrogates,
+	isInsideRegionalIndicatorPair,
 	findEmojiByName: (emojiName: string): UnicodeEmoji | null => {
 		return getEmojiIndex().nameToEmoji[emojiName] || null;
 	},
@@ -400,8 +475,5 @@ export default {
 	get EMOTICON_PREFIX_RE(): RegExp {
 		return getEmojiIndex().emojiShortcutRegex;
 	},
-	get EMOJI_SURROGATE_RE(): RegExp {
-		return getEmojiIndex().emojiSurrogateRegex;
-	},
 	EMOJI_SPRITES,
 };

fluxer_app/src/features/lexical/composer/ComposerEmojiShortcode.ts

@@ -29,17 +29,19 @@ export function registerComposerEmojiShortcode(editor: LexicalEditor, resolve: C
 }
 
 function findUnicodeEmoji(text: string, startIndex: number): TypedEmojiMatch | null {
-	const pattern = new RegExp(UnicodeEmojis.EMOJI_SURROGATE_RE.source, 'g');
-	pattern.lastIndex = startIndex;
-	const match = pattern.exec(text);
-	if (match == null) {
+	const matches = UnicodeEmojis.matchEmojiSurrogates(text, startIndex);
+	let match = matches.next().value;
+	if (match && UnicodeEmojis.isInsideRegionalIndicatorPair(text, match.end)) {
+		match = matches.next().value;
+	}
+	if (!match) {
 		return null;
 	}
-	const name = UnicodeEmojis.nameForSurrogate(match[0], false);
+	const name = match.name;
 	if (!name) {
 		return null;
 	}
-	return {start: match.index, end: match.index + match[0].length, name};
+	return {start: match.start, end: match.end, name};
 }
 
 interface EmojiToken extends TypedEmojiMatch {

fluxer_app/src/features/messaging/utils/markdown/EmojiProviderSetup.ts

@@ -5,7 +5,7 @@ import {type EmojiProvider, setEmojiParserConfig} from '@app/features/messaging/
 import {SKIN_TONE_SURROGATES} from '@fluxer/constants/src/EmojiConstants';
 
 const emojiProvider: EmojiProvider = {
-	getSurrogateName: UnicodeEmojis.getSurrogateName,
+	matchEmojiSurrogates: UnicodeEmojis.matchEmojiSurrogates,
 	findEmojiByName: UnicodeEmojis.findEmojiByShortcodeName,
 	findEmojiWithSkinTone: UnicodeEmojis.findEmojiWithSkinTone,
 };
@@ -13,9 +13,6 @@ const emojiProvider: EmojiProvider = {
 export function initializeEmojiParser(): void {
 	setEmojiParserConfig({
 		emojiProvider,
-		get emojiRegex() {
-			return UnicodeEmojis.EMOJI_SURROGATE_RE;
-		},
 		skinToneSurrogates: SKIN_TONE_SURROGATES,
 	});
 }

fluxer_app/src/features/messaging/utils/markdown/parser/EmojiParsers.ts

@@ -4,17 +4,21 @@ export interface UnicodeEmoji {
 	surrogates: string;
 }
 
+export interface EmojiSurrogateMatch {
+	start: number;
+	end: number;
+	name: string | null;
+}
+
 export interface EmojiProvider {
-	getSurrogateName(surrogate: string): string | null;
+	matchEmojiSurrogates(text: string): Iterable<EmojiSurrogateMatch>;
 	findEmojiByName(name: string): UnicodeEmoji | null;
 	findEmojiWithSkinTone(baseName: string, skinToneSurrogate: string): UnicodeEmoji | null;
 }
 
 export interface EmojiParserConfig {
 	emojiProvider?: EmojiProvider;
-	emojiRegex?: RegExp;
 	skinToneSurrogates?: ReadonlyArray<string>;
-	convertToCodePoints?: (emoji: string) => string;
 }
 
 let globalEmojiConfig: EmojiParserConfig | null = null;

fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {convertToCodePoints} from '@app/features/expressions/utils/EmojiCodepointUtils';
 import {flattenAST} from '@app/features/messaging/utils/markdown/parser/AstUtils';
 import {getEmojiParserConfig} from '@app/features/messaging/utils/markdown/parser/EmojiParsers';
 import {MARKDOWN_PARSER_WASM_BASE64} from '@app/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes';
@@ -202,14 +203,6 @@ class Utf8OffsetTracker {
 	}
 }
 
-function defaultCodepoints(emoji: string): string {
-	const containsZwJ = emoji.includes('‍');
-	const processed = containsZwJ ? emoji : emoji.replace(/️/g, '');
-	return Array.from(processed)
-		.map((char) => char.codePointAt(0)?.toString(16).replace(/^0+/, '') || '')
-		.join('-');
-}
-
 const PLAINTEXT_SYMBOLS = new Set(['™', '™️', '©', '©️', '®', '®️']);
 const SPECIAL_SHORTCODES: Record<string, string> = {
 	tm: '™',
@@ -226,29 +219,23 @@ function buildEmojiContext(input: string): string {
 	const provider = config?.emojiProvider;
 	let context = '';
 	if (!provider) return context;
-	const convertToCodePoints = config.convertToCodePoints || defaultCodepoints;
-	const emojiRegex = config.emojiRegex;
-	if (emojiRegex) {
-		const offsetTracker = new Utf8OffsetTracker();
-		emojiRegex.lastIndex = 0;
-		let match: RegExpExecArray | null;
-		while ((match = emojiRegex.exec(input)) !== null) {
-			const candidate = match[0];
-			if (!candidate || PLAINTEXT_SYMBOLS.has(candidate)) continue;
-			const name = provider.getSurrogateName(candidate);
-			if (!name) continue;
-			const candidateBytes = textEncoder.encode(candidate).byteLength;
-			const byteOffset = offsetTracker.offsetFor(input, match.index);
-			context += appendContextLine([
-				'S',
-				String(byteOffset),
-				String(candidateBytes),
-				candidate,
-				name,
-				convertToCodePoints(candidate),
-			]);
-			offsetTracker.advance(candidate.length, candidateBytes);
-		}
+	const offsetTracker = new Utf8OffsetTracker();
+	for (const match of provider.matchEmojiSurrogates(input)) {
+		const candidate = input.slice(match.start, match.end);
+		if (PLAINTEXT_SYMBOLS.has(candidate)) continue;
+		const name = match.name;
+		if (!name) continue;
+		const candidateBytes = textEncoder.encode(candidate).byteLength;
+		const byteOffset = offsetTracker.offsetFor(input, match.start);
+		context += appendContextLine([
+			'S',
+			String(byteOffset),
+			String(candidateBytes),
+			candidate,
+			name,
+			convertToCodePoints(candidate),
+		]);
+		offsetTracker.advance(candidate.length, candidateBytes);
 	}
 	const shortcodeRegex = /:([\p{L}\p{N}_-]+):/gu;
 	const seen = new Set<string>();