feat(app): rank forward destinations and preview the message (#2790)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-15 07:23:26 +02:00
1 parent c5aaf65a10
commit f4c5fee17e
80 files changed
+6564 -2712

No files matched your search

@@ -0,0 +1,36 @@
UNICODE LICENSE V3
COPYRIGHT AND PERMISSION NOTICE
Copyright (c) 1991-2023 Unicode, Inc.
NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING,
INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR SOFTWARE, YOU
UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS
OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY,
DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
Permission is hereby granted, free of charge, to any person obtaining a copy of
data files and any associated documentation (the "Data Files") or software and
any associated documentation (the "Software") to deal in the Data Files or
Software without restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, and/or sell copies of the Data Files
or Software, and to permit persons to whom the Data Files or Software are
furnished to do so, provided that either (a) this copyright and permission
notice appear with all copies of the Data Files or Software, or (b) this
copyright and permission notice appear in associated Documentation.
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD
PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN
THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR
SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall not be
used in advertising or otherwise to promote the sale, use or other dealings in
these Data Files or Software without prior written authorization of the
copyright holder.
@@ -0,0 +1,164 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
buildChannelSearchTerms,
consumeBestSearchTerm,
createSearchTerm,
escapeSearchPattern,
fuzzySearch,
getLoadedUnicodeConfusables,
loadUnicodeConfusables,
scoreSearchTerm,
stripCombiningMarks,
toConfusableSkeleton,
} from '@app/features/search/utils/SearchTextMatching';
import {describe, expect, it} from 'vitest';
const PATTERN_SPECIAL_CHARACTERS = '-[]/{}()*+?.\\^$|';
describe('escapeSearchPattern', () => {
it('escapes every regex special character', () => {
const expected = [...PATTERN_SPECIAL_CHARACTERS].map((character) => `\\${character}`).join('');
expect(escapeSearchPattern(PATTERN_SPECIAL_CHARACTERS)).toBe(expected);
});
it('leaves whitespace and other punctuation alone', () => {
expect(escapeSearchPattern('a b\tc#,!@&')).toBe('a b\tc#,!@&');
});
it('produces a pattern that matches the literal text', () => {
const text = `gen (${PATTERN_SPECIAL_CHARACTERS}) chat`;
expect(new RegExp(`^${escapeSearchPattern(text)}$`).test(text)).toBe(true);
});
});
describe('stripCombiningMarks', () => {
it('removes combining diacritics from precomposed and decomposed text', () => {
expect(stripCombiningMarks('José')).toBe('Jose');
expect(stripCombiningMarks('cafe\u{301}')).toBe('cafe');
});
it('keeps marks outside the combining diacritical block', () => {
expect(stripCombiningMarks('x\u{483}')).toBe('x\u{483}');
});
});
describe('fuzzySearch', () => {
it('matches an in-order subsequence', () => {
expect(fuzzySearch('gnrl', 'general')).toBe(true);
expect(fuzzySearch('lrg', 'general')).toBe(false);
});
it('requires equality when the lengths match', () => {
expect(fuzzySearch('gen', 'gen')).toBe(true);
expect(fuzzySearch('abc', 'abd')).toBe(false);
});
it('rejects a needle longer than the haystack', () => {
expect(fuzzySearch('general', 'gen')).toBe(false);
});
it('is case-sensitive', () => {
expect(fuzzySearch('G', 'general')).toBe(false);
});
it('compares UTF-16 code units', () => {
expect(fuzzySearch('\u{1f600}', 'a\u{1f600}b')).toBe(true);
});
});
describe('scoreSearchTerm', () => {
it('ranks exact, prefix, contains, every word and fuzzy matches', () => {
expect(scoreSearchTerm('general', createSearchTerm('general'))).toBe(10);
expect(scoreSearchTerm('general-chat', createSearchTerm('general'))).toBe(7);
expect(scoreSearchTerm('my-general', createSearchTerm('general'))).toBe(5);
expect(scoreSearchTerm('general-chat', createSearchTerm('chat general'))).toBe(3);
expect(scoreSearchTerm('general', createSearchTerm('gnrl'))).toBe(1);
expect(scoreSearchTerm('general', createSearchTerm('xyz'))).toBe(0);
});
it('skips the fuzzy tier when fuzzy matching is off', () => {
expect(scoreSearchTerm('general', createSearchTerm('gnrl'), false)).toBe(0);
});
it('splits words on commas and spaces', () => {
expect(scoreSearchTerm('alice and bob', createSearchTerm('bob,alice'))).toBe(3);
});
});
describe('buildChannelSearchTerms', () => {
it('builds one term per word and drops empty tokens', () => {
const terms = buildChannelSearchTerms('general');
expect(terms.map((term) => [term.queryLower, term.isFullMatch])).toEqual([['general', false]]);
});
it('puts a full-phrase term first when the query has a space', () => {
const terms = buildChannelSearchTerms(' General Chat');
expect(terms.map((term) => [term.queryLower, term.isFullMatch])).toEqual([
[' general chat', true],
['general', false],
['chat', false],
]);
});
it('lets only the first space of the full phrase match a dash', () => {
const [fullPhrase] = buildChannelSearchTerms('a b c');
expect(fullPhrase.exactQuery.test('a-b c')).toBe(true);
expect(fullPhrase.exactQuery.test('a b c')).toBe(true);
expect(fullPhrase.exactQuery.test('a b-c')).toBe(false);
});
});
describe('consumeBestSearchTerm', () => {
it('removes the winning word term and returns its score', () => {
const terms = [...buildChannelSearchTerms('chat general')];
expect(consumeBestSearchTerm('general-chat', terms, true)).toBe(7);
expect(terms.map((term) => term.queryLower)).toEqual(['chat general', 'chat']);
});
it('empties the terms when the full phrase wins', () => {
const terms = [...buildChannelSearchTerms('general chat')];
expect(consumeBestSearchTerm('general-chat', terms, true)).toBe(7);
expect(terms).toEqual([]);
});
it('keeps the terms when nothing matches', () => {
const terms = [...buildChannelSearchTerms('xyz')];
expect(consumeBestSearchTerm('general', terms, true)).toBe(0);
expect(terms).toHaveLength(1);
});
});
describe('unicode confusables', () => {
it('has no table before it loads', () => {
expect(getLoadedUnicodeConfusables()).toBeNull();
});
it('loads the full Unicode 15.1 table once', async () => {
const confusables = await loadUnicodeConfusables();
expect(confusables.size).toBe(6311);
expect(await loadUnicodeConfusables()).toBe(confusables);
expect(getLoadedUnicodeConfusables()).toBe(confusables);
});
it('maps known look-alikes', async () => {
const confusables = await loadUnicodeConfusables();
expect(confusables.get('0')).toBe('O');
expect(confusables.get('1')).toBe('l');
expect(confusables.get('I')).toBe('l');
expect(confusables.get('m')).toBe('rn');
expect(confusables.get('\u{1d7ce}')).toBe('O');
});
it('builds lowercase skeletons', async () => {
const confusables = await loadUnicodeConfusables();
expect(toConfusableSkeleton('b0b', confusables)).toBe('bob');
expect(toConfusableSkeleton('Il1|', confusables)).toBe('llll');
expect(toConfusableSkeleton('mod', confusables)).toBe('rnod');
});
it('only lowercases and decomposes without a table', () => {
expect(toConfusableSkeleton('B0B', new Map())).toBe('b0b');
expect(toConfusableSkeleton('É', new Map())).toBe('e\u{301}');
});
});
@@ -0,0 +1,127 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const SEARCH_PATTERN_SPECIAL_CHARACTERS = /[-[\]/{}()*+?.\\^$|]/g;
const COMBINING_MARKS = /[\u{300}-\u{36f}]/gu;
const QUERY_WORD_SEPARATORS = /[, ]+/;
interface SearchTerm {
readonly containQuery: RegExp;
readonly exactQuery: RegExp;
readonly isFullMatch: boolean;
readonly queryLower: string;
}
let loadedUnicodeConfusables: ReadonlyMap<string, string> | null = null;
let pendingUnicodeConfusables: Promise<ReadonlyMap<string, string>> | null = null;
export function escapeSearchPattern(text: string): string {
return text.replace(SEARCH_PATTERN_SPECIAL_CHARACTERS, '\\$&');
}
export function stripCombiningMarks(text: string): string {
return text.normalize('NFD').replace(COMBINING_MARKS, '').normalize('NFC');
}
export function toConfusableSkeleton(text: string, confusables: ReadonlyMap<string, string>): string {
let skeleton = '';
for (const character of text.normalize('NFD')) {
skeleton += confusables.get(character) ?? character;
}
return skeleton.normalize('NFD').toLocaleLowerCase();
}
export function fuzzySearch(needle: string, haystack: string): boolean {
if (needle.length > haystack.length) return false;
if (needle.length === haystack.length) return needle === haystack;
let searchFrom = 0;
for (let index = 0; index < needle.length; index++) {
searchFrom = haystack.indexOf(needle[index], searchFrom) + 1;
if (searchFrom === 0) return false;
}
return true;
}
export function scoreSearchTerm(value: string, term: SearchTerm, fuzzy = true): number {
if (term.exactQuery.test(value)) return value.toLocaleLowerCase() === term.queryLower ? 10 : 7;
if (term.containQuery.test(value)) return 5;
const words = term.queryLower.split(QUERY_WORD_SEPARATORS);
if (words.every((word) => new RegExp(escapeSearchPattern(word), 'i').test(value))) return 3;
if (fuzzy && fuzzySearch(term.queryLower, value)) return 1;
return 0;
}
export function consumeBestSearchTerm(value: string, terms: Array<SearchTerm>, fuzzy: boolean): number {
let bestScore = 0;
let bestIndex = -1;
for (let index = 0; index < terms.length; index++) {
const score = scoreSearchTerm(value, terms[index], fuzzy);
if (score <= bestScore) continue;
bestScore = score;
bestIndex = index;
}
if (bestIndex === -1) return 0;
if (terms[bestIndex].isFullMatch) {
terms.length = 0;
} else {
terms.splice(bestIndex, 1);
}
return bestScore;
}
function buildSearchTerm(queryLower: string, pattern: string, isFullMatch: boolean): SearchTerm {
return Object.freeze({
containQuery: new RegExp(pattern, 'i'),
exactQuery: new RegExp(`^${pattern}`, 'i'),
isFullMatch,
queryLower,
});
}
export function createSearchTerm(queryLower: string): SearchTerm {
return buildSearchTerm(queryLower, escapeSearchPattern(queryLower), false);
}
export function buildChannelSearchTerms(query: string): ReadonlyArray<SearchTerm> {
const terms = query
.split(' ')
.filter((token) => token !== '')
.map((token) => createSearchTerm(token.toLocaleLowerCase()));
if (query.includes(' ')) {
const queryLower = query.toLocaleLowerCase();
terms.unshift(buildSearchTerm(queryLower, escapeSearchPattern(queryLower).replace(' ', '( |-)'), true));
}
return terms;
}
function decodeCodePoints(encoded: string): string {
return String.fromCodePoint(...encoded.split(' ').map((codePoint) => Number.parseInt(codePoint, 16)));
}
function decodeUnicodeConfusables(encoded: string): ReadonlyMap<string, string> {
const confusables = new Map<string, string>();
for (const entry of encoded.split(',')) {
const separator = entry.indexOf(':');
confusables.set(decodeCodePoints(entry.slice(0, separator)), decodeCodePoints(entry.slice(separator + 1)));
}
return confusables;
}
export function getLoadedUnicodeConfusables(): ReadonlyMap<string, string> | null {
return loadedUnicodeConfusables;
}
export function loadUnicodeConfusables(): Promise<ReadonlyMap<string, string>> {
if (pendingUnicodeConfusables == null) {
pendingUnicodeConfusables = import('@app/features/search/utils/UnicodeConfusables')
.then(({UNICODE_CONFUSABLES}) => {
const confusables = decodeUnicodeConfusables(UNICODE_CONFUSABLES);
loadedUnicodeConfusables = confusables;
return confusables;
})
.catch((error: unknown) => {
pendingUnicodeConfusables = null;
throw error;
});
}
return pendingUnicodeConfusables;
}
@@ -0,0 +1,5 @@
// SPDX-License-Identifier: Unicode-3.0
// SPDX-FileCopyrightText: 1991-2023 Unicode, Inc.
export const UNICODE_CONFUSABLES =
'5ad:596,5ae:598,5a8:599,5a4:59a,1ab4:6db,20db:6db,619:313,8f3:313,343:313,315:313,64f:313,65d:314,59c:301,59d:301,618:301,747:301,341:301,954:301,64e:301,340:300,953:300,30c:306,a67c:306,658:306,65a:306,36e:306,6e8:306 307,310:306 307,901:306 307,981:306 307,a81:306 307,b01:306 307,c00:306 307,c81:306 307,d01:306 307,114bf:306 307,1cd0:302,311:302,65b:302,7ee:302,a6f0:302,5af:30a,6df:30a,17d3:30a,309a:30a,652:30a,b82:30a,1036:30a,17c6:30a,11300:30a,e4d:30a,ecd:30a,366:30a,2dea:30a,8eb:308,7f3:308,64b:30b,8f0:30b,342:303,653:303,5c4:307,6ec:307,740:307,8ea:307,741:307,358:307,5b9:307,5ba:307,5c2:307,5c1:307,7ed:307,902:307,a02:307,a82:307,bcd:307,337:338,1ab7:328,322:328,345:328,1cd2:304,305:304,659:304,7eb:304,a6f1:304,1cda:30e,657:312,357:350,8ff:350,8f8:350,900:352,1ced:316,1cdc:329,656:329,1cd5:32b,347:333,8f9:354,8fa:355,309b:ff9e,309c:ff9f,336:335,302c:309,5c5:323,8ed:323,1cdd:323,5b4:323,65c:323,93c:323,9bc:323,a3c:323,abc:323,b3c:323,111ca:323,114c3:323,10a3a:323,8ee:324,1cde:324,f37:325,302d:325,327:326,321:326,339:326,1cd9:32d,1cd8:32e,952:331,320:331,8f1:64c,8e8:64c,8e5:64c,fc5e:fe72 651,8f2:64d,fc5f:fe74 651,fcf2:fe77 651,fc60:fe76 651,fcf3:fe79 651,fc61:fe78 651,61a:650,317:650,fcf4:fe7b 651,fc62:fe7a 651,fc63:fe7c 670,65f:655,30d:670,742:73c,a03:983,c03:983,c83:983,d03:983,d83:983,1038:983,114c1:983,17cb:e48,ec8:e48,ec9:e49,eca:e4a,ecb:e4b,a66f:20e9,2028:20,2029:20,1680:20,2000:20,2001:20,2002:20,2003:20,2004:20,2005:20,2006:20,2008:20,2009:20,200a:20,205f:20,a0:20,2007:20,202f:20,7fa:5f,fe4d:5f,fe4e:5f,fe4f:5f,2010:2d,2011:2d,2012:2d,2013:2d,fe58:2d,6d4:2d,2043:2d,2d7:2d,2212:2d,2796:2d,2cba:2d,2a29:2d 313,2e1a:2d 308,fb29:2d 307,2238:2d 307,2a2a:2d 323,a4fe:2d 2e,ff5e:301c,60d:2c,66b:2c,201a:2c,b8:2c,a4f9:2c,2e32:60c,66c:60c,37e:3b,2e35:61b,903:3a,a83:3a,ff1a:3a,589:3a,703:3a,704:3a,16ec:3a,fe30:3a,1803:3a,1809:3a,205a:3a,5c3:3a,2f8:3a,a789:3a,2236:3a,2d0:3a,a4fd:3a,2a74:3a 3a 3d,29f4:3a 2192,ff01:21,1c3:21,2d51:21,203c:21 21,2049:21 3f,294:3f,241:3f,97d:3f,13ae:3f,a6eb:3f,2048:3f 21,2047:3f 3f,2e2e:61f,1d16d:2e,2024:2e,701:2e,702:2e,a60e:2e,10a50:2e,660:2e,6f0:2e,a4f8:2e,a4fb:2e 2c,2025:2e 2e,a4fa:2e 2e,2026:2e 2e 2e,a6f4:a6f3 a6f3,30fb:b7,ff65:b7,16eb:b7,387:b7,2e31:b7,10101:b7,2022:b7,2027:b7,2219:b7,22c5:b7,a78f:b7,1427:b7,22ef:b7 b7 b7,2d48:b7 b7 b7,1444:b7 3c,22d7:b7 3e,1437:b7 3e,1440:b7 3e,152f:b7 34,147e:b7 62,1480:b7 62 307,147a:b7 64,1498:b7 4a,14b6:b7 4c,1476:b7 50,1457:b7 55,143a:b7 56,143c:b7 245,14ae:b7 393,140e:b7 394,1459:b7 548,140c:b7 1401,1410:b7 1404,1412:b7 1405,1414:b7 1406,1417:b7 140a,1419:b7 140b,143e:b7 1432,1442:b7 1434,1446:b7 1439,145b:b7 144f,1454:b7 1450,145d:b7 1450,145f:b7 1451,1461:b7 1455,1463:b7 1456,1474:b7 146b,1478:b7 146e,147c:b7 1470,1492:b7 1489,1494:b7 148b,1496:b7 148c,149a:b7 148e,149c:b7 1490,149e:b7 1491,14ac:b7 14a3,14b0:b7 14a6,14b2:b7 14a7,14b4:b7 14a8,14b8:b7 14ab,14c9:b7 14c0,18c6:b7 14c2,18c8:b7 14c3,18ca:b7 14c4,18cc:b7 14c5,14cb:b7 14c7,14cd:b7 14c8,14dc:b7 14d3,14de:b7 14d5,14e0:b7 14d6,14e2:b7 14d7,14e4:b7 14d8,14e6:b7 14da,14e8:b7 14db,14f6:b7 14ed,14f8:b7 14ef,14fa:b7 14f0,14fc:b7 14f1,14fe:b7 14f2,1500:b7 14f4,1502:b7 14f5,1517:b7 1510,1519:b7 1511,151b:b7 1512,151d:b7 1513,151f:b7 1514,1521:b7 1515,1523:b7 1516,1531:b7 1528,1533:b7 1529,1535:b7 152a,1537:b7 152b,1539:b7 152d,153b:b7 152e,18ce:b7 1543,18cf:b7 1546,18d0:b7 1547,18d1:b7 1548,18d2:b7 1549,18d3:b7 154b,154e:b7 154c,155b:b7 155a,1568:b7 1567,18b3:b7 18b1,18b6:b7 18b4,18b9:b7 18b8,18c2:b7 18c0,a830:964,965:964 964,1c3c:1c3b 1c3b,104b:104a 104a,1aa9:1aa8 1aa8,1aab:1aaa 1aa8,1b5f:1b5e 1b5e,10a57:10a56 10a56,1144c:1144b 1144b,11642:11641 11641,11c42:11c41 11c41,1c7f:1c7e 1c7e,55d:27,ff07:27,2018:27,2019:27,201b:27,2032:27,2035:27,55a:27,5f3:27,60:27,1fef:27,ff40:27,b4:27,384:27,1ffd:27,1fbd:27,1fbf:27,1ffe:27,2b9:27,374:27,2c8:27,2ca:27,2cb:27,2f4:27,2bb:27,2bd:27,2bc:27,2be:27,a78c:27,5d9:27,7f4:27,7f5:27,144a:27,16cc:27,16f51:27,16f52:27,1cd3:27 27,22:27 27,ff02:27 27,201c:27 27,201d:27 27,201f:27 27,2033:27 27,2036:27 27,3003:27 27,5f4:27 27,2dd:27 27,2ba:27 27,2f6:27 27,2ee:27 27,5f2:27 27,2034:27 27 27,2037:27 27 27,2057:27 27 27 27,181:27 42,18a:27 44,149:27 6e,1a4:27 50,1ac:27 54,1b3:27 59,ff3b:28,2768:28,2772:28,3014:28,fd3e:28,2e28:28 28,3220:28 30fc 29,2475:28 32 29,2487:28 32 4f 29,2476:28 33 29,2477:28 34 29,2478:28 35 29,2479:28 36 29,247a:28 37 29,247b:28 38 29,247c:28 39 29,249c:28 61 29,1f110:28 41 29,249d:28 62 29,1f111:28 42 29,249e:28 63 29,1f112:28 43 29,249f:28 64 29,1f113:28 44 29,24a0:28 65 29,1f114:28 45 29,24a1:28 66 29,1f115:28 46 29,24a2:28 67 29,1f116:28 47 29,24a3:28 68 29,1f117:28 48 29,24a4:28 69 29,24a5:28 6a 29,1f119:28 4a 29,24a6:28 6b 29,1f11a:28 4b 29,2474:28 6c 29,1f118:28 6c 29,24a7:28 6c 29,1f11b:28 4c 29,247f:28 6c 32 29,2480:28 6c 33 29,2481:28 6c 34 29,2482:28 6c 35 29,2483:28 6c 36 29,2484:28 6c 37 29,2485:28 6c 38 29,2486:28 6c 39 29,247e:28 6c 6c 29,247d:28 6c 4f 29,1f11c:28 4d 29,24a9:28 6e 29,1f11d:28 4e 29,24aa:28 6f 29,1f11Line truncated