diff --git a/fluxer_api/src/api/Validator.ts b/fluxer_api/src/api/Validator.ts index 44ef94e83..276677a52 100644 --- a/fluxer_api/src/api/Validator.ts +++ b/fluxer_api/src/api/Validator.ts @@ -10,7 +10,7 @@ import type {ValidationError} from '@fluxer/errors/src/domains/core/ValidationEr import type {Context, Env, Input, MiddlewareHandler, TypedResponse, ValidationTargets} from 'hono'; import {getCookie} from 'hono/cookie'; import type {ZodError, ZodTypeAny} from 'zod'; -import {parseJsonPreservingLargeIntegers} from './utils/LosslessJsonParser'; +import {readRequestJsonBody} from './utils/RequestJsonBody'; import {initializeFluxerErrorMap} from './ZodErrorMap'; initializeFluxerErrorMap(); @@ -200,12 +200,7 @@ export const Validator = < let value: unknown; switch (target) { case 'json': - try { - const raw = await c.req.text(); - value = raw.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(raw); - } catch { - value = {}; - } + value = (await readRequestJsonBody(c.req)).value; break; case 'form': { const formData = await c.req.formData(); diff --git a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts b/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts index 634b05c66..edbb59667 100644 --- a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts +++ b/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts @@ -3,6 +3,7 @@ import {ContentBlockedError} from '@fluxer/errors/src/domains/content/ContentBlockedError'; import {createMiddleware} from 'hono/factory'; import {Logger} from '../Logger'; +import {readRequestJsonBody} from '../utils/RequestJsonBody'; import {extractUrlCandidates} from '../utils/UrlNormalizer'; import {phraseBlocklistCache} from './PhraseBlocklistCache'; import {urlBlocklistCache} from './UrlBlocklistCache'; @@ -136,13 +137,11 @@ const ContentFilterMiddleware = createMiddleware(async (ctx, next) => { if (!contentType.includes('application/json')) { return next(); } - let body: unknown; - try { - body = await ctx.req.json(); - } catch { + const body = await readRequestJsonBody(ctx.req); + if (!body.parsed) { return next(); } - const strings = extractStringValues(body); + const strings = extractStringValues(body.value); if (strings.length === 0) { return next(); } diff --git a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts index 76c0ab76d..a53c6dcad 100644 --- a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts +++ b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts @@ -5,14 +5,16 @@ import {AdminRepository} from '../admin/AdminRepository'; import {BANNED_PHRASES_REFRESH_CHANNEL} from '../constants/ContentModeration'; import {Logger} from '../Logger'; import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../utils/PhraseBlocklistNormalization'; +import {SubstringMatcher} from '../utils/SubstringMatcher'; export class PhraseBlocklistCache { private rawPhrases: Array = []; private rawPhraseSet = new Set(); - private wordPhrases: Array = []; - private compactPhrases: Array = []; - private asciiWordPhrases: Array = []; - private asciiCompactPhrases: Array = []; + private rawMatcher: SubstringMatcher | null = null; + private wordMatcher: SubstringMatcher | null = null; + private compactMatcher: SubstringMatcher | null = null; + private asciiWordMatcher: SubstringMatcher | null = null; + private asciiCompactMatcher: SubstringMatcher | null = null; private isInitialized = false; private adminRepository = new AdminRepository(); private kvClient: IKVProvider | null = null; @@ -81,22 +83,14 @@ export class PhraseBlocklistCache { if (this.rawPhrases.length === 0) return false; const forms = buildPhraseMatchForms(text); return ( - this.matchAny(forms.raw, this.rawPhrases) || - this.matchAny(forms.words, this.wordPhrases) || - this.matchAny(forms.compact, this.compactPhrases) || - this.matchAny(forms.asciiWords, this.asciiWordPhrases) || - this.matchAny(forms.asciiCompact, this.asciiCompactPhrases) + this.rawMatcher?.test(forms.raw) === true || + this.wordMatcher?.test(forms.words) === true || + this.compactMatcher?.test(forms.compact) === true || + this.asciiWordMatcher?.test(forms.asciiWords) === true || + this.asciiCompactMatcher?.test(forms.asciiCompact) === true ); } - private matchAny(text: string, phrases: Array): boolean { - if (!text || phrases.length === 0) return false; - for (const phrase of phrases) { - if (text.includes(phrase)) return true; - } - return false; - } - private rebuildMatchers(): void { const rawPhraseSet = new Set(); const wordPhraseSet = new Set(); @@ -115,10 +109,11 @@ export class PhraseBlocklistCache { } this.rawPhraseSet = rawPhraseSet; this.rawPhrases = Array.from(rawPhraseSet); - this.wordPhrases = Array.from(wordPhraseSet); - this.compactPhrases = Array.from(compactPhraseSet); - this.asciiWordPhrases = Array.from(asciiWordPhraseSet); - this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet); + this.rawMatcher = SubstringMatcher.fromPatterns(rawPhraseSet); + this.wordMatcher = SubstringMatcher.fromPatterns(wordPhraseSet); + this.compactMatcher = SubstringMatcher.fromPatterns(compactPhraseSet); + this.asciiWordMatcher = SubstringMatcher.fromPatterns(asciiWordPhraseSet); + this.asciiCompactMatcher = SubstringMatcher.fromPatterns(asciiCompactPhraseSet); } isPhraseBanned(phrase: string): boolean { diff --git a/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts new file mode 100644 index 000000000..a1e9bc808 --- /dev/null +++ b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {transliterate} from 'transliteration'; +import {describe, expect, test} from 'vitest'; +import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../../utils/PhraseBlocklistNormalization'; +import {SubstringMatcher} from '../../utils/SubstringMatcher'; +import {PhraseBlocklistCache} from '../PhraseBlocklistCache'; + +const LEGACY_CONTROL_OR_FORMAT_REGEX = /[\p{Cc}\p{Cf}\uFFFE\uFFFF]/gu; +const LEGACY_VARIATION_SELECTOR_REGEX = /(?:[\uFE00-\uFE0F]|[\u{E0100}-\u{E01EF}])/gu; +const LEGACY_COMBINING_MARKS_REGEX = /\p{M}+/gu; +const LEGACY_NON_ALPHANUMERIC_REGEX = /[^\p{L}\p{N}]+/gu; +const LEGACY_ASCII_NON_ALPHANUMERIC_REGEX = /[^a-z0-9]+/g; +const LEGACY_WHITESPACE_REGEX = /\s+/gu; +const LEGACY_MIN_AGGRESSIVE_FORM_LENGTH = 3; + +interface LegacyForms { + raw: string; + words: string; + compact: string; + asciiWords: string; + asciiCompact: string; +} + +function legacyCollapseWhitespace(value: string): string { + return value.replace(LEGACY_WHITESPACE_REGEX, ' ').trim(); +} + +function legacyStripIgnorableCharacters(value: string): string { + return value.replace(LEGACY_CONTROL_OR_FORMAT_REGEX, '').replace(LEGACY_VARIATION_SELECTOR_REGEX, ''); +} + +function legacyMaybeKeepAggressiveForm(value: string): string { + return value.length >= LEGACY_MIN_AGGRESSIVE_FORM_LENGTH ? value : ''; +} + +function legacyBuildWordForm(value: string): string { + return legacyCollapseWhitespace( + value.normalize('NFKD').replace(LEGACY_COMBINING_MARKS_REGEX, '').replace(LEGACY_NON_ALPHANUMERIC_REGEX, ' '), + ); +} + +function legacyBuildAsciiWordForm(value: string): string { + return legacyCollapseWhitespace( + transliterate(value) + .toLowerCase() + .normalize('NFKD') + .replace(LEGACY_COMBINING_MARKS_REGEX, '') + .replace(LEGACY_ASCII_NON_ALPHANUMERIC_REGEX, ' '), + ); +} + +function legacyCanonicalize(value: string): string { + return legacyStripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim(); +} + +function legacyBuildPhraseMatchForms(value: string): LegacyForms { + const raw = legacyCanonicalize(value); + const words = legacyMaybeKeepAggressiveForm(legacyBuildWordForm(raw)); + const compact = legacyMaybeKeepAggressiveForm(words.replace(LEGACY_WHITESPACE_REGEX, '')); + const asciiWords = legacyMaybeKeepAggressiveForm(legacyBuildAsciiWordForm(raw)); + const asciiCompact = legacyMaybeKeepAggressiveForm(asciiWords.replace(LEGACY_WHITESPACE_REGEX, '')); + return {raw, words, compact, asciiWords, asciiCompact}; +} + +class LegacyPhraseBlocklist { + private rawPhrases: Array = []; + private rawPhraseSet = new Set(); + private wordPhrases: Array = []; + private compactPhrases: Array = []; + private asciiWordPhrases: Array = []; + private asciiCompactPhrases: Array = []; + + add(phrase: string): void { + const canonical = legacyCanonicalize(phrase); + if (!canonical || this.rawPhraseSet.has(canonical)) return; + this.rawPhrases.push(canonical); + this.rebuildMatchers(); + } + + remove(phrase: string): void { + const canonical = legacyCanonicalize(phrase); + if (!canonical || !this.rawPhraseSet.has(canonical)) return; + this.rawPhrases = this.rawPhrases.filter((item) => item !== canonical); + this.rebuildMatchers(); + } + + private rebuildMatchers(): void { + const rawPhraseSet = new Set(); + const wordPhraseSet = new Set(); + const compactPhraseSet = new Set(); + const asciiWordPhraseSet = new Set(); + const asciiCompactPhraseSet = new Set(); + for (const phrase of this.rawPhrases) { + const canonical = legacyCanonicalize(phrase); + if (!canonical) continue; + const forms = legacyBuildPhraseMatchForms(canonical); + rawPhraseSet.add(forms.raw); + if (forms.words) wordPhraseSet.add(forms.words); + if (forms.compact) compactPhraseSet.add(forms.compact); + if (forms.asciiWords) asciiWordPhraseSet.add(forms.asciiWords); + if (forms.asciiCompact) asciiCompactPhraseSet.add(forms.asciiCompact); + } + this.rawPhraseSet = rawPhraseSet; + this.rawPhrases = Array.from(rawPhraseSet); + this.wordPhrases = Array.from(wordPhraseSet); + this.compactPhrases = Array.from(compactPhraseSet); + this.asciiWordPhrases = Array.from(asciiWordPhraseSet); + this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet); + } + + private matchAny(text: string, phrases: Array): boolean { + if (!text || phrases.length === 0) return false; + for (const phrase of phrases) { + if (text.includes(phrase)) return true; + } + return false; + } + + containsBannedPhrase(text: string): boolean { + if (this.rawPhrases.length === 0) return false; + const forms = legacyBuildPhraseMatchForms(text); + return ( + this.matchAny(forms.raw, this.rawPhrases) || + this.matchAny(forms.words, this.wordPhrases) || + this.matchAny(forms.compact, this.compactPhrases) || + this.matchAny(forms.asciiWords, this.asciiWordPhrases) || + this.matchAny(forms.asciiCompact, this.asciiCompactPhrases) + ); + } + + isPhraseBanned(phrase: string): boolean { + const canonical = legacyCanonicalize(phrase); + return !!canonical && this.rawPhraseSet.has(canonical); + } + + get size(): number { + return this.rawPhraseSet.size; + } +} + +function createRandom(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +const CORPUS_ALPHABET: ReadonlyArray = [ + 'a', + 'b', + 'c', + 'n', + 'o', + 'r', + 't', + 'u', + '0', + '1', + '4', + '8', + ' ', + ' ', + '\t', + '\n', + '.', + '-', + '_', + '/', + '\\', + '+', + '*', + '?', + '[', + ']', + '(', + ')', + '{', + '}', + '|', + '^', + '$', + '\u200B', + '\u200C', + '\u200D', + '\uFE0F', + '\u0301', + '\u0335', + '\u3000', + '\uFF55', + '\uFF4E', + '\u043E', + '\u03BF', + '\u00E9', + 'ß', + 'fi', + '中', + 'א', + '🔥', + '😀', +]; + +function randomString(random: () => number, maxUnits: number): string { + const count = Math.floor(random() * (maxUnits + 1)); + let result = ''; + for (let index = 0; index < count; index++) { + result += CORPUS_ALPHABET[Math.floor(random() * CORPUS_ALPHABET.length)]!; + } + return result; +} + +const TABLE_PHRASE_LISTS: ReadonlyArray> = [ + [], + [''], + ['a'], + ['ab', 'b'], + ['abc', 'bc', 'c'], + ['unban tor', 'unban', 'ban'], + ['c++', 'c+', '++'], + ['a.b', 'a*b', 'a|b', 'a(b)c', 'a[b]c', 'a{2}', '^a$', 'a\\b', 'a?b'], + ['🔥', '🔥🔥'], + ['он', 'on', 'o'], + ['\u00E9', 'e', 'e\u0301'], + ['ｕｎ', 'un'], + ['aaa', 'aa', 'a'], + ['aaaa', 'aab', 'aba', 'baa'], + ['\u200B', 'x\u200By'], + ['tor', 'rot', 'ort'], +]; + +const TABLE_TEXTS: ReadonlyArray = [ + '', + 'a', + 'ab', + 'abc', + 'xabcx', + 'aa', + 'aaa', + 'aaaa', + 'banana', + 'please UNBAN TOR right now', + 'u n b a n t o r', + 'u.n-b_a_n t/o\\r', + 'ｕｎｂａｎ　ｔｏｒ', + 'u\u200Bn\u200Bb\u200Ba\u200Bn t\u200Co\u200Dr', + 'u̵n̵b̵a̵n̵ t̵o̵r̵', + 'unban tоr', + 'unban tοr', + 'please keep tor banned', + 'ship c++ code', + 'compiler', + 'a\\b', + 'a?b', + '^a$', + 'a{2}', + '🔥🔥🔥', + 'x🔥y', + 'caf\u00E9', + 'cafe\u0301', + '中中中', + 'אא', + ' ', + '\t\n\t', + '\u200B\u200C\u200D', +]; + +function collectPhraseMismatches(phrases: ReadonlyArray, texts: ReadonlyArray): Array { + const legacy = new LegacyPhraseBlocklist(); + const current = new PhraseBlocklistCache(); + for (const phrase of phrases) { + legacy.add(phrase); + current.add(phrase); + } + const mismatches: Array = []; + if (legacy.size !== current.size) { + mismatches.push(`size ${JSON.stringify(phrases)}: legacy=${legacy.size} current=${current.size}`); + } + for (const phrase of phrases) { + const expected = legacy.isPhraseBanned(phrase); + const actual = current.isPhraseBanned(phrase); + if (expected !== actual) { + mismatches.push(`isPhraseBanned ${JSON.stringify(phrase)}: legacy=${expected} current=${actual}`); + } + } + for (const text of texts) { + const expected = legacy.containsBannedPhrase(text); + const actual = current.containsBannedPhrase(text); + if (expected !== actual) { + mismatches.push( + `containsBannedPhrase ${JSON.stringify(phrases)} / ${JSON.stringify(text)}: legacy=${expected} current=${actual}`, + ); + } + } + return mismatches; +} + +describe('phrase blocklist matcher equivalence', () => { + test('table-driven phrase lists match the legacy linear scan', () => { + const mismatches: Array = []; + for (const phrases of TABLE_PHRASE_LISTS) { + mismatches.push(...collectPhraseMismatches(phrases, TABLE_TEXTS)); + } + expect(mismatches).toEqual([]); + }); + + test('generated phrase lists and texts match the legacy linear scan', () => { + const random = createRandom(0x5eed1234); + const mismatches: Array = []; + for (let round = 0; round < 300; round++) { + const phraseCount = 1 + Math.floor(random() * 8); + const phrases: Array = []; + for (let index = 0; index < phraseCount; index++) { + phrases.push(randomString(random, 6)); + } + const texts: Array = []; + for (let index = 0; index < 8; index++) { + texts.push(randomString(random, 40)); + } + for (const phrase of phrases) { + texts.push(phrase); + texts.push(`${randomString(random, 5)}${phrase}${randomString(random, 5)}`); + } + mismatches.push(...collectPhraseMismatches(phrases, texts)); + } + expect(mismatches).toEqual([]); + }); + + test('removal keeps both implementations in step', () => { + const random = createRandom(0x1234abcd); + const mismatches: Array = []; + const legacy = new LegacyPhraseBlocklist(); + const current = new PhraseBlocklistCache(); + const phrases: Array = []; + for (let index = 0; index < 40; index++) { + phrases.push(randomString(random, 6)); + } + for (const phrase of phrases) { + legacy.add(phrase); + current.add(phrase); + } + for (let index = 0; index < phrases.length; index += 3) { + legacy.remove(phrases[index]!); + current.remove(phrases[index]!); + } + for (let index = 0; index < 200; index++) { + const text = randomString(random, 30); + const expected = legacy.containsBannedPhrase(text); + const actual = current.containsBannedPhrase(text); + if (expected !== actual) { + mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`); + } + } + expect(mismatches).toEqual([]); + }); + + test('a large blocklist stays exact', () => { + const random = createRandom(0xfeedface); + const legacy = new LegacyPhraseBlocklist(); + const current = new PhraseBlocklistCache(); + for (let index = 0; index < 300; index++) { + const phrase = `${randomString(random, 4)}${index}`; + legacy.add(phrase); + current.add(phrase); + } + expect(current.size).toBe(legacy.size); + const mismatches: Array = []; + for (let index = 0; index < 300; index++) { + const text = randomString(random, 200); + const expected = legacy.containsBannedPhrase(text); + const actual = current.containsBannedPhrase(text); + if (expected !== actual) { + mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`); + } + } + expect(mismatches).toEqual([]); + }); +}); + +describe('buildPhraseMatchForms laziness', () => { + test('lazy forms equal the eager forms for the whole corpus', () => { + const random = createRandom(0xc0ffee11); + const values: Array = [...TABLE_TEXTS]; + for (let index = 0; index < 500; index++) { + values.push(randomString(random, 40)); + } + const mismatches: Array = []; + for (const value of values) { + const expected = legacyBuildPhraseMatchForms(value); + const forms = buildPhraseMatchForms(value); + const actual = { + raw: forms.raw, + words: forms.words, + compact: forms.compact, + asciiWords: forms.asciiWords, + asciiCompact: forms.asciiCompact, + }; + if (JSON.stringify(expected) !== JSON.stringify(actual)) { + mismatches.push(`${JSON.stringify(value)}: ${JSON.stringify(expected)} vs ${JSON.stringify(actual)}`); + } + } + expect(mismatches).toEqual([]); + }); + + test('reading a later form after an earlier one is still correct', () => { + const value = 'unban tоr'; + const forms = buildPhraseMatchForms(value); + const expected = legacyBuildPhraseMatchForms(value); + expect(forms.asciiCompact).toBe(expected.asciiCompact); + expect(forms.raw).toBe(expected.raw); + expect(forms.words).toBe(expected.words); + expect(forms.compact).toBe(expected.compact); + expect(forms.asciiWords).toBe(expected.asciiWords); + }); + + test('canonicalizeStoredPhrase is unchanged', () => { + const random = createRandom(0x0badf00d); + const mismatches: Array = []; + for (let index = 0; index < 500; index++) { + const value = randomString(random, 30); + if (canonicalizeStoredPhrase(value) !== legacyCanonicalize(value)) { + mismatches.push(JSON.stringify(value)); + } + } + expect(mismatches).toEqual([]); + }); +}); + +describe('SubstringMatcher', () => { + test('is null only for an empty pattern set', () => { + expect(SubstringMatcher.fromPatterns([])).toBeNull(); + expect(SubstringMatcher.fromPatterns(new Set())).toBeNull(); + expect(SubstringMatcher.fromPatterns([''])).not.toBeNull(); + }); + + test('agrees with String.includes on generated patterns and texts', () => { + const random = createRandom(0x2468ace0); + const mismatches: Array = []; + for (let round = 0; round < 500; round++) { + const patternCount = 1 + Math.floor(random() * 6); + const patterns: Array = []; + for (let index = 0; index < patternCount; index++) { + patterns.push(randomString(random, 5)); + } + const matcher = SubstringMatcher.fromPatterns(patterns); + if (matcher === null) { + mismatches.push(`unexpected null for ${JSON.stringify(patterns)}`); + continue; + } + for (let index = 0; index < 10; index++) { + const text = randomString(random, 30); + const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern)); + const actual = matcher.test(text); + if (expected !== actual) { + mismatches.push(`${JSON.stringify(patterns)} / ${JSON.stringify(text)}: ${expected} vs ${actual}`); + } + } + } + expect(mismatches).toEqual([]); + }); + + test('an empty pattern matches any non-empty text', () => { + const matcher = SubstringMatcher.fromPatterns(['', 'zzz']); + expect(matcher).not.toBeNull(); + expect(matcher!.test('')).toBe(false); + expect(matcher!.test('anything')).toBe(true); + }); + + test('finds patterns that are suffixes of other pattern prefixes', () => { + const matcher = SubstringMatcher.fromPatterns(['abcd', 'bc']); + expect(matcher!.test('xabcx')).toBe(true); + expect(matcher!.test('abd')).toBe(false); + expect(matcher!.test('zabcdz')).toBe(true); + }); + + test('scales to a very large pattern set without a compile limit', () => { + const random = createRandom(0x13579bdf); + const patterns: Array = []; + for (let index = 0; index < 20000; index++) { + patterns.push(`${randomString(random, 4)}${index}`); + } + const matcher = SubstringMatcher.fromPatterns(patterns); + expect(matcher).not.toBeNull(); + const mismatches: Array = []; + for (let index = 0; index < 100; index++) { + const text = index % 2 === 0 ? randomString(random, 400) : `${randomString(random, 50)}${patterns[index]!}`; + const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern)); + const actual = matcher!.test(text); + if (expected !== actual) { + mismatches.push(`${JSON.stringify(text)}: ${expected} vs ${actual}`); + } + } + expect(mismatches).toEqual([]); + }); +}); diff --git a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts index b6b66fccf..f267c953d 100644 --- a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts +++ b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts @@ -11,11 +11,11 @@ const WHITESPACE_REGEX = /\s+/gu; const MIN_AGGRESSIVE_FORM_LENGTH = 3; interface PhraseMatchForms { - raw: string; - words: string; - compact: string; - asciiWords: string; - asciiCompact: string; + readonly raw: string; + readonly words: string; + readonly compact: string; + readonly asciiWords: string; + readonly asciiCompact: string; } function collapseWhitespace(value: string): string { @@ -46,21 +46,50 @@ function buildAsciiWordFormFromCanonical(value: string): string { ); } +class LazyPhraseMatchForms implements PhraseMatchForms { + readonly raw: string; + private wordsForm: string | null = null; + private compactForm: string | null = null; + private asciiWordsForm: string | null = null; + private asciiCompactForm: string | null = null; + + constructor(value: string) { + this.raw = canonicalizeStoredPhrase(value); + } + + get words(): string { + if (this.wordsForm === null) { + this.wordsForm = maybeKeepAggressiveForm(buildWordFormFromCanonical(this.raw)); + } + return this.wordsForm; + } + + get compact(): string { + if (this.compactForm === null) { + this.compactForm = maybeKeepAggressiveForm(this.words.replace(WHITESPACE_REGEX, '')); + } + return this.compactForm; + } + + get asciiWords(): string { + if (this.asciiWordsForm === null) { + this.asciiWordsForm = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(this.raw)); + } + return this.asciiWordsForm; + } + + get asciiCompact(): string { + if (this.asciiCompactForm === null) { + this.asciiCompactForm = maybeKeepAggressiveForm(this.asciiWords.replace(WHITESPACE_REGEX, '')); + } + return this.asciiCompactForm; + } +} + export function canonicalizeStoredPhrase(value: string): string { return stripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim(); } export function buildPhraseMatchForms(value: string): PhraseMatchForms { - const raw = canonicalizeStoredPhrase(value); - const words = maybeKeepAggressiveForm(buildWordFormFromCanonical(raw)); - const compact = maybeKeepAggressiveForm(words.replace(WHITESPACE_REGEX, '')); - const asciiWords = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(raw)); - const asciiCompact = maybeKeepAggressiveForm(asciiWords.replace(WHITESPACE_REGEX, '')); - return { - raw, - words, - compact, - asciiWords, - asciiCompact, - }; + return new LazyPhraseMatchForms(value); } diff --git a/fluxer_api/src/api/utils/RequestJsonBody.ts b/fluxer_api/src/api/utils/RequestJsonBody.ts new file mode 100644 index 000000000..e9b03774a --- /dev/null +++ b/fluxer_api/src/api/utils/RequestJsonBody.ts @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {HonoRequest} from 'hono'; +import {parseJsonPreservingLargeIntegers} from './LosslessJsonParser'; + +interface RequestJsonBody { + parsed: boolean; + value: unknown; +} + +const bodyCache = new WeakMap(); + +export async function readRequestJsonBody(req: HonoRequest): Promise { + const raw = req.raw; + const cached = bodyCache.get(raw); + if (cached) return cached; + let body: RequestJsonBody; + try { + const text = await req.text(); + body = {parsed: true, value: text.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(text)}; + } catch { + body = {parsed: false, value: {}}; + } + bodyCache.set(raw, body); + return body; +} diff --git a/fluxer_api/src/api/utils/SubstringMatcher.ts b/fluxer_api/src/api/utils/SubstringMatcher.ts new file mode 100644 index 000000000..22083297a --- /dev/null +++ b/fluxer_api/src/api/utils/SubstringMatcher.ts @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +const ROOT_NODE = 0; +const ASCII_ROOT_SIZE = 128; +const NO_TRANSITION = -1; +const EDGE_KEY_STRIDE = 65536; + +export class SubstringMatcher { + private readonly rowStart: Int32Array; + private readonly edgeCodes: Int32Array; + private readonly edgeTargets: Int32Array; + private readonly failures: Int32Array; + private readonly terminals: Uint8Array; + private readonly asciiRoot: Int32Array; + private readonly matchesEverything: boolean; + + private constructor( + rowStart: Int32Array, + edgeCodes: Int32Array, + edgeTargets: Int32Array, + failures: Int32Array, + terminals: Uint8Array, + matchesEverything: boolean, + ) { + this.rowStart = rowStart; + this.edgeCodes = edgeCodes; + this.edgeTargets = edgeTargets; + this.failures = failures; + this.terminals = terminals; + this.matchesEverything = matchesEverything; + this.asciiRoot = new Int32Array(ASCII_ROOT_SIZE); + for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) { + const code = edgeCodes[edge]!; + if (code < ASCII_ROOT_SIZE) { + this.asciiRoot[code] = edgeTargets[edge]!; + } + } + } + + static fromPatterns(patterns: Iterable): SubstringMatcher | null { + const children = new Map(); + const edgeSources: Array = []; + const edgeLabels: Array = []; + const edgeChildren: Array = []; + const terminalNodes: Array = []; + let nodeCount = 1; + let matchesEverything = false; + let count = 0; + for (const pattern of patterns) { + count++; + if (pattern.length === 0) { + matchesEverything = true; + continue; + } + let node = ROOT_NODE; + for (let index = 0; index < pattern.length; index++) { + const code = pattern.charCodeAt(index); + const key = node * EDGE_KEY_STRIDE + code; + let next = children.get(key); + if (next === undefined) { + next = nodeCount++; + children.set(key, next); + edgeSources.push(node); + edgeLabels.push(code); + edgeChildren.push(next); + } + node = next; + } + terminalNodes.push(node); + } + if (count === 0) { + return null; + } + children.clear(); + const edgeCount = edgeSources.length; + const rowStart = new Int32Array(nodeCount + 1); + for (let edge = 0; edge < edgeCount; edge++) { + rowStart[edgeSources[edge]! + 1]!++; + } + for (let node = 0; node < nodeCount; node++) { + rowStart[node + 1]! += rowStart[node]!; + } + const cursor = rowStart.slice(0, nodeCount); + const edgeCodes = new Int32Array(edgeCount); + const edgeTargets = new Int32Array(edgeCount); + for (let edge = 0; edge < edgeCount; edge++) { + const source = edgeSources[edge]!; + const slot = cursor[source]!; + cursor[source] = slot + 1; + edgeCodes[slot] = edgeLabels[edge]!; + edgeTargets[slot] = edgeChildren[edge]!; + } + for (let node = 0; node < nodeCount; node++) { + const start = rowStart[node]!; + const end = rowStart[node + 1]!; + for (let index = start + 1; index < end; index++) { + const code = edgeCodes[index]!; + const target = edgeTargets[index]!; + let scan = index - 1; + while (scan >= start && edgeCodes[scan]! > code) { + edgeCodes[scan + 1] = edgeCodes[scan]!; + edgeTargets[scan + 1] = edgeTargets[scan]!; + scan--; + } + edgeCodes[scan + 1] = code; + edgeTargets[scan + 1] = target; + } + } + const failures = new Int32Array(nodeCount); + const terminals = new Uint8Array(nodeCount); + for (const node of terminalNodes) { + terminals[node] = 1; + } + const queue = new Int32Array(nodeCount); + let tail = 0; + for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) { + queue[tail] = edgeTargets[edge]!; + tail++; + } + for (let head = 0; head < tail; head++) { + const node = queue[head]!; + if (terminals[failures[node]!]! === 1) { + terminals[node] = 1; + } + for (let edge = rowStart[node]!; edge < rowStart[node + 1]!; edge++) { + const code = edgeCodes[edge]!; + const child = edgeTargets[edge]!; + let candidate = failures[node]!; + let target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code); + while (target === NO_TRANSITION && candidate !== ROOT_NODE) { + candidate = failures[candidate]!; + target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code); + } + failures[child] = target === NO_TRANSITION ? ROOT_NODE : target; + queue[tail] = child; + tail++; + } + } + return new SubstringMatcher(rowStart, edgeCodes, edgeTargets, failures, terminals, matchesEverything); + } + + test(text: string): boolean { + if (text.length === 0) { + return false; + } + if (this.matchesEverything) { + return true; + } + const rowStart = this.rowStart; + const edgeCodes = this.edgeCodes; + const edgeTargets = this.edgeTargets; + const failures = this.failures; + const terminals = this.terminals; + const asciiRoot = this.asciiRoot; + let node = ROOT_NODE; + for (let index = 0; index < text.length; index++) { + const code = text.charCodeAt(index); + if (node === ROOT_NODE && code < ASCII_ROOT_SIZE) { + node = asciiRoot[code]!; + } else { + let next = findTransition(rowStart, edgeCodes, edgeTargets, node, code); + while (next === NO_TRANSITION && node !== ROOT_NODE) { + node = failures[node]!; + next = findTransition(rowStart, edgeCodes, edgeTargets, node, code); + } + node = next === NO_TRANSITION ? ROOT_NODE : next; + } + if (terminals[node]! === 1) { + return true; + } + } + return false; + } +} + +function findTransition( + rowStart: Int32Array, + edgeCodes: Int32Array, + edgeTargets: Int32Array, + node: number, + code: number, +): number { + let low = rowStart[node]!; + let high = rowStart[node + 1]! - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const candidate = edgeCodes[middle]!; + if (candidate === code) { + return edgeTargets[middle]!; + } + if (candidate < code) { + low = middle + 1; + } else { + high = middle - 1; + } + } + return NO_TRANSITION; +}