feat(blocklist): add url-domain host patterns (#3164)

fluxer_api/src/api/admin/controllers/BanAdminController.ts

@@ -99,7 +99,8 @@ const BLOCKLIST_CATALOG = [
 	},
 	{
 		list_type: 'url-domain' as const,
-		description: 'Domains blocked from being linked, optionally covering every subdomain rooted at the domain.',
+		description:
+			'Domains blocked from being linked, optionally covering every subdomain rooted at the domain. A value whose leftmost label contains * is a pattern that matches that one label under a registrable domain.',
 		value_field: 'domain',
 		fields: ['match_subdomains', 'category', 'severity', 'source_url', 'notes'],
 		scoped: false,
@@ -488,7 +489,7 @@ export function BanAdminController(app: HonoApp) {
 			security: ['adminApiKey'],
 			tags: ['Admin'],
 			description:
-				'Report whether a value is currently blocked by a blocklist. The value is percent-encoded in the path. An IP address can still match a broader stored CIDR entry, and a URL can match a banned domain. The profile-substring blocklist requires a scope.',
+				'Report whether a value is currently blocked by a blocklist. The value is percent-encoded in the path. An IP address can still match a broader stored CIDR entry, and a url-domain value can be a hostname or an http(s) URL that a stored domain or pattern covers. The profile-substring blocklist requires a scope.',
 		}),
 		async (ctx) => {
 			const adminService = ctx.get('adminService');

fluxer_api/src/api/admin/services/AdminBanManagementService.ts

@@ -24,6 +24,7 @@ import {phraseBlocklistCache} from '@app/api/middleware/PhraseBlocklistCache';
 import {profileSubstringBlocklistCache} from '@app/api/middleware/ProfileSubstringBlocklistCache';
 import {urlBlocklistCache} from '@app/api/middleware/UrlBlocklistCache';
 import {canonicalizeStoredPhrase} from '@app/api/utils/PhraseBlocklistNormalization';
+import {parseUrlDomainEntry} from '@app/api/utils/UrlHostRules';
 import {canonicalizeUrl} from '@app/api/utils/UrlNormalizer';
 import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
 import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
@@ -108,6 +109,16 @@ function normalizeAvatarHashes(hashes: Array<string>): Array<string> {
 	return Array.from(new Set(hashes.map((hash) => stripAvatarAnimationPrefix(hash.toLowerCase()))));
 }
 
+function hostFromUrlOrHostname(value: string): string | null {
+	const trimmed = value.trim();
+	if (!/^https?:\/\//i.test(trimmed)) return trimmed;
+	try {
+		return new URL(trimmed).hostname;
+	} catch {
+		return null;
+	}
+}
+
 function withReasonMetadata(entries: Array<[string, string]>, reason: string | undefined): Map<string, string> {
 	if (!reason) {
 		return new Map(entries);
@@ -366,7 +377,9 @@ export class AdminBanManagementService {
 	) {
 		const {adminRepository} = this.deps;
 		const {cache: cacheService} = this.deps.apiContext.services;
-		const d = data.domain.toLowerCase();
+		const entry = parseUrlDomainEntry(data.domain);
+		if (!entry.ok) throw InputValidationError.create('domain', entry.message);
+		const d = entry.value;
 		const matchSubs = data.match_subdomains ?? true;
 		await adminRepository.banUrlDomain({
 			domain: d,
@@ -378,7 +391,7 @@ export class AdminBanManagementService {
 			added_by: adminUserId,
 			notes: data.notes ?? null,
 		});
-		urlBlocklistCache.addDomain(d);
+		urlBlocklistCache.addDomain(d, matchSubs);
 		await cacheService.publish(BANNED_URL_DOMAINS_REFRESH_CHANNEL, 'refresh');
 		await this.createBlocklistAuditLog({
 			adminUserId,
@@ -388,6 +401,7 @@ export class AdminBanManagementService {
 			metadata: new Map([
 				['domain', d],
 				['match_subdomains', String(matchSubs)],
+				['pattern', String(entry.pattern)],
 			]),
 		});
 	}
@@ -401,7 +415,8 @@ export class AdminBanManagementService {
 	) {
 		const {adminRepository} = this.deps;
 		const {cache: cacheService} = this.deps.apiContext.services;
-		const d = data.domain.toLowerCase();
+		const entry = parseUrlDomainEntry(data.domain);
+		const d = entry.ok ? entry.value : data.domain.trim().toLowerCase();
 		await adminRepository.unbanUrlDomain(d);
 		urlBlocklistCache.removeDomain(d);
 		await cacheService.publish(BANNED_URL_DOMAINS_REFRESH_CHANNEL, 'refresh');
@@ -417,7 +432,8 @@ export class AdminBanManagementService {
 	async checkUrlDomainBan(data: {domain: string}): Promise<{
 		banned: boolean;
 	}> {
-		return {banned: urlBlocklistCache.isHostnameBanned(data.domain)};
+		const host = hostFromUrlOrHostname(data.domain);
+		return {banned: host != null && urlBlocklistCache.isHostnameBanned(host)};
 	}
 
 	async banFileSha(

fluxer_api/src/api/admin/tests/AdminBlocklistUrlDomain.test.ts

@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {createChannel, createGuild} from '@app/api/channel/tests/ChannelTestUtils';
+import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
+import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+interface ValidationErrorResponse {
+	code: string;
+	errors?: Array<{path: string; message: string}>;
+}
+
+interface EntryPage {
+	items: Array<{value: string; match_subdomains: boolean | null}>;
+}
+
+describe('Admin url-domain blocklist patterns', () => {
+	let harness: ApiTestHarness;
+	let admin: TestAccount;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		await harness.reset();
+		admin = await setUserACLs(harness, await createTestAccount(harness), [
+			'admin:authenticate',
+			'ban:url_domain:add',
+			'ban:url_domain:check',
+			'ban:url_domain:remove',
+		]);
+	});
+
+	afterAll(async () => {
+		await harness?.shutdown();
+	});
+
+	async function add(domain: string, matchSubdomains?: boolean): Promise<void> {
+		await createBuilder(harness, admin.token)
+			.post('/admin/blocklists/url-domain/entries')
+			.body(matchSubdomains === undefined ? {domain} : {domain, match_subdomains: matchSubdomains})
+			.expect(204)
+			.execute();
+	}
+
+	async function check(value: string): Promise<boolean> {
+		const json = await createBuilder<{banned: boolean}>(harness, admin.token)
+			.get(`/admin/blocklists/url-domain/entries/${encodeURIComponent(value)}`)
+			.expect(200)
+			.execute();
+		return json.banned;
+	}
+
+	async function list(): Promise<EntryPage['items']> {
+		const json = await createBuilder<EntryPage>(harness, admin.token)
+			.get('/admin/blocklists/url-domain/entries?limit=200')
+			.expect(200)
+			.execute();
+		return json.items;
+	}
+
+	it('stores a canonical pattern and reports the hosts it covers', async () => {
+		await add('**Shop**.OnRender.com.');
+		expect(await list()).toMatchObject([{value: '*shop*.onrender.com', match_subdomains: true}]);
+		expect(await check('shop-2.onrender.com')).toBe(true);
+		expect(await check('https://www.myshop.onrender.com/checkout')).toBe(true);
+		expect(await check('onrender.com')).toBe(false);
+		expect(await check('docs.onrender.com')).toBe(false);
+	});
+
+	it('records whether the entry is a pattern in the audit log', async () => {
+		await add('*shop*.onrender.com', false);
+		await add('shop.example.com');
+		const logs = (await getAdminRepository().listAllAuditLogsPaginated(1000)).filter(
+			(log) => log.action === 'ban_url_domain',
+		);
+		const metadata = logs.map((log) => Object.fromEntries(log.metadata));
+		expect(metadata).toEqual(
+			expect.arrayContaining([
+				{domain: '*shop*.onrender.com', match_subdomains: 'false', pattern: 'true'},
+				{domain: 'shop.example.com', match_subdomains: 'true', pattern: 'false'},
+			]),
+		);
+	});
+
+	it('rejects patterns that are too broad or malformed', async () => {
+		for (const domain of ['*', '*.com', '*shop*.co.uk', '*.onrender.com', '*ab*.onrender.com', 'shop.*.example.com']) {
+			const json = await createBuilder<ValidationErrorResponse>(harness, admin.token)
+				.post('/admin/blocklists/url-domain/entries')
+				.body({domain})
+				.expect(400, 'INVALID_FORM_BODY')
+				.execute();
+			expect(json.errors?.[0]?.path, domain).toBe('domain');
+		}
+		expect(await list()).toEqual([]);
+	});
+
+	it('validates the value on update', async () => {
+		const json = await createBuilder<ValidationErrorResponse>(harness, admin.token)
+			.patch(`/admin/blocklists/url-domain/entries/${encodeURIComponent('*.com')}`)
+			.body({})
+			.expect(400, 'INVALID_FORM_BODY')
+			.execute();
+		expect(json.errors?.[0]?.path).toBe('domain');
+	});
+
+	it('stores internationalized domains in ASCII form', async () => {
+		await add('Bücher.Example.');
+		expect((await list()).map((entry) => entry.value)).toEqual(['xn--bcher-kva.example']);
+		expect(await check('www.bücher.example')).toBe(true);
+	});
+
+	it('accepts an add for a domain that is already blocked', async () => {
+		await add('shop.example.com');
+		await add('shop.example.com', false);
+		expect(await list()).toMatchObject([{value: 'shop.example.com', match_subdomains: false}]);
+	});
+
+	it('removes a pattern through any spelling that canonicalizes to it', async () => {
+		await add('*shop*.onrender.com');
+		await createBuilder(harness, admin.token)
+			.delete(`/admin/blocklists/url-domain/entries/${encodeURIComponent('*SHOP**.onrender.com')}`)
+			.expect(204)
+			.execute();
+		expect(await list()).toEqual([]);
+		expect(await check('shop.onrender.com')).toBe(false);
+	});
+
+	it('blocks messages whose masked links or autolinks point at a covered host', async () => {
+		await add('*shop*.onrender.com');
+		const member = await createTestAccount(harness);
+		const guild = await createGuild(harness, member.token, 'Links');
+		const channel = await createChannel(harness, member.token, guild.id, 'general');
+		await ensureSessionStarted(harness, member.token);
+		for (const content of [
+			'[open the store](https://shop-2.onrender.com)',
+			'<https://[email protected]:8443/x>',
+			'https://SHOP.onrender.com./',
+		]) {
+			await createBuilder(harness, member.token)
+				.post(`/channels/${channel.id}/messages`)
+				.body({content})
+				.expect(403, APIErrorCodes.CONTENT_BLOCKED)
+				.execute();
+		}
+		await createBuilder(harness, member.token)
+			.post(`/channels/${channel.id}/messages`)
+			.body({content: '[docs](https://docs.onrender.com) and https://onrender.com'})
+			.expect(200)
+			.execute();
+	});
+
+	it('blocks rich embeds that link to a covered host', async () => {
+		await add('*shop*.onrender.com');
+		const member = await createTestAccount(harness);
+		const guild = await createGuild(harness, member.token, 'Embeds');
+		const channel = await createChannel(harness, member.token, guild.id, 'general');
+		await ensureSessionStarted(harness, member.token);
+		await createBuilder(harness, member.token)
+			.post(`/channels/${channel.id}/messages`)
+			.body({embeds: [{title: 'Store', url: 'https://shop.onrender.com/'}]})
+			.expect(403, APIErrorCodes.CONTENT_BLOCKED)
+			.execute();
+	});
+});