fix(api): delete guilds in a background job (#3417)

fluxer_api/src/api/guild/services/GuildDataService.ts

@@ -16,6 +16,8 @@ import type {InviteRepository} from '@app/api/invite/InviteRepository';
 import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
 import type {UserRepository} from '@app/api/user/repositories/UserRepository';
 import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 
 export class GuildDataService {
 	private readonly helpers: GuildDataHelpers;
@@ -35,6 +37,7 @@ export class GuildDataService {
 		private readonly webhookRepository: WebhookRepository,
 		private readonly guildAuditLogService: GuildAuditLogService,
 		private readonly limitConfigService: LimitConfigService,
+		private readonly workerService: IWorkerService<WorkerTaskName>,
 	) {
 		this.helpers = new GuildDataHelpers(this.gatewayService, this.guildAuditLogService, this.userRepository);
 		this.operations = new GuildOperationsService(
@@ -50,6 +53,7 @@ export class GuildDataService {
 			this.helpers,
 			this.limitConfigService,
 			new GuildDiscoveryRepository(),
+			this.workerService,
 		);
 		this.vanity = new GuildVanityService(this.guildRepository, this.inviteRepository, this.helpers);
 		this.ownership = new GuildOwnershipService(this.guildRepository, this.userRepository, this.helpers);

fluxer_api/src/api/guild/services/GuildService.ts

@@ -141,6 +141,7 @@ export class GuildService {
 			webhookRepository,
 			guildAuditLogService,
 			limitConfigService,
+			workerService,
 		);
 		this.members = new GuildMemberService(
 			guildRepository,

fluxer_api/src/api/guild/services/data/GuildOperationsService.ts

@@ -48,6 +48,7 @@ import {addGuildToUncategorizedFolder, removeGuildFromUserFolders} from '@app/ap
 import {serializeGuildForAudit} from '@app/api/utils/AuditSerializationUtils';
 import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
 import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
 import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
 import {ALL_PERMISSIONS, ChannelTypes, DEFAULT_PERMISSIONS, Permissions} from '@fluxer/constants/src/ChannelConstants';
 import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
@@ -91,6 +92,7 @@ import type {
 	TemplateSerializedGuild,
 } from '@fluxer/schema/src/domains/guild/GuildTemplateSchemas';
 import {extractTimestamp} from '@fluxer/snowflake/src/SnowflakeUtils';
+import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 
 const GUILD_IMAGE_FIELDS = [
 	{field: 'icon', hash: 'icon_hash', dimensions: null, gate: null},
@@ -183,6 +185,8 @@ function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
 
 const TEMPLATE_AFK_TIMEOUT_MIN_SECONDS = 60;
 const THREAD_PURGE_CONCURRENCY = 8;
+const GUILD_DELETION_MEMBER_CONCURRENCY = 25;
+const GUILD_DELETION_PENDING_FEATURES = [GuildFeatures.UNAVAILABLE_FOR_EVERYONE, GuildFeatures.UNAVAILABLE_HIDDEN];
 const TEMPLATE_AFK_TIMEOUT_MAX_SECONDS = 3600;
 const THE_OTHER_PLATFORM_GUILD_STAGE_VOICE_CHANNEL_TYPE = 13;
 
@@ -204,6 +208,7 @@ export class GuildOperationsService {
 		private readonly helpers: GuildDataHelpers,
 		private readonly limitConfigService: LimitConfigService,
 		private readonly discoveryRepository: GuildDiscoveryRepository,
+		private readonly workerService: IWorkerService<WorkerTaskName>,
 	) {}
 
 	async getGuild(params: {userId: UserID; guildId: GuildID}): Promise<GuildResponse> {
@@ -748,18 +753,43 @@ export class GuildOperationsService {
 		if (!guildData || guildData.owner_id !== user.id.toString()) {
 			throw new MissingPermissionsError();
 		}
-		await this.performGuildDeletion(guildId, 'source_deleted');
+		await this.scheduleGuildDeletion(guildId, 'source_deleted');
 	}
 
 	async deleteGuildById(guildId: GuildID): Promise<void> {
-		await this.performGuildDeletion(guildId, 'purge');
+		await this.scheduleGuildDeletion(guildId, 'purge');
 	}
 
-	private async performGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
+	private async scheduleGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
 		const guild = await this.guildRepository.findUnique(guildId);
 		if (!guild) {
 			throw new UnknownGuildError();
 		}
+		const features = new Set(guild.features);
+		for (const feature of GUILD_DELETION_PENDING_FEATURES) {
+			features.add(feature);
+		}
+		await this.guildRepository.upsertPartial(guildId, {features}, guild.toRow());
+		await this.gatewayService.reloadGuild(guildId).catch((error) => {
+			Logger.warn({guildId, error}, 'Failed to reload guild after scheduling deletion');
+		});
+		await this.gatewayService.dispatchGuild({
+			guildId,
+			event: 'GUILD_DELETE',
+			data: {id: guildId.toString()},
+		});
+		await this.workerService.addJob(
+			'guildProcessDeletion',
+			{guildId: guildId.toString(), copyMode},
+			{jobKey: `guild-deletion-${guildId}`},
+		);
+	}
+
+	async purgeGuild(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
+		const guild = await this.guildRepository.findUnique(guildId);
+		if (!guild) {
+			return;
+		}
 		const channels = await this.channelRepository.channelData.listGuildChannels(guildId, 'complete');
 		for (const channel of channels) {
 			await scheduleDeletedChannelFollowerRemoval({channel, crossposts: this.channelRepository.crossposts, copyMode});
@@ -770,25 +800,19 @@ export class GuildOperationsService {
 			event: 'GUILD_DELETE',
 			data: {id: guildId.toString()},
 		});
-		await Promise.all(
-			members.map(async (member) => {
-				await this.gatewayService.leaveGuild({userId: member.userId, guildId});
-			}),
-		);
-		await Promise.all(members.map((member) => this.userRepository.deleteGuildSettings(member.userId, guildId)));
-		await Promise.all(
-			members.map(async (member) => {
-				const user = await this.userRepository.findUnique(member.userId);
-				if (user && !user.isBot) {
-					await removeGuildFromUserFolders({
-						userId: member.userId,
-						guildId,
-						userRepository: this.userRepository,
-						gatewayService: this.gatewayService,
-					});
-				}
-			}),
-		);
+		await mapWithConcurrency(members, GUILD_DELETION_MEMBER_CONCURRENCY, async (member) => {
+			await this.gatewayService.leaveGuild({userId: member.userId, guildId});
+			await this.userRepository.deleteGuildSettings(member.userId, guildId);
+			const user = await this.userRepository.findUnique(member.userId);
+			if (user && !user.isBot) {
+				await removeGuildFromUserFolders({
+					userId: member.userId,
+					guildId,
+					userRepository: this.userRepository,
+					gatewayService: this.gatewayService,
+				});
+			}
+		});
 		const invites = await this.inviteRepository.listGuildInvites(guildId);
 		await Promise.all(invites.map((invite) => this.inviteRepository.delete(invite.code)));
 		const webhooks = await this.webhookRepository.listByGuild(guildId);

fluxer_api/src/api/guild/tests/GuildDeletionJob.test.ts

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {createGuildID, createUserID, type GuildID} from '@app/api/BrandedTypes';
+import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
+import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
+import {getGuildRepository} from '@app/api/middleware/ServiceSingletons';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
+import type {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
+import {HTTP_STATUS} from '@app/api/test/TestConstants';
+import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
+import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest';
+
+describe('Guild deletion job', () => {
+	let harness: ApiTestHarness;
+	let worker: RecordingWorkerService;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		await harness.reset();
+		worker = recordGuildDeletions();
+	});
+
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	afterAll(async () => {
+		await harness?.shutdown();
+	});
+
+	async function guildWithMember(): Promise<{owner: TestAccount; member: TestAccount; guild: GuildResponse}> {
+		const owner = await createTestAccount(harness);
+		const member = await createTestAccount(harness);
+		const guild = await createGuild(harness, owner.token, 'Deletion Guild');
+		const channel = await getChannel(harness, owner.token, guild.system_channel_id!);
+		const invite = await createChannelInvite(harness, owner.token, channel.id);
+		await acceptInvite(harness, member.token, invite.code);
+		return {owner, member, guild};
+	}
+
+	async function deleteAsOwner(owner: TestAccount, guildId: string): Promise<void> {
+		await createBuilder(harness, owner.token)
+			.post(`/guilds/${guildId}/delete`)
+			.body({password: owner.password})
+			.expect(HTTP_STATUS.NO_CONTENT)
+			.execute();
+	}
+
+	function storedGuild(guildId: string) {
+		return getGuildRepository().findUnique(createGuildID(BigInt(guildId)));
+	}
+
+	test('the request hides the guild and leaves the purge to the worker', async () => {
+		const {owner, member, guild} = await guildWithMember();
+		const dispatched = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild');
+		const leaves = vi.spyOn(NoopGatewayService.prototype, 'leaveGuild');
+		await deleteAsOwner(owner, guild.id);
+		const jobs = worker.byTask('guildProcessDeletion');
+		expect(jobs).toHaveLength(1);
+		expect(jobs[0]?.payload).toEqual({guildId: guild.id, copyMode: 'source_deleted'});
+		expect(jobs[0]?.options?.jobKey).toBe(`guild-deletion-${guild.id}`);
+		expect(dispatched).toHaveBeenCalledWith(expect.objectContaining({event: 'GUILD_DELETE'}));
+		expect(leaves).not.toHaveBeenCalled();
+		const pending = await storedGuild(guild.id);
+		expect(pending?.features.has(GuildFeatures.UNAVAILABLE_FOR_EVERYONE)).toBe(true);
+		expect(pending?.features.has(GuildFeatures.UNAVAILABLE_HIDDEN)).toBe(true);
+		await createBuilder(harness, member.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.FORBIDDEN).execute();
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+		await createBuilder(harness, owner.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.NOT_FOUND).execute();
+	});
+
+	test('an admin deletion enqueues a purge job', async () => {
+		const admin = await createTestAccount(harness);
+		await setUserACLs(harness, admin, ['admin:authenticate', 'guild:lookup', 'guild:delete']);
+		const guild = await createGuild(harness, admin.token, 'Admin Deletion Guild');
+		await createBuilder(harness, admin.token)
+			.delete(`/admin/guilds/${guild.id}`)
+			.body(null)
+			.expect(HTTP_STATUS.OK)
+			.execute();
+		expect(worker.byTask('guildProcessDeletion').map((job) => job.payload)).toEqual([
+			{guildId: guild.id, copyMode: 'purge'},
+		]);
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+	});
+
+	test('a purge interrupted by a gateway failure completes when retried', async () => {
+		const {owner, guild} = await guildWithMember();
+		await deleteAsOwner(owner, guild.id);
+		const leaveGuild = NoopGatewayService.prototype.leaveGuild;
+		const failing = vi
+			.spyOn(NoopGatewayService.prototype, 'leaveGuild')
+			.mockRejectedValueOnce(new Error('gateway timeout'))
+			.mockImplementation(function (this: NoopGatewayService, params) {
+				return leaveGuild.call(this, params);
+			});
+		await expect(processRecordedGuildDeletions(worker)).rejects.toThrow('gateway timeout');
+		expect(await storedGuild(guild.id)).not.toBeNull();
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+		await processRecordedGuildDeletions(worker);
+		expect(failing).toHaveBeenCalled();
+	});
+
+	test('member teardown runs with bounded concurrency', async () => {
+		const {owner, guild} = await guildWithMember();
+		await deleteAsOwner(owner, guild.id);
+		const guildId: GuildID = createGuildID(BigInt(guild.id));
+		const repository = getGuildRepository();
+		const realMembers = await repository.listMembers(guildId);
+		const extraMembers = Array.from({length: 120}, (_, index) => ({
+			...realMembers[0]!,
+			userId: createUserID(BigInt(realMembers[0]!.userId) + BigInt(index + 1)),
+		}));
+		vi.spyOn(Object.getPrototypeOf(repository), 'listMembers').mockResolvedValueOnce([...realMembers, ...extraMembers]);
+		let inFlight = 0;
+		let peak = 0;
+		vi.spyOn(NoopGatewayService.prototype, 'leaveGuild').mockImplementation(async () => {
+			inFlight += 1;
+			peak = Math.max(peak, inFlight);
+			await new Promise((resolve) => setTimeout(resolve, 1));
+			inFlight -= 1;
+		});
+		await createTestGuildStack().guildService.data.operations.purgeGuild(guildId, 'source_deleted');
+		expect(peak).toBeGreaterThan(1);
+		expect(peak).toBeLessThanOrEqual(25);
+		expect(await storedGuild(guild.id)).toBeNull();
+	});
+});

fluxer_api/src/api/guild/tests/GuildDeletionTestUtils.ts

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createGuildID} from '@app/api/BrandedTypes';
+import {resetApiServicesForTesting} from '@app/api/CreateApiContext';
+import type {ChannelFollowerRemovalCopyMode} from '@app/api/channel/services/ChannelFollowers';
+import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
+import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
+import {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
+
+export function recordGuildDeletions(): RecordingWorkerService {
+	const worker = new RecordingWorkerService();
+	setInjectedWorkerService(worker);
+	resetApiServicesForTesting();
+	return worker;
+}
+
+export async function processRecordedGuildDeletions(worker: RecordingWorkerService): Promise<void> {
+	const operations = createTestGuildStack().guildService.data.operations;
+	for (const job of worker.byTask('guildProcessDeletion')) {
+		await operations.purgeGuild(
+			createGuildID(BigInt(String(job.payload.guildId))),
+			job.payload.copyMode as ChannelFollowerRemovalCopyMode,
+		);
+	}
+}

fluxer_api/src/api/guild/tests/GuildFolderOperations.test.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
 import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
 import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
 import {HTTP_STATUS} from '@app/api/test/TestConstants';
@@ -113,11 +114,13 @@ describe('Guild Folder Operations', () => {
 		const invite = await createChannelInvite(harness, owner.token, channel.id);
 		await acceptInvite(harness, member1.token, invite.code);
 		await acceptInvite(harness, member2.token, invite.code);
+		const worker = recordGuildDeletions();
 		await createBuilder(harness, owner.token)
 			.post(`/guilds/${guild.id}/delete`)
 			.body({password: owner.password})
 			.expect(HTTP_STATUS.NO_CONTENT)
 			.executeWithResponse();
+		await processRecordedGuildDeletions(worker);
 		const {json: ownerSettings} = await createBuilder<UserSettingsResponse>(harness, owner.token)
 			.get('/users/@me/settings')
 			.expect(HTTP_STATUS.OK)