fix(api): delete guilds in a background job (#3417)

fluxer_admin/openapi-admin.json

@@ -3074,7 +3074,7 @@
 						"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
 					}
 				},
-				"description": "Permanently deletes a guild. Deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.",
+				"description": "Permanently deletes a guild. The guild becomes unavailable at once, and a background job deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.",
 				"security": [{"adminApiKey": []}],
 				"parameters": [
 					{

fluxer_api/src/api/admin/controllers/GuildAdminController.ts

@@ -253,7 +253,7 @@ export function GuildAdminController(app: HonoApp) {
 			operationId: 'delete_admin_guild',
 			summary: 'Delete guild',
 			description:
-				'Permanently deletes a guild. Deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.',
+				'Permanently deletes a guild. The guild becomes unavailable at once, and a background job deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.',
 			responseSchema: SuccessResponse,
 			statusCode: 200,
 			security: 'adminApiKey',

fluxer_api/src/api/channel/tests/AnnouncementChannelConversion.test.ts

@@ -22,6 +22,7 @@ import {
 	sendMessage,
 	setupFanoutWorld,
 } from '@app/api/channel/tests/CrosspostWorkerTestUtils';
+import {processRecordedGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
 import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
 import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
 import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
@@ -315,6 +316,7 @@ describe('Announcement channel conversion', () => {
 			.body({password: owner.password})
 			.expect(HTTP_STATUS.NO_CONTENT)
 			.executeWithResponse();
+		await processRecordedGuildDeletions(worker);
 		const removals = worker.byTask('removeChannelFollowers');
 		expect(removals.map((job) => job.payload.sourceChannelId).sort()).toEqual([first.id, second.id].sort());
 		for (const job of removals) {
@@ -333,6 +335,7 @@ describe('Announcement channel conversion', () => {
 			.body(null)
 			.expect(HTTP_STATUS.OK)
 			.execute();
+		await processRecordedGuildDeletions(worker);
 		const removals = worker.byTask('removeChannelFollowers');
 		expect(removals).toHaveLength(1);
 		expect(removals[0]?.payload).toEqual({

fluxer_api/src/api/channel/tests/AnnouncementTestUtils.ts

@@ -11,7 +11,7 @@ import {
 	createPermissionOverwrite,
 	createRole,
 } from '@app/api/channel/tests/ChannelTestUtils';
-import {createTestChannelService} from '@app/api/channel/tests/CrosspostTestUtils';
+import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
 import {getGatewayService, getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
 import {
 	getAvatarService,
@@ -31,6 +31,7 @@ import {createBuilder} from '@app/api/test/TestRequestBuilder';
 import {WebhookRepository} from '@app/api/webhook/WebhookRepository';
 import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage';
 import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk';
+import guildProcessDeletion from '@app/api/worker/tasks/GuildProcessDeletion';
 import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers';
 import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies';
 import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage';
@@ -201,6 +202,7 @@ export const crosspostTaskHandlers: Record<string, WorkerTaskHandler> = {
 	syncCrosspostedMessage,
 	syncCrosspostCopies,
 	removeChannelFollowers,
+	guildProcessDeletion,
 };
 
 export function enableCrosspostWorker(
@@ -210,6 +212,7 @@ export function enableCrosspostWorker(
 	const worker = new SyncTaskWorkerService({...crosspostTaskHandlers, ...extraHandlers}, {deferred: true});
 	setInjectedWorkerService(worker);
 	resetApiServicesForTesting();
+	const guildStack = createTestGuildStack();
 	setWorkerDependenciesForTest({
 		channelRepository: getChannelRepository(),
 		webhookRepository: getWebhookRepository(),
@@ -222,7 +225,8 @@ export function enableCrosspostWorker(
 		snowflakeService: getSnowflakeService(),
 		cacheService: getCacheService(),
 		limitConfigService: getLimitConfigService(),
-		channelService: createTestChannelService(),
+		channelService: guildStack.channelService,
+		guildService: guildStack.guildService,
 		workerService: worker,
 		...extraDependencies,
 	});

fluxer_api/src/api/channel/tests/CrosspostTestUtils.ts

@@ -22,7 +22,7 @@ import {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
 import {DisabledLiveKitService} from '@app/api/infrastructure/DisabledLiveKitService';
 import {InMemoryVoiceRoomStore} from '@app/api/infrastructure/InMemoryVoiceRoomStore';
 import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
-import {createGuildStackServices} from '@app/api/middleware/GuildStackServiceFactory';
+import {createGuildStackServices, type GuildStackServices} from '@app/api/middleware/GuildStackServiceFactory';
 import {getVoiceAvailabilityService} from '@app/api/middleware/ServiceRegistry';
 import {
 	getAssetDeletionQueue,
@@ -190,7 +190,7 @@ export function createStallGate(): StallGate & {hit: () => Promise<void>} {
 	};
 }
 
-export function createTestChannelService(): ChannelService {
+export function createTestGuildStack(): GuildStackServices {
 	return createGuildStackServices({
 		apiContext: createApiContext(),
 		channelRepository: getChannelRepository(),
@@ -214,7 +214,11 @@ export function createTestChannelService(): ChannelService {
 		voiceRoomStore: new InMemoryVoiceRoomStore(),
 		liveKitService: new DisabledLiveKitService(),
 		voiceAvailabilityService: getVoiceAvailabilityService(),
-	}).channelService;
+	});
+}
+
+export function createTestChannelService(): ChannelService {
+	return createTestGuildStack().channelService;
 }
 
 export {sendChannelMessage as postMessage} from '@app/api/channel/tests/ChannelTestUtils';

fluxer_api/src/api/channel/tests/ThreadMessagePath.test.ts

@@ -9,6 +9,7 @@ import {acceptInvite, createChannel, createChannelInvite, createGuild} from '@ap
 import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';
 import {upsertOne} from '@app/api/database/CassandraQueryExecution';
 import {clearChannelThreadsTaintCacheForTesting} from '@app/api/experiment/ChannelThreadsGate';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
 import {ensureSessionStarted, sendMessage} from '@app/api/message/tests/MessageTestUtils';
 import {getKVThreadAutoArchiveQueue} from '@app/api/middleware/ServiceSingletons';
 import {ThreadState as ThreadStateTable} from '@app/api/Tables';
@@ -481,11 +482,13 @@ describe('thread message path', () => {
 			const queue = getKVThreadAutoArchiveQueue();
 			const guildId = createGuildID(BigInt(s.guildId));
 			expect(await queue.listGuilds()).toContain(guildId);
+			const worker = recordGuildDeletions();
 			await createBuilder(harness, s.owner.token)
 				.post(`/guilds/${s.guildId}/delete`)
 				.body({password: s.owner.password})
 				.expect(204)
 				.execute();
+			await processRecordedGuildDeletions(worker);
 			expect(await queue.listGuilds()).not.toContain(guildId);
 			expect(await queue.getDue(guildId, Number.MAX_SAFE_INTEGER, 10)).toEqual([]);
 		});

fluxer_api/src/api/guild/services/GuildDataService.ts

@@ -16,6 +16,8 @@ import type {InviteRepository} from '@app/api/invite/InviteRepository';
 import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
 import type {UserRepository} from '@app/api/user/repositories/UserRepository';
 import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 
 export class GuildDataService {
 	private readonly helpers: GuildDataHelpers;
@@ -35,6 +37,7 @@ export class GuildDataService {
 		private readonly webhookRepository: WebhookRepository,
 		private readonly guildAuditLogService: GuildAuditLogService,
 		private readonly limitConfigService: LimitConfigService,
+		private readonly workerService: IWorkerService<WorkerTaskName>,
 	) {
 		this.helpers = new GuildDataHelpers(this.gatewayService, this.guildAuditLogService, this.userRepository);
 		this.operations = new GuildOperationsService(
@@ -50,6 +53,7 @@ export class GuildDataService {
 			this.helpers,
 			this.limitConfigService,
 			new GuildDiscoveryRepository(),
+			this.workerService,
 		);
 		this.vanity = new GuildVanityService(this.guildRepository, this.inviteRepository, this.helpers);
 		this.ownership = new GuildOwnershipService(this.guildRepository, this.userRepository, this.helpers);

fluxer_api/src/api/guild/services/GuildService.ts

@@ -141,6 +141,7 @@ export class GuildService {
 			webhookRepository,
 			guildAuditLogService,
 			limitConfigService,
+			workerService,
 		);
 		this.members = new GuildMemberService(
 			guildRepository,

fluxer_api/src/api/guild/services/data/GuildOperationsService.ts

@@ -48,6 +48,7 @@ import {addGuildToUncategorizedFolder, removeGuildFromUserFolders} from '@app/ap
 import {serializeGuildForAudit} from '@app/api/utils/AuditSerializationUtils';
 import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
 import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
 import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
 import {ALL_PERMISSIONS, ChannelTypes, DEFAULT_PERMISSIONS, Permissions} from '@fluxer/constants/src/ChannelConstants';
 import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
@@ -91,6 +92,7 @@ import type {
 	TemplateSerializedGuild,
 } from '@fluxer/schema/src/domains/guild/GuildTemplateSchemas';
 import {extractTimestamp} from '@fluxer/snowflake/src/SnowflakeUtils';
+import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 
 const GUILD_IMAGE_FIELDS = [
 	{field: 'icon', hash: 'icon_hash', dimensions: null, gate: null},
@@ -183,6 +185,8 @@ function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
 
 const TEMPLATE_AFK_TIMEOUT_MIN_SECONDS = 60;
 const THREAD_PURGE_CONCURRENCY = 8;
+const GUILD_DELETION_MEMBER_CONCURRENCY = 25;
+const GUILD_DELETION_PENDING_FEATURES = [GuildFeatures.UNAVAILABLE_FOR_EVERYONE, GuildFeatures.UNAVAILABLE_HIDDEN];
 const TEMPLATE_AFK_TIMEOUT_MAX_SECONDS = 3600;
 const THE_OTHER_PLATFORM_GUILD_STAGE_VOICE_CHANNEL_TYPE = 13;
 
@@ -204,6 +208,7 @@ export class GuildOperationsService {
 		private readonly helpers: GuildDataHelpers,
 		private readonly limitConfigService: LimitConfigService,
 		private readonly discoveryRepository: GuildDiscoveryRepository,
+		private readonly workerService: IWorkerService<WorkerTaskName>,
 	) {}
 
 	async getGuild(params: {userId: UserID; guildId: GuildID}): Promise<GuildResponse> {
@@ -748,18 +753,43 @@ export class GuildOperationsService {
 		if (!guildData || guildData.owner_id !== user.id.toString()) {
 			throw new MissingPermissionsError();
 		}
-		await this.performGuildDeletion(guildId, 'source_deleted');
+		await this.scheduleGuildDeletion(guildId, 'source_deleted');
 	}
 
 	async deleteGuildById(guildId: GuildID): Promise<void> {
-		await this.performGuildDeletion(guildId, 'purge');
+		await this.scheduleGuildDeletion(guildId, 'purge');
 	}
 
-	private async performGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
+	private async scheduleGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
 		const guild = await this.guildRepository.findUnique(guildId);
 		if (!guild) {
 			throw new UnknownGuildError();
 		}
+		const features = new Set(guild.features);
+		for (const feature of GUILD_DELETION_PENDING_FEATURES) {
+			features.add(feature);
+		}
+		await this.guildRepository.upsertPartial(guildId, {features}, guild.toRow());
+		await this.gatewayService.reloadGuild(guildId).catch((error) => {
+			Logger.warn({guildId, error}, 'Failed to reload guild after scheduling deletion');
+		});
+		await this.gatewayService.dispatchGuild({
+			guildId,
+			event: 'GUILD_DELETE',
+			data: {id: guildId.toString()},
+		});
+		await this.workerService.addJob(
+			'guildProcessDeletion',
+			{guildId: guildId.toString(), copyMode},
+			{jobKey: `guild-deletion-${guildId}`},
+		);
+	}
+
+	async purgeGuild(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
+		const guild = await this.guildRepository.findUnique(guildId);
+		if (!guild) {
+			return;
+		}
 		const channels = await this.channelRepository.channelData.listGuildChannels(guildId, 'complete');
 		for (const channel of channels) {
 			await scheduleDeletedChannelFollowerRemoval({channel, crossposts: this.channelRepository.crossposts, copyMode});
@@ -770,25 +800,19 @@ export class GuildOperationsService {
 			event: 'GUILD_DELETE',
 			data: {id: guildId.toString()},
 		});
-		await Promise.all(
+		await mapWithConcurrency(members, GUILD_DELETION_MEMBER_CONCURRENCY, async (member) => {
-			members.map(async (member) => {
+			await this.gatewayService.leaveGuild({userId: member.userId, guildId});
-				await this.gatewayService.leaveGuild({userId: member.userId, guildId});
+			await this.userRepository.deleteGuildSettings(member.userId, guildId);
-			}),
+			const user = await this.userRepository.findUnique(member.userId);
-		);
+			if (user && !user.isBot) {
-		await Promise.all(members.map((member) => this.userRepository.deleteGuildSettings(member.userId, guildId)));
+				await removeGuildFromUserFolders({
-		await Promise.all(
+					userId: member.userId,
-			members.map(async (member) => {
+					guildId,
-				const user = await this.userRepository.findUnique(member.userId);
+					userRepository: this.userRepository,
-				if (user && !user.isBot) {
+					gatewayService: this.gatewayService,
-					await removeGuildFromUserFolders({
+				});
-						userId: member.userId,
+			}
-						guildId,
+		});
-						userRepository: this.userRepository,
-						gatewayService: this.gatewayService,
-					});
-				}
-			}),
-		);
 		const invites = await this.inviteRepository.listGuildInvites(guildId);
 		await Promise.all(invites.map((invite) => this.inviteRepository.delete(invite.code)));
 		const webhooks = await this.webhookRepository.listByGuild(guildId);

fluxer_api/src/api/guild/tests/GuildDeletionJob.test.ts

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {createGuildID, createUserID, type GuildID} from '@app/api/BrandedTypes';
+import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
+import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
+import {getGuildRepository} from '@app/api/middleware/ServiceSingletons';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
+import type {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
+import {HTTP_STATUS} from '@app/api/test/TestConstants';
+import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
+import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest';
+
+describe('Guild deletion job', () => {
+	let harness: ApiTestHarness;
+	let worker: RecordingWorkerService;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		await harness.reset();
+		worker = recordGuildDeletions();
+	});
+
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	afterAll(async () => {
+		await harness?.shutdown();
+	});
+
+	async function guildWithMember(): Promise<{owner: TestAccount; member: TestAccount; guild: GuildResponse}> {
+		const owner = await createTestAccount(harness);
+		const member = await createTestAccount(harness);
+		const guild = await createGuild(harness, owner.token, 'Deletion Guild');
+		const channel = await getChannel(harness, owner.token, guild.system_channel_id!);
+		const invite = await createChannelInvite(harness, owner.token, channel.id);
+		await acceptInvite(harness, member.token, invite.code);
+		return {owner, member, guild};
+	}
+
+	async function deleteAsOwner(owner: TestAccount, guildId: string): Promise<void> {
+		await createBuilder(harness, owner.token)
+			.post(`/guilds/${guildId}/delete`)
+			.body({password: owner.password})
+			.expect(HTTP_STATUS.NO_CONTENT)
+			.execute();
+	}
+
+	function storedGuild(guildId: string) {
+		return getGuildRepository().findUnique(createGuildID(BigInt(guildId)));
+	}
+
+	test('the request hides the guild and leaves the purge to the worker', async () => {
+		const {owner, member, guild} = await guildWithMember();
+		const dispatched = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild');
+		const leaves = vi.spyOn(NoopGatewayService.prototype, 'leaveGuild');
+		await deleteAsOwner(owner, guild.id);
+		const jobs = worker.byTask('guildProcessDeletion');
+		expect(jobs).toHaveLength(1);
+		expect(jobs[0]?.payload).toEqual({guildId: guild.id, copyMode: 'source_deleted'});
+		expect(jobs[0]?.options?.jobKey).toBe(`guild-deletion-${guild.id}`);
+		expect(dispatched).toHaveBeenCalledWith(expect.objectContaining({event: 'GUILD_DELETE'}));
+		expect(leaves).not.toHaveBeenCalled();
+		const pending = await storedGuild(guild.id);
+		expect(pending?.features.has(GuildFeatures.UNAVAILABLE_FOR_EVERYONE)).toBe(true);
+		expect(pending?.features.has(GuildFeatures.UNAVAILABLE_HIDDEN)).toBe(true);
+		await createBuilder(harness, member.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.FORBIDDEN).execute();
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+		await createBuilder(harness, owner.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.NOT_FOUND).execute();
+	});
+
+	test('an admin deletion enqueues a purge job', async () => {
+		const admin = await createTestAccount(harness);
+		await setUserACLs(harness, admin, ['admin:authenticate', 'guild:lookup', 'guild:delete']);
+		const guild = await createGuild(harness, admin.token, 'Admin Deletion Guild');
+		await createBuilder(harness, admin.token)
+			.delete(`/admin/guilds/${guild.id}`)
+			.body(null)
+			.expect(HTTP_STATUS.OK)
+			.execute();
+		expect(worker.byTask('guildProcessDeletion').map((job) => job.payload)).toEqual([
+			{guildId: guild.id, copyMode: 'purge'},
+		]);
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+	});
+
+	test('a purge interrupted by a gateway failure completes when retried', async () => {
+		const {owner, guild} = await guildWithMember();
+		await deleteAsOwner(owner, guild.id);
+		const leaveGuild = NoopGatewayService.prototype.leaveGuild;
+		const failing = vi
+			.spyOn(NoopGatewayService.prototype, 'leaveGuild')
+			.mockRejectedValueOnce(new Error('gateway timeout'))
+			.mockImplementation(function (this: NoopGatewayService, params) {
+				return leaveGuild.call(this, params);
+			});
+		await expect(processRecordedGuildDeletions(worker)).rejects.toThrow('gateway timeout');
+		expect(await storedGuild(guild.id)).not.toBeNull();
+		await processRecordedGuildDeletions(worker);
+		expect(await storedGuild(guild.id)).toBeNull();
+		await processRecordedGuildDeletions(worker);
+		expect(failing).toHaveBeenCalled();
+	});
+
+	test('member teardown runs with bounded concurrency', async () => {
+		const {owner, guild} = await guildWithMember();
+		await deleteAsOwner(owner, guild.id);
+		const guildId: GuildID = createGuildID(BigInt(guild.id));
+		const repository = getGuildRepository();
+		const realMembers = await repository.listMembers(guildId);
+		const extraMembers = Array.from({length: 120}, (_, index) => ({
+			...realMembers[0]!,
+			userId: createUserID(BigInt(realMembers[0]!.userId) + BigInt(index + 1)),
+		}));
+		vi.spyOn(Object.getPrototypeOf(repository), 'listMembers').mockResolvedValueOnce([...realMembers, ...extraMembers]);
+		let inFlight = 0;
+		let peak = 0;
+		vi.spyOn(NoopGatewayService.prototype, 'leaveGuild').mockImplementation(async () => {
+			inFlight += 1;
+			peak = Math.max(peak, inFlight);
+			await new Promise((resolve) => setTimeout(resolve, 1));
+			inFlight -= 1;
+		});
+		await createTestGuildStack().guildService.data.operations.purgeGuild(guildId, 'source_deleted');
+		expect(peak).toBeGreaterThan(1);
+		expect(peak).toBeLessThanOrEqual(25);
+		expect(await storedGuild(guild.id)).toBeNull();
+	});
+});

fluxer_api/src/api/guild/tests/GuildDeletionTestUtils.ts

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createGuildID} from '@app/api/BrandedTypes';
+import {resetApiServicesForTesting} from '@app/api/CreateApiContext';
+import type {ChannelFollowerRemovalCopyMode} from '@app/api/channel/services/ChannelFollowers';
+import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
+import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
+import {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
+
+export function recordGuildDeletions(): RecordingWorkerService {
+	const worker = new RecordingWorkerService();
+	setInjectedWorkerService(worker);
+	resetApiServicesForTesting();
+	return worker;
+}
+
+export async function processRecordedGuildDeletions(worker: RecordingWorkerService): Promise<void> {
+	const operations = createTestGuildStack().guildService.data.operations;
+	for (const job of worker.byTask('guildProcessDeletion')) {
+		await operations.purgeGuild(
+			createGuildID(BigInt(String(job.payload.guildId))),
+			job.payload.copyMode as ChannelFollowerRemovalCopyMode,
+		);
+	}
+}

fluxer_api/src/api/guild/tests/GuildFolderOperations.test.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
 import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
 import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
 import {HTTP_STATUS} from '@app/api/test/TestConstants';
@@ -113,11 +114,13 @@ describe('Guild Folder Operations', () => {
 		const invite = await createChannelInvite(harness, owner.token, channel.id);
 		await acceptInvite(harness, member1.token, invite.code);
 		await acceptInvite(harness, member2.token, invite.code);
+		const worker = recordGuildDeletions();
 		await createBuilder(harness, owner.token)
 			.post(`/guilds/${guild.id}/delete`)
 			.body({password: owner.password})
 			.expect(HTTP_STATUS.NO_CONTENT)
 			.executeWithResponse();
+		await processRecordedGuildDeletions(worker);
 		const {json: ownerSettings} = await createBuilder<UserSettingsResponse>(harness, owner.token)
 			.get('/users/@me/settings')
 			.expect(HTTP_STATUS.OK)

fluxer_api/src/api/webhook/tests/FollowerWebhookManagement.test.ts

@@ -24,6 +24,7 @@ import {
 	setupFanoutWorld,
 } from '@app/api/channel/tests/CrosspostWorkerTestUtils';
 import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils';
+import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
 import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
 import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
 import {HTTP_STATUS} from '@app/api/test/TestConstants';
@@ -295,11 +296,13 @@ describe('Follower webhook management', () => {
 		await follow(harness, b.owner.token, a.ann.id, b.t1.id);
 		await follow(harness, b.owner.token, a.ann.id, b.t2.id);
 		expect(await sourceIndex(a.ann.id)).toHaveLength(2);
+		const worker = recordGuildDeletions();
 		await createBuilder(harness, b.owner.token)
 			.post(`/guilds/${b.guild.id}/delete`)
 			.body({password: b.owner.password})
 			.expect(HTTP_STATUS.NO_CONTENT)
 			.execute();
+		await processRecordedGuildDeletions(worker);
 		expect(await sourceIndex(a.ann.id)).toEqual([]);
 	});
 

fluxer_api/src/api/worker/WorkerLaneConfig.ts

@@ -39,6 +39,7 @@ const LANE_CONFIG = {
 			'userProcessPendingDeletion',
 			'userProcessPendingDeletions',
 			'applicationProcessDeletion',
+			'guildProcessDeletion',
 			'bulkDeleteSelfMessagesImmediate',
 			'bulkDeleteUserMessages',
 			'deleteUserMessagesInGuildByTime',

fluxer_api/src/api/worker/WorkerTaskRegistry.ts

@@ -20,6 +20,7 @@ import expireReportSnapshots from '@app/api/worker/tasks/ExpireReportSnapshots';
 import expireStaleJobs from '@app/api/worker/tasks/ExpireStaleJobs';
 import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds';
 import flushUserActivityBuffer from '@app/api/worker/tasks/FlushUserActivityBuffer';
+import guildProcessDeletion from '@app/api/worker/tasks/GuildProcessDeletion';
 import handleMentionChunk from '@app/api/worker/tasks/HandleMentionChunk';
 import handleMentions from '@app/api/worker/tasks/HandleMentions';
 import harvestGuildData from '@app/api/worker/tasks/HarvestGuildData';
@@ -91,6 +92,7 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
 	expireReportSnapshots,
 	expireStaleJobs,
 	extractEmbeds,
+	guildProcessDeletion,
 	handleMentions,
 	handleMentionChunk,
 	harvestGuildData,

fluxer_api/src/api/worker/tasks/GuildProcessDeletion.ts

@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createGuildID} from '@app/api/BrandedTypes';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import {z} from 'zod';
+
+const PayloadSchema = z.object({
+	guildId: z.string(),
+	copyMode: z.enum(['source_deleted', 'purge']),
+});
+
+const guildProcessDeletion: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = PayloadSchema.parse(payload);
+	helpers.logger.debug({payload: validated}, 'Processing guildProcessDeletion task');
+	const {guildService} = getWorkerDependencies();
+	await guildService.data.operations.purgeGuild(createGuildID(BigInt(validated.guildId)), validated.copyMode);
+};
+
+export default guildProcessDeletion;

fluxer_docs/src/content/docs/admin-api/guilds.mdx

@@ -405,7 +405,7 @@ Each applied group records one Admin audit entry with the target type `guild` an
 
 <RouteHeader method="DELETE" path="/v1/admin/guilds/{guild_id}" auditReason />
 
-Permanently deletes a guild and every record it owns. Requires `guild:delete`.
+Permanently deletes a guild and every record it owns. Requires `guild:delete`. The guild becomes unavailable to every member at once, and a `guildProcessDeletion` job on the `lifecycle` lane removes its records.
 
 ### Path parameters
 

fluxer_docs/src/content/docs/admin-api/jobs.mdx

@@ -126,7 +126,7 @@ The queue drops a job 7 days after it was queued. A job still recorded as `queue
 
 Use the returned `task_type` value to filter [List jobs](#list-jobs).
 
-The tasks queued by [Bulk jobs](/admin-api/bulk-jobs/) are `bulkUpdateUserFlags`, `bulkUpdateGuildFeatures`, `bulkAddGuildMembers`, and `bulkScheduleUserDeletion`. `harvestUserData` and `harvestGuildData` build [archives](/admin-api/archives/), `sendSystemDm` delivers a [system DM broadcast](/admin-api/system-dms/), and `refreshSearchIndex` rebuilds a [search index](/admin-api/search-indexes/). `removeChannelFollowers` runs on the `crosspost` lane after an announcement channel is deleted or converted into a text channel, and removes the channel follower webhooks that follow it.
+The tasks queued by [Bulk jobs](/admin-api/bulk-jobs/) are `bulkUpdateUserFlags`, `bulkUpdateGuildFeatures`, `bulkAddGuildMembers`, and `bulkScheduleUserDeletion`. `harvestUserData` and `harvestGuildData` build [archives](/admin-api/archives/), `sendSystemDm` delivers a [system DM broadcast](/admin-api/system-dms/), and `refreshSearchIndex` rebuilds a [search index](/admin-api/search-indexes/). `guildProcessDeletion` removes the records of a deleted guild after [Delete guild](/admin-api/guilds/#delete-guild) or the owner deletes it. `removeChannelFollowers` runs on the `crosspost` lane after an announcement channel is deleted or converted into a text channel, and removes the channel follower webhooks that follow it.
 
 Thread work runs on the `batch` lane. `deleteChannelThreads` deletes the threads of a deleted parent channel, `removeThreadMembershipsForGuildMember` removes the thread memberships of a user who is no longer a guild member, and `syncThreadSearchDocument` and `backfillThreadSearch` keep the thread search index current. `archiveInactiveThreads` runs every minute with no job record and archives the threads whose auto archive time has passed. `repairThreadIndexes` rebuilds the thread list entries of the threads it names. `seedThreadPermissions` runs once for a guild. It clears the thread permission bits from every overwrite mask. It then adds `CREATE_PUBLIC_THREADS`, `CREATE_PRIVATE_THREADS`, and `SEND_MESSAGES_IN_THREADS` to each overwrite mask that has `SEND_MESSAGES`, and to the everyone role when that role has `SEND_MESSAGES`.
 

fluxer_docs/src/content/docs/http-api/guilds.mdx

@@ -739,7 +739,7 @@ Replacing an image removes the previous asset. A failed replacement leaves the p
 
 <RouteHeader method="POST" path="/v1/guilds/{guild_id}/delete" mfa />
 
-Permanently deletes the guild and returns 204 with an empty body. Requires the guild owner and sudo mode. Emits a [Guild Delete](/gateway/events/#guild-delete) Gateway event to every member session.
+Permanently deletes the guild and returns 204 with an empty body. Requires the guild owner and sudo mode. Emits a [Guild Delete](/gateway/events/#guild-delete) Gateway event to every member session. The guild is unavailable from that point, and its channels, messages and other records are removed in the background shortly after.
 
 Fluxer refuses a non-owner with 403 `MISSING_PERMISSIONS`. A bot can never own a guild, so a bot credential never satisfies the requirement.