feat: add announcement channels, publishing and following (#3090)

fluxer_api/src/api/worker/WorkerDependencies.ts

@@ -107,6 +107,7 @@ import {UserHarvestRepository} from '@app/api/user/UserHarvestRepository';
 import type {UserPermissionUtils} from '@app/api/utils/UserPermissionUtils';
 import type {VoiceRepository} from '@app/api/voice/VoiceRepository';
 import type {VoiceTopology} from '@app/api/voice/VoiceTopology';
+import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
 import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
 import type {ICacheService} from '@pkgs/cache/src/ICacheService';
 import type {IEmailService} from '@pkgs/email/src/IEmailService';
@@ -149,6 +150,7 @@ export interface WorkerDependencies {
 	emailService: IEmailService;
 	instanceConfigRepository: InstanceConfigRepository;
 	inviteService: InviteService;
+	webhookRepository: WebhookRepository;
 	workerService: IWorkerService<WorkerTaskName>;
 	unfurlerService: IUnfurlerService;
 	embedService: EmbedService;
@@ -305,6 +307,7 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS
 		emailService,
 		instanceConfigRepository,
 		inviteService,
+		webhookRepository,
 		workerService,
 		unfurlerService,
 		embedService,

fluxer_api/src/api/worker/WorkerLaneConfig.ts

@@ -96,6 +96,21 @@ const LANE_CONFIG = {
 		ackWaitMs: 120000,
 		maxDeliver: 25,
 	},
+	crosspost: {
+		consumerName: 'workers_crosspost',
+		tasks: [
+			'crosspostMessage',
+			'crosspostMessageChunk',
+			'syncCrosspostedMessage',
+			'syncCrosspostCopies',
+			'removeChannelFollowers',
+		] as const,
+		retiredTasks: [],
+		concurrency: 8,
+		maxAckPending: 64,
+		ackWaitMs: 60000,
+		maxDeliver: 25,
+	},
 } satisfies Record<APIWorkerLaneName, LaneSettings>;
 
 export type WorkerTaskName = (typeof LANE_CONFIG)[keyof typeof LANE_CONFIG]['tasks'][number];
@@ -137,6 +152,7 @@ const WORKER_LANES: ReadonlyArray<WorkerLaneDefinition> = [
 	makeLane('unfurl'),
 	makeLane('lifecycle'),
 	makeLane('batch'),
+	makeLane('crosspost'),
 ];
 const WORKER_LANE_MAP = buildWorkerLaneMap();
 const WORKER_LANE_NAMES = Object.freeze(WORKER_LANES.map((lane) => lane.name));

fluxer_api/src/api/worker/WorkerMain.ts

@@ -17,6 +17,7 @@ import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRep
 import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
 import {Logger} from '@app/api/Logger';
 import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
+import {startContentBlocklistCaches, stopContentBlocklistCaches} from '@app/api/middleware/ContentBlocklistCaches';
 import {
 	closeOwnedKVClient,
 	createSnowflakeService,
@@ -148,6 +149,7 @@ export async function startWorkerMain(): Promise<void> {
 			await jsConnectionManager?.drain();
 			jsConnectionManager = null;
 		});
+		await cleanupStep('content blocklist caches', stopContentBlocklistCaches);
 		await cleanupStep('worker dependencies', () => {
 			dependencies = null;
 			clearWorkerDependencies();
@@ -268,6 +270,8 @@ export async function startWorkerMain(): Promise<void> {
 		}
 		dependencies = await initializeWorkerDependencies(snowflakeService);
 		setWorkerDependencies(dependencies);
+		await startContentBlocklistCaches({kvClient: dependencies.kvClient, storageService: dependencies.storageService});
+		Logger.info('Content blocklist caches initialised for worker backend');
 		await queueBlocklistFeedStartupJobs(dependencies.kvClient, workerService, Config.blocklistFeeds.enabled);
 		setActivityProcessChannel('worker');
 		await startActivityEvents({

fluxer_api/src/api/worker/WorkerTaskRegistry.ts

@@ -12,6 +12,8 @@ import batchGuildAuditLogMessageDeletes from '@app/api/worker/tasks/BatchGuildAu
 import bulkDeleteSelfMessagesImmediate from '@app/api/worker/tasks/BulkDeleteSelfMessagesImmediate';
 import bulkDeleteUserMessages from '@app/api/worker/tasks/BulkDeleteUserMessages';
 import bulkDeleteUserMessagesScoped from '@app/api/worker/tasks/BulkDeleteUserMessagesScoped';
+import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage';
+import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk';
 import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime';
 import drainActivitySpool from '@app/api/worker/tasks/DrainActivitySpool';
 import expireAttachments from '@app/api/worker/tasks/ExpireAttachments';
@@ -42,7 +44,10 @@ import prunePostgresKvTtl from '@app/api/worker/tasks/PrunePostgresKvTtl';
 import reconcileUserPayments from '@app/api/worker/tasks/ReconcileUserPayments';
 import refreshSearchIndex from '@app/api/worker/tasks/RefreshSearchIndex';
 import refreshStorePurchase from '@app/api/worker/tasks/RefreshStorePurchase';
+import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers';
 import {sendSystemDm} from '@app/api/worker/tasks/SendSystemDm';
+import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies';
+import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage';
 import syncDiscoveryIndex from '@app/api/worker/tasks/SyncDiscoveryIndex';
 import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists';
 import syncUrlBlocklists from '@app/api/worker/tasks/SyncUrlBlocklists';
@@ -64,6 +69,8 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
 	bulkUpdateGuildFeatures: bulkUpdateGuildFeatures,
 	bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
 	bulkUpdateUserFlags: bulkUpdateUserFlags,
+	crosspostMessage,
+	crosspostMessageChunk,
 	deleteUserMessagesInGuildByTime,
 	drainActivitySpool,
 	expireAttachments,
@@ -93,10 +100,13 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
 	pollAppStoreNotificationHistory,
 	prunePostgresKvTtl,
 	refreshSearchIndex,
+	removeChannelFollowers,
 	sendSystemDm,
 	syncFileShaBlocklists,
 	syncUrlBlocklists,
 	syncDiscoveryIndex,
+	syncCrosspostCopies,
+	syncCrosspostedMessage,
 	flushUserActivityBuffer,
 	userProcessPendingDeletion,
 	userProcessPendingDeletions,

fluxer_api/src/api/worker/tasks/BulkDeleteSelfMessagesImmediate.ts

@@ -48,6 +48,7 @@ const bulkDeleteSelfMessagesImmediate: WorkerTaskHandler = async (payload, helpe
 		gatewayService,
 		storageService,
 		purgeQueue,
+		workerService,
 	});
 	const result = await deletionService.deleteUserMessagesFiltered(
 		userId,

fluxer_api/src/api/worker/tasks/BulkDeleteUserMessages.ts

@@ -16,7 +16,8 @@ const bulkDeleteUserMessages: WorkerTaskHandler = async (payload, helpers) => {
 	helpers.logger.debug({payload: validated}, 'Processing bulkDeleteUserMessages task');
 	const userId = createUserID(BigInt(validated.userId));
 	const scheduledAtMs = validated.scheduledAt ?? Number.POSITIVE_INFINITY;
-	const {channelRepository, gatewayService, userRepository, storageService, purgeQueue} = getWorkerDependencies();
+	const {channelRepository, gatewayService, userRepository, storageService, purgeQueue, workerService} =
+		getWorkerDependencies();
 	const user = await userRepository.findUniqueAssert(userId);
 	if (!user.pendingBulkMessageDeletionAt) {
 		Logger.debug({userId}, 'User has no pending bulk message deletion, skipping (already completed)');
@@ -27,6 +28,7 @@ const bulkDeleteUserMessages: WorkerTaskHandler = async (payload, helpers) => {
 		gatewayService,
 		storageService,
 		purgeQueue,
+		workerService,
 	});
 	const totalDeleted = await deletionService.deleteUserMessagesBulk(userId, {
 		beforeTimestamp: scheduledAtMs,

fluxer_api/src/api/worker/tasks/BulkDeleteUserMessagesScoped.ts

@@ -16,12 +16,13 @@ const bulkDeleteUserMessagesScoped: WorkerTaskHandler = async (payload, helpers)
 	const validated = PayloadSchema.parse(payload);
 	helpers.logger.debug({payload: validated}, 'Processing bulkDeleteUserMessagesScoped task');
 	const userId = createUserID(BigInt(validated.userId));
-	const {channelRepository, gatewayService, storageService, purgeQueue} = getWorkerDependencies();
+	const {channelRepository, gatewayService, storageService, purgeQueue, workerService} = getWorkerDependencies();
 	const deletionService = new UserMessageDeletionService({
 		channelRepository,
 		gatewayService,
 		storageService,
 		purgeQueue,
+		workerService,
 	});
 	const totalDeleted = await deletionService.deleteUserMessagesInScope(
 		userId,

fluxer_api/src/api/worker/tasks/CrosspostMessage.ts

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes';
+import {
+	CrosspostMessagePayloadSchema,
+	CrosspostTaskNames,
+	crosspostChunkJobKey,
+	crosspostFanoutJobKey,
+	isCrosspostedMessage,
+} from '@app/api/channel/services/message/CrosspostPropagation';
+import {chunkArray} from '@app/api/worker/tasks/utils/CrosspostWorker';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import {CROSSPOST_FANOUT_CHUNK_SIZE, CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const CHUNK_ENQUEUE_CONCURRENCY = 16;
+
+const crosspostMessage: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = CrosspostMessagePayloadSchema.parse(payload);
+	const {channelRepository, webhookRepository} = getWorkerDependencies();
+	const channelId = createChannelID(BigInt(validated.channelId));
+	const messageId = createMessageID(BigInt(validated.messageId));
+	const source = await channelRepository.messages.getMessage(channelId, messageId);
+	if (!source || !isCrosspostedMessage(source)) {
+		helpers.logger.info({messageId: validated.messageId}, 'Skipping crosspost fan-out: source is not published');
+		return;
+	}
+	const pageSize = CROSSPOST_FANOUT_PAGE_SIZE;
+	const page = await webhookRepository.listIdsBySourceChannel(channelId, {
+		afterWebhookId: validated.afterWebhookId ? createWebhookID(BigInt(validated.afterWebhookId)) : undefined,
+		limit: pageSize,
+	});
+	const chunks = chunkArray(
+		page.map((entry) => entry.webhookId.toString()),
+		CROSSPOST_FANOUT_CHUNK_SIZE,
+	);
+	for (let offset = 0; offset < chunks.length; offset += CHUNK_ENQUEUE_CONCURRENCY) {
+		await Promise.all(
+			chunks
+				.slice(offset, offset + CHUNK_ENQUEUE_CONCURRENCY)
+				.map((webhookIds) =>
+					helpers.addJob(
+						CrosspostTaskNames.CROSSPOST_MESSAGE_CHUNK,
+						{channelId: validated.channelId, messageId: validated.messageId, webhookIds, attempt: 0},
+						{jobKey: crosspostChunkJobKey(validated.messageId, webhookIds[0]!, 0), skipLedger: true},
+					),
+				),
+		);
+	}
+	if (page.length === pageSize && page.length > 0) {
+		const afterWebhookId = page[page.length - 1]!.webhookId.toString();
+		await helpers.addJob(
+			CrosspostTaskNames.CROSSPOST_MESSAGE,
+			{channelId: validated.channelId, messageId: validated.messageId, afterWebhookId},
+			{jobKey: crosspostFanoutJobKey(validated.messageId, afterWebhookId), skipLedger: true},
+		);
+	}
+};
+
+export default crosspostMessage;

fluxer_api/src/api/worker/tasks/CrosspostMessageChunk.ts

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes';
+import {
+	CrosspostMessageChunkPayloadSchema,
+	CrosspostTaskNames,
+	crosspostChunkJobKey,
+} from '@app/api/channel/services/message/CrosspostPropagation';
+import {createCrosspostDeliveryService, mapWithConcurrency} from '@app/api/worker/tasks/utils/CrosspostWorker';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import {
+	CROSSPOST_CHUNK_RETRY_DELAYS_MS,
+	CROSSPOST_FANOUT_CONCURRENCY,
+} from '@fluxer/constants/src/AnnouncementConstants';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const crosspostMessageChunk: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = CrosspostMessageChunkPayloadSchema.parse(payload);
+	const channelId = createChannelID(BigInt(validated.channelId));
+	const messageId = createMessageID(BigInt(validated.messageId));
+	const delivery = createCrosspostDeliveryService();
+	const loaded = await delivery.loadSourceForDelivery(channelId, messageId);
+	if (loaded.kind === 'missing') {
+		const {channelRepository} = getWorkerDependencies();
+		const mode = await delivery.removalModeFor(messageId);
+		for (const webhookId of validated.webhookIds) {
+			const row = await channelRepository.crossposts.get(messageId, createWebhookID(BigInt(webhookId)));
+			if (row) {
+				await delivery.syncCopy(row, mode, null);
+			}
+		}
+		return;
+	}
+	if (loaded.kind === 'inactive') {
+		helpers.logger.info({messageId: validated.messageId}, 'Skipping crosspost chunk: source is not deliverable');
+		return;
+	}
+	const failed: Array<string> = [];
+	await mapWithConcurrency(validated.webhookIds, CROSSPOST_FANOUT_CONCURRENCY, async (webhookId) => {
+		try {
+			await delivery.deliverToWebhook(loaded.context, createWebhookID(BigInt(webhookId)));
+		} catch (error) {
+			failed.push(webhookId);
+			helpers.logger.warn({error, messageId: validated.messageId, webhookId}, 'Crosspost delivery failed');
+		}
+	});
+	if (failed.length === 0) {
+		return;
+	}
+	const orderedFailed = validated.webhookIds.filter((webhookId) => failed.includes(webhookId));
+	if (validated.attempt >= CROSSPOST_CHUNK_RETRY_DELAYS_MS.length) {
+		helpers.logger.error(
+			{messageId: validated.messageId, webhookIds: orderedFailed, attempt: validated.attempt},
+			'Dropping crosspost deliveries after the last retry',
+		);
+		return;
+	}
+	const nextAttempt = validated.attempt + 1;
+	await helpers.addJob(
+		CrosspostTaskNames.CROSSPOST_MESSAGE_CHUNK,
+		{
+			channelId: validated.channelId,
+			messageId: validated.messageId,
+			webhookIds: orderedFailed,
+			attempt: nextAttempt,
+		},
+		{
+			jobKey: crosspostChunkJobKey(validated.messageId, orderedFailed[0]!, nextAttempt),
+			runAt: new Date(Date.now() + CROSSPOST_CHUNK_RETRY_DELAYS_MS[validated.attempt]!),
+			skipLedger: true,
+		},
+	);
+};
+
+export default crosspostMessageChunk;

fluxer_api/src/api/worker/tasks/ExtractEmbeds.ts

@@ -3,6 +3,11 @@
 import type {ChannelID, GuildID, MessageID} from '@app/api/BrandedTypes';
 import {createChannelID, createGuildID, createMessageID, createUserID} from '@app/api/BrandedTypes';
 import type {ChannelRepository} from '@app/api/channel/ChannelRepository';
+import {
+	enqueueCrosspostSourceRemoval,
+	enqueueCrosspostSync,
+	isCrosspostedMessage,
+} from '@app/api/channel/services/message/CrosspostPropagation';
 import {buildBroadcastMessageData} from '@app/api/channel/services/message/MessageGatewayDispatch';
 import type {MessageEmbed, MessageEmbedChild} from '@app/api/database/types/MessageTypes';
 import type {ModerationContext} from '@app/api/infrastructure/ContentModerationService';
@@ -457,6 +462,7 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => {
 			return;
 		}
 		const orderedEmbeds = buildOrderedEmbeds(urls, unfurledEmbedsByUrl);
+		let propagated: Message | null = null;
 		const handled = await withMessageWriteLock(cacheService, channelId, messageId, async () => {
 			const latestExpectedMessage = await channelRepository.getMessage(channelId, messageId);
 			if (!latestExpectedMessage) {
@@ -496,6 +502,10 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => {
 					await deleteMessageSearchDocuments([messageId], {context: {source: 'blocked_embed_unfurl'}});
 					const eventDispatcher = new ChannelEventDispatcher({gatewayService});
 					await eventDispatcher.dispatchMessageDelete(channel, messageId);
+					await enqueueCrosspostSourceRemoval(getWorkerDependencies().workerService, {
+						messages: [latestExpectedMessage],
+						mode: 'purge',
+					});
 					return true;
 				}
 				Logger.error(
@@ -507,6 +517,9 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => {
 			if (!latestMessage) {
 				return false;
 			}
+			if (latestMessage !== latestExpectedMessage) {
+				propagated = latestMessage;
+			}
 			if (!(latestMessage.flags & MessageFlags.SUPPRESS_EMBEDS)) {
 				await dispatchEmbedUpdate({
 					latestMessage,
@@ -519,6 +532,9 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => {
 			}
 			return true;
 		});
+		if (propagated && isCrosspostedMessage(propagated)) {
+			await enqueueCrosspostSync(getWorkerDependencies().workerService, {channelId, messageId, mode: 'update'});
+		}
 		if (!handled) {
 			return;
 		}

fluxer_api/src/api/worker/tasks/MessageShred.ts

@@ -2,6 +2,7 @@
 
 import type {ChannelID, MessageID} from '@app/api/BrandedTypes';
 import {createChannelID, createMessageID, createUserID} from '@app/api/BrandedTypes';
+import {enqueueCrosspostSourceRemoval} from '@app/api/channel/services/message/CrosspostPropagation';
 import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers';
 import type {Message} from '@app/api/models/Message';
 import {deleteMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup';
@@ -30,7 +31,8 @@ const STATUS_TTL_SECONDS = seconds('1 hour');
 const messageShredTask: WorkerTaskHandler = async (payload, helpers) => {
 	const data = PayloadSchema.parse(payload);
 	helpers.logger.debug({payload: data}, 'Processing messageShred task');
-	const {kvClient, channelRepository, gatewayService, storageService, purgeQueue} = getWorkerDependencies();
+	const {kvClient, channelRepository, gatewayService, storageService, purgeQueue, workerService} =
+		getWorkerDependencies();
 	const progressKey = `message_shred_status:${data.job_id}`;
 	const requestedEntries = data.entries.length;
 	const startedAt = new Date().toISOString();
@@ -127,9 +129,7 @@ const messageShredTask: WorkerTaskHandler = async (payload, helpers) => {
 				await Promise.all(
 					deletionChunk.map(
 						async ({channelId, messageId, message}: {channelId: ChannelID; messageId: MessageID; message: Message}) => {
-							if (message.attachments.length > 0) {
-								await purgeMessageAttachments(message, storageService, purgeQueue);
-							}
+							await purgeMessageAttachments(message, storageService, purgeQueue);
 							return channelRepository.deleteMessage(channelId, messageId, authorId);
 						},
 					),
@@ -139,6 +139,10 @@ const messageShredTask: WorkerTaskHandler = async (payload, helpers) => {
 					deletionChunk.map(({messageId}) => messageId),
 					{context: {source: 'message_shred', jobId: data.job_id}},
 				);
+				await enqueueCrosspostSourceRemoval(workerService, {
+					messages: deletionChunk.map(({message}) => message),
+					mode: 'purge',
+				});
 				await persistStatus('in_progress');
 				for (const {channelId, messageId} of deletionChunk) {
 					bulkDeleteDispatcher.track(channelId, messageId);

fluxer_api/src/api/worker/tasks/RemoveChannelFollowers.ts

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, type MessageID, type WebhookID} from '@app/api/BrandedTypes';
+import {
+	CrosspostTaskNames,
+	crosspostSyncJobKey,
+	RemoveChannelFollowersPayloadSchema,
+} from '@app/api/channel/services/message/CrosspostPropagation';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import {CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants';
+import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const removeChannelFollowers: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = RemoveChannelFollowersPayloadSchema.parse(payload);
+	const {channelRepository, webhookRepository, gatewayService} = getWorkerDependencies();
+	const sourceChannelId = createChannelID(BigInt(validated.sourceChannelId));
+	if (validated.reason === 'converted') {
+		const channel = await channelRepository.findUnique(sourceChannelId);
+		if (channel?.type === ChannelTypes.GUILD_ANNOUNCEMENT) {
+			helpers.logger.info(
+				{sourceChannelId: validated.sourceChannelId},
+				'Keeping channel followers: the channel is an announcement channel again',
+			);
+			return;
+		}
+	}
+	const pageSize = CROSSPOST_FANOUT_PAGE_SIZE;
+	let afterWebhookId: WebhookID | undefined;
+	let removed = 0;
+	for (;;) {
+		const page = await webhookRepository.listIdsBySourceChannel(sourceChannelId, {afterWebhookId, limit: pageSize});
+		for (const entry of page) {
+			const webhook = await webhookRepository.findUnique(entry.webhookId);
+			await webhookRepository.delete(entry.webhookId);
+			removed++;
+			const channelId = webhook?.channelId;
+			if (channelId) {
+				await gatewayService.dispatchGuild({
+					guildId: webhook.guildId ?? entry.guildId,
+					event: 'WEBHOOKS_UPDATE',
+					data: {channel_id: channelId.toString()},
+				});
+			}
+		}
+		if (page.length < pageSize || page.length === 0) break;
+		afterWebhookId = page[page.length - 1]!.webhookId;
+	}
+	let requeued = 0;
+	if (validated.copyMode) {
+		let afterMessageId: MessageID | undefined;
+		for (;;) {
+			const sources = await channelRepository.crossposts.listSourcesByChannel(sourceChannelId, {
+				afterMessageId,
+				limit: pageSize,
+			});
+			for (const messageId of sources) {
+				await helpers.addJob(
+					CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE,
+					{channelId: validated.sourceChannelId, messageId: messageId.toString(), mode: validated.copyMode},
+					{jobKey: crosspostSyncJobKey(messageId.toString(), validated.copyMode), skipLedger: true},
+				);
+				requeued++;
+			}
+			if (sources.length < pageSize || sources.length === 0) break;
+			afterMessageId = sources[sources.length - 1];
+		}
+	}
+	helpers.logger.info(
+		{sourceChannelId: validated.sourceChannelId, reason: validated.reason, removed, requeued},
+		'Removed channel followers',
+	);
+};
+
+export default removeChannelFollowers;

fluxer_api/src/api/worker/tasks/SyncCrosspostCopies.ts

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes';
+import {SyncCrosspostCopiesPayloadSchema} from '@app/api/channel/services/message/CrosspostPropagation';
+import {createCrosspostDeliveryService} from '@app/api/worker/tasks/utils/CrosspostWorker';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const syncCrosspostCopies: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = SyncCrosspostCopiesPayloadSchema.parse(payload);
+	const {channelRepository} = getWorkerDependencies();
+	const channelId = createChannelID(BigInt(validated.channelId));
+	const messageId = createMessageID(BigInt(validated.messageId));
+	const delivery = createCrosspostDeliveryService();
+	const source = validated.mode === 'update' ? await delivery.loadSource(channelId, messageId) : null;
+	if (validated.mode === 'update' && !source) {
+		return;
+	}
+	const errors: Array<unknown> = [];
+	for (const webhookId of validated.webhookIds) {
+		const row = await channelRepository.crossposts.get(messageId, createWebhookID(BigInt(webhookId)));
+		if (!row) continue;
+		try {
+			await delivery.syncCopy(row, validated.mode, source);
+		} catch (error) {
+			errors.push(error);
+			helpers.logger.warn(
+				{error, messageId: validated.messageId, webhookId, mode: validated.mode},
+				'Crosspost copy sync failed',
+			);
+		}
+	}
+	if (errors.length > 0) {
+		throw errors[0];
+	}
+};
+
+export default syncCrosspostCopies;

fluxer_api/src/api/worker/tasks/SyncCrosspostedMessage.ts

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, createMessageID, type WebhookID} from '@app/api/BrandedTypes';
+import {
+	CrosspostTaskNames,
+	crosspostSyncBucket,
+	crosspostSyncChunkJobKey,
+	SyncCrosspostedMessagePayloadSchema,
+} from '@app/api/channel/services/message/CrosspostPropagation';
+import {chunkArray, createCrosspostDeliveryService} from '@app/api/worker/tasks/utils/CrosspostWorker';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+import {CROSSPOST_FANOUT_CHUNK_SIZE, CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const syncCrosspostedMessage: WorkerTaskHandler = async (payload, helpers) => {
+	const validated = SyncCrosspostedMessagePayloadSchema.parse(payload);
+	const {channelRepository} = getWorkerDependencies();
+	const channelId = createChannelID(BigInt(validated.channelId));
+	const messageId = createMessageID(BigInt(validated.messageId));
+	if (validated.deleteSource) {
+		await createCrosspostDeliveryService().deleteSourceMessage(channelId, messageId);
+	}
+	const bucketOrMode = validated.mode === 'update' ? String(crosspostSyncBucket(Date.now())) : validated.mode;
+	const pageSize = CROSSPOST_FANOUT_PAGE_SIZE;
+	let afterWebhookId: WebhookID | undefined;
+	for (;;) {
+		const rows = await channelRepository.crossposts.listBySourceMessage(messageId, {afterWebhookId, limit: pageSize});
+		for (const chunk of chunkArray(rows, CROSSPOST_FANOUT_CHUNK_SIZE)) {
+			const webhookIds = chunk.map((row) => row.webhook_id.toString());
+			await helpers.addJob(
+				CrosspostTaskNames.SYNC_CROSSPOST_COPIES,
+				{channelId: validated.channelId, messageId: validated.messageId, mode: validated.mode, webhookIds},
+				{
+					jobKey: crosspostSyncChunkJobKey({
+						messageId: validated.messageId,
+						mode: validated.mode,
+						bucketOrMode,
+						firstWebhookId: webhookIds[0]!,
+					}),
+					skipLedger: true,
+				},
+			);
+		}
+		if (rows.length < pageSize || rows.length === 0) break;
+		afterWebhookId = rows[rows.length - 1]!.webhook_id;
+	}
+	if (validated.mode !== 'update') {
+		await channelRepository.crossposts.deleteSource({sourceChannelId: channelId, sourceMessageId: messageId});
+	}
+};
+
+export default syncCrosspostedMessage;

fluxer_api/src/api/worker/tasks/admin_bulk/BulkDeleteMessagesForUsers.ts

@@ -26,6 +26,7 @@ const handler: WorkerTaskHandler = async (rawPayload, helpers) => {
 		gatewayService: deps.gatewayService,
 		storageService: deps.storageService,
 		purgeQueue: deps.purgeQueue,
+		workerService: deps.workerService,
 	});
 	const adminUserId = createUserID(BigInt(payload.admin_user_id));
 	const total = payload.user_ids.length;

fluxer_api/src/api/worker/tasks/utils/CrosspostWorker.ts

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {CrosspostDeliveryService} from '@app/api/channel/services/message/CrosspostDeliveryService';
+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
+
+export function createCrosspostDeliveryService(): CrosspostDeliveryService {
+	const deps = getWorkerDependencies();
+	return new CrosspostDeliveryService({
+		channelRepository: deps.channelRepository,
+		webhookRepository: deps.webhookRepository,
+		userRepository: deps.userRepository,
+		guildRepository: deps.guildRepository,
+		gatewayService: deps.gatewayService,
+		storageService: deps.storageService,
+		avatarService: deps.avatarService,
+		purgeQueue: deps.purgeQueue,
+		snowflakeService: deps.snowflakeService,
+		cacheService: deps.cacheService,
+		limitConfigService: deps.limitConfigService,
+		persistenceService: deps.channelService.messages.persistence,
+		searchService: deps.channelService.messages.search,
+	});
+}
+
+export async function mapWithConcurrency<T, R>(
+	items: ReadonlyArray<T>,
+	concurrency: number,
+	fn: (item: T) => Promise<R>,
+): Promise<Array<R>> {
+	const results: Array<R> = new Array(items.length);
+	let nextIndex = 0;
+	const workers = Array.from({length: Math.min(concurrency, items.length)}, async () => {
+		while (nextIndex < items.length) {
+			const index = nextIndex++;
+			results[index] = await fn(items[index]!);
+		}
+	});
+	await Promise.all(workers);
+	return results;
+}
+
+export function chunkArray<T>(items: ReadonlyArray<T>, size: number): Array<Array<T>> {
+	const chunks: Array<Array<T>> = [];
+	for (let index = 0; index < items.length; index += size) {
+		chunks.push(items.slice(index, index + size));
+	}
+	return chunks;
+}

fluxer_api/src/api/worker/tests/CrosspostTasks.test.ts

@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createChannelID, createGuildID, createMessageID, createWebhookID} from '@app/api/BrandedTypes';
+import {ChannelRepository} from '@app/api/channel/ChannelRepository';
+import {CrosspostTaskNames} from '@app/api/channel/services/message/CrosspostPropagation';
+import {disableCrosspostWorker, follow} from '@app/api/channel/tests/AnnouncementTestUtils';
+import {
+	type FanoutWorld,
+	followInto,
+	patchChannelRow,
+	postAndPublish,
+	setupFanoutWorld,
+	sourceIndex,
+	workerHelpers,
+} from '@app/api/channel/tests/CrosspostWorkerTestUtils';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
+import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService';
+import {WebhookRepository} from '@app/api/webhook/WebhookRepository';
+import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage';
+import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk';
+import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers';
+import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies';
+import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage';
+import {
+	findLaneForTask,
+	resolveWorkerLanes,
+	validateLaneCompleteness,
+	WORKER_LANES,
+} from '@app/api/worker/WorkerLaneConfig';
+import {workerTasks} from '@app/api/worker/WorkerTaskRegistry';
+import {ChannelTypes, WebhookTypes} from '@fluxer/constants/src/ChannelConstants';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest';
+import {ZodError} from 'zod';
+
+const pageSizes = vi.hoisted(() => ({chunk: null as number | null, page: null as number | null}));
+
+vi.mock('@fluxer/constants/src/AnnouncementConstants', async (importOriginal) => {
+	const actual = await importOriginal<typeof import('@fluxer/constants/src/AnnouncementConstants')>();
+	return {
+		...actual,
+		get CROSSPOST_FANOUT_CHUNK_SIZE() {
+			return pageSizes.chunk ?? actual.CROSSPOST_FANOUT_CHUNK_SIZE;
+		},
+		get CROSSPOST_FANOUT_PAGE_SIZE() {
+			return pageSizes.page ?? actual.CROSSPOST_FANOUT_PAGE_SIZE;
+		},
+	};
+});
+
+const CROSSPOST_TASKS = Object.values(CrosspostTaskNames);
+
+describe('crosspost worker lane', () => {
+	test('every crosspost task runs in the crosspost lane', () => {
+		for (const task of CROSSPOST_TASKS) {
+			expect(findLaneForTask(task)).toBe('crosspost');
+			expect(workerTasks).toHaveProperty(task);
+		}
+		const lane = WORKER_LANES.find((entry) => entry.name === 'crosspost');
+		expect(lane).toMatchObject({
+			consumerName: 'workers_crosspost',
+			concurrency: 8,
+			maxAckPending: 64,
+			ackWaitMs: 60000,
+			maxDeliver: 25,
+		});
+		expect([...lane!.taskTypes].sort()).toEqual([...CROSSPOST_TASKS].sort());
+		expect(() => validateLaneCompleteness(workerTasks)).not.toThrow();
+	});
+
+	test('the lane resolves on its own and takes a concurrency override', () => {
+		const [lane] = resolveWorkerLanes({
+			mode: 'single_lane',
+			laneName: 'crosspost',
+			laneConcurrencyOverrides: {crosspost: 3},
+		});
+		expect(lane!.name).toBe('crosspost');
+		expect(lane!.concurrency).toBe(3);
+		const [single] = resolveWorkerLanes({
+			mode: 'single_task',
+			taskName: 'crosspostMessageChunk',
+			laneConcurrencyOverrides: {},
+		});
+		expect(single!.name).toBe('crosspost');
+		expect(single!.taskTypes).toEqual(['crosspostMessageChunk']);
+	});
+});
+
+describe('crosspost task payloads', () => {
+	const cases: Array<[string, WorkerTaskHandler, Record<string, unknown>]> = [
+		['crosspostMessage', crosspostMessage, {channelId: 1}],
+		['crosspostMessageChunk', crosspostMessageChunk, {channelId: '1', messageId: '2', webhookIds: [], attempt: 0}],
+		['crosspostMessageChunk', crosspostMessageChunk, {channelId: '1', messageId: '2', webhookIds: ['3'], attempt: -1}],
+		['syncCrosspostedMessage', syncCrosspostedMessage, {channelId: '1', messageId: '2', mode: 'rewrite'}],
+		['syncCrosspostCopies', syncCrosspostCopies, {channelId: '1', messageId: '2', mode: 'update', webhookIds: []}],
+		['removeChannelFollowers', removeChannelFollowers, {sourceChannelId: '1', reason: 'archived'}],
+		['removeChannelFollowers', removeChannelFollowers, {sourceChannelId: '1', reason: 'deleted', copyMode: 'update'}],
+	];
+
+	test.each(cases)('%s rejects a malformed payload', async (_name, handler, payload) => {
+		const worker = new SyncTaskWorkerService({}, {deferred: true});
+		await expect(handler(payload, workerHelpers(worker))).rejects.toBeInstanceOf(ZodError);
+	});
+});
+
+describe('deferred SyncTaskWorkerService', () => {
+	test('nested jobs go through the same queue and job keys dedupe', async () => {
+		const seen: Array<string> = [];
+		const worker = new SyncTaskWorkerService(
+			{
+				parent: async (payload, helpers) => {
+					seen.push(`parent:${payload.n}`);
+					await helpers.addJob('child', {n: payload.n}, {jobKey: 'child-key'});
+					await helpers.addJob('child', {n: 99}, {jobKey: 'child-key'});
+				},
+				child: async (payload) => {
+					seen.push(`child:${payload.n}`);
+				},
+			},
+			{deferred: true},
+		);
+		await worker.addJob('parent', {n: 1}, {jobKey: 'parent-key', runAt: new Date(Date.now() + 60_000)});
+		await worker.addJob('parent', {n: 2}, {jobKey: 'parent-key'});
+		expect(seen).toEqual([]);
+		expect(worker.pending).toHaveLength(1);
+		const result = await worker.drain();
+		expect(result).toEqual({ran: 2, failed: 0, remaining: 0});
+		expect(seen).toEqual(['parent:1', 'child:1']);
+		expect(worker.recorded.map((job) => job.jobKey)).toEqual(['parent-key', 'child-key']);
+		expect(worker.recorded[0]!.runAt).toBeInstanceOf(Date);
+		expect(worker.deduped.map((job) => job.payload)).toEqual([{n: 2}, {n: 99}]);
+		await worker.addJob('parent', {n: 3}, {jobKey: 'parent-key'});
+		await worker.drain();
+		expect(seen.at(-1)).toBe('child:3');
+	});
+
+	test('a failing job retries and then lands in dead letters', async () => {
+		let flakyCalls = 0;
+		let onceCalls = 0;
+		const worker = new SyncTaskWorkerService(
+			{
+				flaky: async () => {
+					flakyCalls++;
+					throw new Error('always fails');
+				},
+				once: async () => {
+					onceCalls++;
+					if (onceCalls === 1) throw new Error('fails once');
+				},
+			},
+			{deferred: true},
+		);
+		await worker.addJob('flaky', {});
+		await worker.addJob('once', {});
+		const result = await worker.drain({maxAttempts: 3});
+		expect(result.failed).toBe(4);
+		expect(flakyCalls).toBe(3);
+		expect(onceCalls).toBe(2);
+		expect(worker.deadLetters).toHaveLength(1);
+		expect(worker.deadLetters[0]!.taskType).toBe('flaky');
+		expect(worker.deadLetters[0]!.attempts).toBe(3);
+	});
+
+	test('drain stops after maxJobs and resumes later', async () => {
+		const ran: Array<number> = [];
+		const worker = new SyncTaskWorkerService(
+			{step: async (payload) => void ran.push(payload.n as number)},
+			{deferred: true},
+		);
+		for (const n of [1, 2, 3]) {
+			await worker.addJob('step', {n});
+		}
+		expect(await worker.drain({maxJobs: 2})).toEqual({ran: 2, failed: 0, remaining: 1});
+		expect(ran).toEqual([1, 2]);
+		await worker.drain();
+		expect(ran).toEqual([1, 2, 3]);
+	});
+
+	test('the default mode still runs inline and drops nested jobs', async () => {
+		const seen: Array<string> = [];
+		const worker = new SyncTaskWorkerService({
+			parent: async (_payload, helpers) => {
+				seen.push('parent');
+				expect(await helpers.addJob('child', {})).toBe(0n);
+			},
+			child: async () => {
+				seen.push('child');
+			},
+		});
+		await worker.addJob('parent', {}, {jobKey: 'k'});
+		await worker.addJob('parent', {}, {jobKey: 'k'});
+		expect(seen).toEqual(['parent', 'parent']);
+		expect(worker.recorded).toHaveLength(0);
+	});
+});
+
+describe('crosspost tasks against storage', () => {
+	let harness: ApiTestHarness;
+	let world: FanoutWorld;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		await harness.reset();
+		world = await setupFanoutWorld(harness);
+	});
+
+	afterEach(() => {
+		pageSizes.chunk = null;
+		pageSizes.page = null;
+		disableCrosspostWorker();
+		vi.restoreAllMocks();
+	});
+
+	afterAll(async () => {
+		await harness?.shutdown();
+	});
+
+	async function followerIds(): Promise<Array<string>> {
+		const entries = await new WebhookRepository().listIdsBySourceChannel(createChannelID(BigInt(world.a.ann.id)), {
+			limit: 100,
+		});
+		return entries.map((entry) => entry.webhookId.toString());
+	}
+
+	test('removeChannelFollowers removes every follower once and announces each target', async () => {
+		await followInto(harness, world, world.b.t1.id);
+		await followInto(harness, world, world.b.t2.id);
+		await patchChannelRow(world.a.ann.id, {type: ChannelTypes.GUILD_TEXT});
+		const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild');
+		const payload = {sourceChannelId: world.a.ann.id, reason: 'converted'};
+		await removeChannelFollowers(payload, workerHelpers(world.worker));
+		expect(await followerIds()).toHaveLength(0);
+		const updates = dispatchSpy.mock.calls
+			.map(([params]) => params)
+			.filter((params) => params.event === 'WEBHOOKS_UPDATE');
+		expect(updates.map((params) => (params.data as {channel_id: string}).channel_id).sort()).toEqual(
+			[world.b.t1.id, world.b.t2.id].sort(),
+		);
+		expect(updates.every((params) => params.guildId.toString() === world.b.guild.id)).toBe(true);
+		dispatchSpy.mockClear();
+		await removeChannelFollowers(payload, workerHelpers(world.worker));
+		expect(dispatchSpy.mock.calls.filter(([params]) => params.event === 'WEBHOOKS_UPDATE')).toHaveLength(0);
+		const webhooks = await new WebhookRepository().listByChannel(createChannelID(BigInt(world.b.t1.id)));
+		expect(webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)).toHaveLength(0);
+	});
+
+	test('removeChannelFollowers keeps followers when the channel is an announcement channel again', async () => {
+		await followInto(harness, world, world.b.t1.id);
+		await removeChannelFollowers({sourceChannelId: world.a.ann.id, reason: 'converted'}, workerHelpers(world.worker));
+		expect(await followerIds()).toHaveLength(1);
+	});
+
+	test('removeChannelFollowers with a copy mode queues one sync per published message, across pages', async () => {
+		await followInto(harness, world, world.b.t1.id);
+		const published = [];
+		for (const content of ['one', 'two', 'three']) {
+			published.push(await postAndPublish(harness, world, {content}));
+		}
+		expect((await sourceIndex(world.a.ann.id)).sort()).toEqual(published.map((message) => message.id).sort());
+		pageSizes.page = 2;
+		const before = world.worker.recorded.length;
+		await removeChannelFollowers(
+			{sourceChannelId: world.a.ann.id, reason: 'deleted', copyMode: 'purge'},
+			workerHelpers(world.worker),
+		);
+		const queued = world.worker.recorded.slice(before);
+		expect(queued.every((job) => job.taskType === CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE)).toBe(true);
+		expect(queued.map((job) => (job.payload as {messageId: string}).messageId).sort()).toEqual(
+			published.map((message) => message.id).sort(),
+		);
+		expect(queued.map((job) => job.payload.mode)).toEqual(['purge', 'purge', 'purge']);
+		expect(queued.map((job) => job.jobKey).sort()).toEqual(
+			published.map((message) => `crosspost-sync:${message.id}:purge`).sort(),
+		);
+		expect(await followerIds()).toHaveLength(0);
+	});
+
+	test('syncCrosspostedMessage pages through every mapping row', async () => {
+		const source = await postAndPublish(harness, world, {content: 'many followers'});
+		const repository = new ChannelRepository();
+		const webhookIds = [
+			'900000000000000001',
+			'900000000000000002',
+			'900000000000000003',
+			'900000000000000004',
+			'900000000000000005',
+		];
+		for (const webhookId of webhookIds) {
+			await repository.crossposts.insertPending({
+				source_message_id: createMessageID(BigInt(source.id)),
+				webhook_id: createWebhookID(BigInt(webhookId)),
+				source_channel_id: createChannelID(BigInt(world.a.ann.id)),
+				target_guild_id: createGuildID(BigInt(world.b.guild.id)),
+				target_channel_id: createChannelID(BigInt(world.b.t1.id)),
+				target_message_id: createMessageID(BigInt(webhookId)),
+				state: 'pending',
+				reserved_at: new Date(Date.now() - 120_000),
+				source_fingerprint: null,
+				created_at: new Date(),
+			});
+		}
+		pageSizes.page = 2;
+		pageSizes.chunk = 1;
+		const before = world.worker.recorded.length;
+		await syncCrosspostedMessage(
+			{channelId: world.a.ann.id, messageId: source.id, mode: 'source_deleted'},
+			workerHelpers(world.worker),
+		);
+		const chunks = world.worker.recorded.slice(before);
+		expect(chunks.map((job) => job.taskType)).toEqual(Array(5).fill(CrosspostTaskNames.SYNC_CROSSPOST_COPIES));
+		expect(chunks.map((job) => (job.payload as {webhookIds: Array<string>}).webhookIds)).toEqual(
+			webhookIds.map((webhookId) => [webhookId]),
+		);
+		expect(chunks.map((job) => job.jobKey)).toEqual(
+			webhookIds.map((webhookId) => `crosspost-sync-chunk:${source.id}:source_deleted:source_deleted:${webhookId}`),
+		);
+		expect(await sourceIndex(world.a.ann.id)).not.toContain(source.id);
+		await world.worker.drain();
+		expect(world.worker.deadLetters).toHaveLength(0);
+		const remaining = await repository.crossposts.listBySourceMessage(createMessageID(BigInt(source.id)), {
+			limit: 100,
+		});
+		expect(remaining).toHaveLength(0);
+	});
+
+	test('crosspostMessage does nothing for a message that is not published', async () => {
+		await follow(harness, world.b.owner.token, world.a.ann.id, world.b.t1.id);
+		await world.worker.drain();
+		const before = world.worker.recorded.length;
+		await crosspostMessage({channelId: world.a.ann.id, messageId: '123456789012345678'}, workerHelpers(world.worker));
+		expect(world.worker.recorded.length).toBe(before);
+	});
+});

fluxer_api/src/api/worker/tests/WorkerLaneConfig.test.ts

@@ -72,6 +72,7 @@ describe('WorkerLaneConfig', () => {
 			unfurl: [],
 			lifecycle: ['sendScheduledMessage'],
 			batch: [],
+			crosspost: [],
 		});
 		for (const lane of lanes) {
 			for (const task of lane.retiredTaskTypes) expect(lane.taskTypes).not.toContain(task);