From 7e4d5137f89b889f25c58d63cfc5d04ff8b609ab Mon Sep 17 00:00:00 2001 From: Hampus Date: Thu, 1 Oct 2026 16:57:21 +0200 Subject: [PATCH] feat: add announcement channels, publishing and following (#3090) --- fluxer_admin/build.rs | 18 + fluxer_admin/openapi-admin.json | 41 +- .../src/templates/pages/bulk_actions.rs | 1 + .../pages/guild_detail_tabs/features.rs | 1 + .../pages/guild_detail_tabs/overview.rs | 1 + fluxer_api/src/api/Config.ts | 1 + fluxer_api/src/api/Tables.ts | 36 + .../api/admin/services/AdminMessageService.ts | 12 +- .../AdminSyntheticUserServiceGuards.test.ts | 2 + fluxer_api/src/api/channel/ChannelMappers.ts | 1 + .../src/api/channel/ChannelRepository.ts | 4 + .../channel/controllers/ChannelController.ts | 20 +- .../controllers/ChannelFollowController.ts | 72 ++ .../channel/controllers/MessageController.ts | 55 + .../src/api/channel/controllers/index.ts | 2 + .../channel/repositories/ChannelRepository.ts | 3 + .../CrosspostedMessageRepository.test.ts | 241 ++++ .../CrosspostedMessageRepository.ts | 185 +++ .../IChannelRepositoryAggregate.ts | 2 + .../ICrosspostedMessageRepository.ts | 57 + .../services/AttachmentUploadService.ts | 51 +- .../services/BaseChannelAuthService.ts | 1 + .../channel/services/ChannelDataService.ts | 11 +- .../api/channel/services/ChannelFollowers.ts | 97 ++ .../channel/services/ChannelRequestService.ts | 3 + .../api/channel/services/ChannelService.ts | 2 + .../api/channel/services/MessageService.ts | 22 +- .../channel_data/ChannelOperationsService.ts | 60 +- .../message/CrosspostDeliveryService.ts | 902 +++++++++++++ .../services/message/CrosspostEmbedObjects.ts | 58 + .../services/message/CrosspostPropagation.ts | 287 ++++ .../message/CrosspostSourceService.ts | 104 ++ .../message/MessageCrosspostService.ts | 218 ++++ .../services/message/MessageDeleteService.ts | 22 + .../services/message/MessageEditService.ts | 82 +- .../message/MessageEmbedAttachmentResolver.ts | 10 +- .../services/message/MessageHelpers.test.ts | 38 +- .../services/message/MessageHelpers.ts | 205 ++- .../services/message/MessageHistoryCutoff.ts | 22 + .../message/MessagePersistenceService.ts | 21 +- .../message/MessageProcessingService.ts | 7 +- .../services/message/MessageRequestService.ts | 33 + .../message/MessageRetrievalService.ts | 14 +- .../services/message/MessageSendService.ts | 65 +- .../services/message/MessageWriteLock.ts | 59 + .../message/UserMessageDeletionService.ts | 10 + .../AnnouncementChannelConversion.test.ts | 479 +++++++ .../tests/AnnouncementChannelType.test.ts | 285 ++++ .../channel/tests/AnnouncementTestUtils.ts | 277 ++++ .../tests/CrosspostConcurrency.test.ts | 245 ++++ .../channel/tests/CrosspostModeration.test.ts | 633 +++++++++ .../tests/CrosspostPropagation.test.ts | 854 ++++++++++++ .../channel/tests/CrosspostRateLimits.test.ts | 334 +++++ .../api/channel/tests/CrosspostSource.test.ts | 270 ++++ .../api/channel/tests/CrosspostTestUtils.ts | 278 ++++ .../channel/tests/CrosspostWorkerTestUtils.ts | 206 +++ .../tests/EmbedAttachmentPurge.test.ts | 330 +++++ .../channel/tests/MessageCrosspost.test.ts | 548 ++++++++ .../tests/MessageCrosspostFanout.test.ts | 1060 +++++++++++++++ .../channel/utils/CrosspostContentRules.ts | 50 + fluxer_api/src/api/config/APIConfig.ts | 3 +- .../src/api/csam/NcmecSubmissionService.ts | 64 +- .../src/api/database/types/ChannelTypes.ts | 51 + .../src/api/database/types/MessageTypes.ts | 2 +- .../api/favorite_meme/FavoriteMemeService.ts | 14 +- .../guild/services/channel/ChannelHelpers.ts | 17 +- .../channel/ChannelOperationsService.ts | 13 +- .../services/data/GuildOperationsService.ts | 34 +- .../GuildDeletionAttachmentPurge.test.ts | 148 +++ .../guild/tests/GuildTemplateImport.test.ts | 4 +- .../src/api/infrastructure/AvatarService.ts | 39 + .../api/middleware/ContentBlocklistCaches.ts | 25 + .../src/api/middleware/ServiceMiddleware.ts | 25 + fluxer_api/src/api/models/MessageRef.ts | 4 +- fluxer_api/src/api/models/Webhook.ts | 6 + fluxer_api/src/api/openapi/openapi.json | 706 +++++++++- .../ChannelRateLimitConfig.ts | 16 + fluxer_api/src/api/report/ReportService.ts | 110 +- .../api/report/tests/CrosspostReport.test.ts | 231 ++++ .../src/api/test/SyncTaskWorkerService.ts | 136 +- ...ositoryBackedMessageResponseDataService.ts | 21 +- fluxer_api/src/api/types/HonoEnv.ts | 2 + .../src/api/webhook/ChannelFollowService.ts | 279 ++++ .../src/api/webhook/IWebhookRepository.ts | 11 + .../src/api/webhook/WebhookController.ts | 3 +- fluxer_api/src/api/webhook/WebhookModel.ts | 113 +- .../src/api/webhook/WebhookRepository.test.ts | 145 +++ .../src/api/webhook/WebhookRepository.ts | 84 +- .../src/api/webhook/WebhookRequestService.ts | 16 + fluxer_api/src/api/webhook/WebhookService.ts | 184 ++- .../api/webhook/tests/ChannelFollow.test.ts | 423 ++++++ .../tests/ChannelFollowPermissions.test.ts | 273 ++++ .../tests/FollowerWebhookManagement.test.ts | 532 ++++++++ .../src/api/webhook/tests/WebhookTestUtils.ts | 13 +- .../src/api/worker/WorkerDependencies.ts | 3 + fluxer_api/src/api/worker/WorkerLaneConfig.ts | 16 + fluxer_api/src/api/worker/WorkerMain.ts | 4 + .../src/api/worker/WorkerTaskRegistry.ts | 10 + .../tasks/BulkDeleteSelfMessagesImmediate.ts | 1 + .../worker/tasks/BulkDeleteUserMessages.ts | 4 +- .../tasks/BulkDeleteUserMessagesScoped.ts | 3 +- .../src/api/worker/tasks/CrosspostMessage.ts | 60 + .../api/worker/tasks/CrosspostMessageChunk.ts | 75 ++ .../src/api/worker/tasks/ExtractEmbeds.ts | 16 + .../src/api/worker/tasks/MessageShred.ts | 12 +- .../worker/tasks/RemoveChannelFollowers.ts | 75 ++ .../api/worker/tasks/SyncCrosspostCopies.ts | 38 + .../worker/tasks/SyncCrosspostedMessage.ts | 52 + .../admin_bulk/BulkDeleteMessagesForUsers.ts | 1 + .../api/worker/tasks/utils/CrosspostWorker.ts | 48 + .../api/worker/tests/CrosspostTasks.test.ts | 337 +++++ .../api/worker/tests/WorkerLaneConfig.test.ts | 1 + .../alerts/RateLimitedConfirmModal.tsx | 15 +- .../dialogs/shared/UseForwardDestinations.ts | 1 + .../layout/dnd/ChannelReorderStateMachine.ts | 6 +- .../layout/utils/ChannelMoveOperation.ts | 4 +- .../layout/utils/ChannelOrganization.ts | 4 +- .../src/features/app/constants/Endpoints.ts | 6 + .../channel/commands/ChannelCommands.ts | 7 + .../channel/commands/ChannelFollowCommands.ts | 23 + .../commands/CrosspostSourceCommands.ts | 20 + .../components/ChannelFollowAddMessage.tsx | 60 + .../components/ChannelHeader.module.css | 5 + .../channel/components/ChannelHeader.tsx | 8 + .../channel/components/ChannelMessage.tsx | 8 +- .../components/ChannelMessages.module.css | 6 +- .../channel/components/ChannelMessages.tsx | 19 +- .../channel/components/ChannelUserTag.tsx | 20 +- .../components/CompactMessageLayout.tsx | 3 +- .../CrosspostCommunityCard.module.css | 158 +++ .../components/CrosspostCommunityCard.tsx | 287 ++++ .../components/CrosspostCommunityPopout.tsx | 98 ++ .../CrosspostPublishNudge.module.css | 80 ++ .../components/CrosspostPublishNudge.tsx | 93 ++ .../components/MessageActionBar.module.css | 11 +- .../channel/components/MessageActionBar.tsx | 50 +- .../channel/components/MessageActionMenu.tsx | 15 + .../channel/components/MessageActionUtils.tsx | 44 + .../channel/components/MessageAttachments.tsx | 21 +- .../channel/components/MessageAuthorInfo.tsx | 3 +- .../channel/components/MessageAvatar.tsx | 24 + .../channel/components/MessageUsername.tsx | 48 +- .../channel/components/MessageViewContext.ts | 1 + .../channel/components/ReplyPreview.tsx | 2 +- .../channel/components/UserMessage.tsx | 70 +- .../components/barriers/BarrierComponents.tsx | 33 + .../ChannelDetailsBottomSheet.tsx | 27 + .../channel_header/useChannelHeaderData.ts | 4 + .../ChannelFollowButton.tsx | 30 + .../channel_view/GuildChannelView.tsx | 30 +- .../modals/ChannelFollowModal.module.css | 178 +++ .../components/modals/ChannelFollowModal.tsx | 267 ++++ .../ChannelFollowSuccessModal.module.css | 90 ++ .../modals/ChannelFollowSuccessModal.tsx | 122 ++ .../channel_tabs/ChannelOverviewTab.tsx | 165 ++- .../ChannelWebhooksTab.module.css | 19 + .../channel_tabs/ChannelWebhooksTab.tsx | 46 +- .../channel_overview_tab/shared.ts | 1 + .../src/features/channel/models/Channel.ts | 5 + .../channel/state/CrosspostSourceCards.ts | 63 + .../features/channel/state/PublishNudge.ts | 64 + .../channel/utils/ChannelCreateModalUtils.ts | 5 + .../channel/utils/ChannelFollowUtils.ts | 104 ++ .../features/channel/utils/ChannelUtils.tsx | 33 + .../features/channel/utils/CrosspostUtils.ts | 17 + .../discovery/DiscoveryGuildCard.module.css | 56 - .../discovery/DiscoveryGuildCard.tsx | 19 +- .../utils/DiscoveryBannerTint.module.css | 57 + .../discovery/utils/DiscoveryBannerTint.ts | 21 + .../modals/AddFavoriteChannelModal.tsx | 1 + .../add_guild_modal/TemplateImportForm.tsx | 4 +- .../modals/add_guild_modal/shared.ts | 5 +- .../guild_tabs/GuildAuditLogSentence.tsx | 5 +- .../guild_tabs/GuildWebhooksTab.module.css | 33 + .../modals/guild_tabs/GuildWebhooksTab.tsx | 78 +- .../guild_tabs/GuildOverviewTabUtils.tsx | 4 +- .../audit_log/AuditLogChannelMessages.ts | 40 + .../audit_log/AuditLogChannelPresentation.ts | 33 +- .../AuditLogInviteWebhookMessages.ts | 10 + .../AuditLogInviteWebhookPresentation.ts | 12 +- .../guild_tabs/audit_log/AuditLogValues.ts | 4 +- .../src/features/i18n/locales/ar/messages.po | 1152 +++++++++++------ .../locales/auto-i18n-reviewed-unchanged.json | 252 ++++ .../src/features/i18n/locales/bg/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/cs/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/da/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/de/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/el/messages.po | 1152 +++++++++++------ .../features/i18n/locales/en-GB/messages.po | 1152 +++++++++++------ .../features/i18n/locales/en-US/messages.po | 1152 +++++++++++------ .../features/i18n/locales/es-419/messages.po | 1152 +++++++++++------ .../features/i18n/locales/es-ES/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/fi/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/fr/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/he/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/hi/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/hr/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/hu/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/id/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/it/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/ja/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/ko/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/lt/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/nl/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/no/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/pl/messages.po | 1152 +++++++++++------ .../features/i18n/locales/pt-BR/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/ro/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/ru/messages.po | 1152 +++++++++++------ .../features/i18n/locales/sv-SE/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/th/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/tr/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/uk/messages.po | 1152 +++++++++++------ .../src/features/i18n/locales/vi/messages.po | 1152 +++++++++++------ .../features/i18n/locales/zh-CN/messages.po | 1152 +++++++++++------ .../features/i18n/locales/zh-TW/messages.po | 1152 +++++++++++------ .../i18n/utils/CommonMessageDescriptors.ts | 5 + .../src/features/invite/utils/InviteUtils.ts | 4 +- .../messaging/commands/MessageCommands.tsx | 161 ++- .../alerts/MessageCrosspostLimitModal.tsx | 38 + .../alerts/PublishedMessageEditLimitModal.tsx | 39 + .../markdown/renderers/MentionRenderer.tsx | 1 + .../modals/MessageCrosspostConfirmModal.tsx | 45 + .../messaging/hooks/useDeleteAttachment.tsx | 5 +- .../messaging/hooks/useTextareaSubmit.ts | 54 +- .../messaging/models/MessagingMessage.ts | 12 + .../messaging/state/MessageReferences.ts | 24 +- .../messaging/utils/MessageComponentUtils.tsx | 8 + .../messaging/utils/MessageGroupingUtils.ts | 3 + .../messaging/utils/MessageSubmitUtils.ts | 4 +- .../messaging/utils/SystemMessageUtils.tsx | 7 + .../messaging/utils/markdown/Plaintext.ts | 1 + .../notification/state/MentionFeed.ts | 1 + .../platform/transport/RestTransport.ts | 32 +- .../state/QuickSwitcherCandidateBuilder.ts | 3 +- .../state/QuickSwitcherResultGenerators.ts | 6 +- .../search/utils/DestinationSearchSources.ts | 8 +- .../ui/action_menu/ContextMenuIcons.tsx | 7 + .../ui/action_menu/MessageContextMenu.tsx | 4 +- .../ui/action_menu/items/ChannelMenuData.tsx | 34 +- .../components/form/FormCombobox.module.css | 7 + .../icons/AnnouncementChannelIcon.tsx | 31 + .../icons/LockedAnnouncementChannelIcon.tsx | 47 + .../icons/MatureAnnouncementChannelIcon.tsx | 47 + .../src/features/ui/state/Notification.ts | 9 +- .../FollowedChannelListItem.module.css | 12 + .../components/FollowedChannelListItem.tsx | 232 ++++ .../webhook/components/WebhookListItem.tsx | 3 +- .../webhook/hooks/useWebhookUpdates.tsx | 3 +- .../src/features/webhook/models/Webhook.ts | 46 +- fluxer_docs/astro.config.ts | 1 + fluxer_docs/scripts/VerifyDocsStyle.ts | 2 +- .../src/content/docs/admin-api/jobs.mdx | 5 +- .../src/content/docs/gateway/events.md | 10 +- .../src/content/docs/http-api/channels.mdx | 169 ++- .../src/content/docs/http-api/errors.md | 44 + .../docs/http-api/guild-audit-logs.mdx | 12 +- .../content/docs/http-api/guild-channels.mdx | 7 +- .../src/content/docs/http-api/guilds.mdx | 13 +- .../src/content/docs/http-api/messages.mdx | 152 ++- .../src/content/docs/http-api/permissions.mdx | 13 +- .../docs/http-api/users/settings-protobuf.md | 11 + .../src/content/docs/http-api/webhooks.mdx | 110 +- .../content/docs/operator/configuration.mdx | 4 +- .../docs/topics/announcement-channels.md | 104 ++ .../src/content/docs/topics/rate-limits.md | 19 +- .../src/guild/guild_data_channels.erl | 2 +- .../src/guild/guild_data_channels_order.erl | 2 +- .../src/guild/guild_state_channels.erl | 45 +- .../src/push/push_message_params.erl | 2 +- .../src/push/push_notification_format.erl | 7 + fluxer_gateway/test/guild_data_tests.erl | 51 + fluxer_messages/benches/message_response.rs | 2 +- fluxer_messages/src/shard_impl.rs | 399 +++++- fluxer_messages/src/types.rs | 29 +- fluxer_svc/Cargo.toml | 3 + fluxer_svc/src/router.rs | 200 ++- packages/config/src/ConfigLoader.ts | 2 +- packages/config/src/MasterConfig.ts | 3 +- .../config/src/__tests__/ConfigLoader.test.ts | 16 + .../constants/src/AnnouncementConstants.ts | 17 + packages/constants/src/ApiErrorCodes.ts | 11 + .../constants/src/ChannelConstants.test.ts | 49 + packages/constants/src/ChannelConstants.ts | 27 +- packages/constants/src/GuildConstants.ts | 1 + .../src/__tests__/AnnouncementErrors.test.ts | 48 + .../AnnouncementChannelRequiredError.ts | 12 + .../channel/ChannelAlreadyFollowedError.ts | 12 + .../ChannelHasFollowedChannelsError.ts | 12 + .../ChannelTypeConversionNotSupportedError.ts | 12 + ...FollowTargetContentWarningRequiredError.ts | 12 + .../FollowTargetNotAgeRestrictedError.ts | 12 + .../InvalidFollowTargetChannelError.ts | 12 + .../channel/MessageAlreadyCrosspostedError.ts | 12 + .../channel/MessageNotCrosspostableError.ts | 12 + packages/errors/src/i18n/ErrorCodeMappings.ts | 11 + packages/errors/src/i18n/ErrorI18nMessages.ts | 18 + packages/errors/src/i18n/locales/ar.ts | 11 + packages/errors/src/i18n/locales/bg.ts | 11 + packages/errors/src/i18n/locales/cs.ts | 11 + packages/errors/src/i18n/locales/da.ts | 11 + packages/errors/src/i18n/locales/de.ts | 11 + packages/errors/src/i18n/locales/el.ts | 11 + packages/errors/src/i18n/locales/en-GB.ts | 11 + packages/errors/src/i18n/locales/es-419.ts | 11 + packages/errors/src/i18n/locales/es-ES.ts | 11 + packages/errors/src/i18n/locales/fi.ts | 11 + packages/errors/src/i18n/locales/fr.ts | 11 + packages/errors/src/i18n/locales/he.ts | 11 + packages/errors/src/i18n/locales/hi.ts | 11 + packages/errors/src/i18n/locales/hr.ts | 11 + packages/errors/src/i18n/locales/hu.ts | 11 + packages/errors/src/i18n/locales/id.ts | 11 + packages/errors/src/i18n/locales/it.ts | 11 + packages/errors/src/i18n/locales/ja.ts | 11 + packages/errors/src/i18n/locales/ko.ts | 11 + packages/errors/src/i18n/locales/lt.ts | 11 + packages/errors/src/i18n/locales/nl.ts | 11 + packages/errors/src/i18n/locales/no.ts | 11 + packages/errors/src/i18n/locales/pl.ts | 11 + packages/errors/src/i18n/locales/pt-BR.ts | 11 + packages/errors/src/i18n/locales/ro.ts | 11 + packages/errors/src/i18n/locales/ru.ts | 11 + packages/errors/src/i18n/locales/sv-SE.ts | 11 + packages/errors/src/i18n/locales/th.ts | 11 + packages/errors/src/i18n/locales/tr.ts | 11 + packages/errors/src/i18n/locales/uk.ts | 11 + packages/errors/src/i18n/locales/vi.ts | 11 + packages/errors/src/i18n/locales/zh-CN.ts | 11 + packages/errors/src/i18n/locales/zh-TW.ts | 11 + .../errors/src/i18n/weblate/locales/ar.json | 11 + .../errors/src/i18n/weblate/locales/bg.json | 11 + .../errors/src/i18n/weblate/locales/cs.json | 11 + .../errors/src/i18n/weblate/locales/da.json | 11 + .../errors/src/i18n/weblate/locales/de.json | 11 + .../errors/src/i18n/weblate/locales/el.json | 11 + .../src/i18n/weblate/locales/en-GB.json | 11 + .../src/i18n/weblate/locales/es-419.json | 11 + .../src/i18n/weblate/locales/es-ES.json | 11 + .../errors/src/i18n/weblate/locales/fi.json | 11 + .../errors/src/i18n/weblate/locales/fr.json | 11 + .../errors/src/i18n/weblate/locales/he.json | 11 + .../errors/src/i18n/weblate/locales/hi.json | 11 + .../errors/src/i18n/weblate/locales/hr.json | 11 + .../errors/src/i18n/weblate/locales/hu.json | 11 + .../errors/src/i18n/weblate/locales/id.json | 11 + .../errors/src/i18n/weblate/locales/it.json | 11 + .../errors/src/i18n/weblate/locales/ja.json | 11 + .../errors/src/i18n/weblate/locales/ko.json | 11 + .../errors/src/i18n/weblate/locales/lt.json | 11 + .../errors/src/i18n/weblate/locales/nl.json | 11 + .../errors/src/i18n/weblate/locales/no.json | 11 + .../errors/src/i18n/weblate/locales/pl.json | 11 + .../src/i18n/weblate/locales/pt-BR.json | 11 + .../errors/src/i18n/weblate/locales/ro.json | 11 + .../errors/src/i18n/weblate/locales/ru.json | 11 + .../src/i18n/weblate/locales/sv-SE.json | 11 + .../errors/src/i18n/weblate/locales/th.json | 11 + .../errors/src/i18n/weblate/locales/tr.json | 11 + .../errors/src/i18n/weblate/locales/uk.json | 11 + .../errors/src/i18n/weblate/locales/vi.json | 11 + .../src/i18n/weblate/locales/zh-CN.json | 11 + .../src/i18n/weblate/locales/zh-TW.json | 11 + .../errors/src/i18n/weblate/messages.json | 11 + .../src/__tests__/LimitDefaults.test.ts | 11 + .../src/__tests__/OpenAPIGenerator.test.ts | 8 +- .../user/preferences/v1/preferences.proto | 5 + .../domains/channel/ChannelFollowSchemas.ts | 24 + .../domains/channel/ChannelRequestSchemas.ts | 34 +- .../domains/channel/GuildChannelOrdering.ts | 7 +- .../src/domains/guild/GuildResponseSchemas.ts | 5 + .../domains/message/CrosspostSourceSchemas.ts | 32 + .../domains/message/MessageResponseSchemas.ts | 6 +- .../tests/AnnouncementChannelSchemas.test.ts | 132 ++ .../tests/GuildChannelOrdering.test.ts | 15 + .../tests/RequestTransportSchemas.test.ts | 7 +- .../src/domains/webhook/WebhookSchemas.ts | 56 +- .../user/preferences/v1/preferences_pb.ts | 40 +- .../src/primitives/ChannelValidators.ts | 5 + .../src/primitives/MessageValidators.ts | 5 + tools/dev/cassandra_target_schema.json | 89 ++ 381 files changed, 45650 insertions(+), 14076 deletions(-) create mode 100644 fluxer_api/src/api/channel/controllers/ChannelFollowController.ts create mode 100644 fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.test.ts create mode 100644 fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.ts create mode 100644 fluxer_api/src/api/channel/repositories/ICrosspostedMessageRepository.ts create mode 100644 fluxer_api/src/api/channel/services/ChannelFollowers.ts create mode 100644 fluxer_api/src/api/channel/services/message/CrosspostDeliveryService.ts create mode 100644 fluxer_api/src/api/channel/services/message/CrosspostEmbedObjects.ts create mode 100644 fluxer_api/src/api/channel/services/message/CrosspostPropagation.ts create mode 100644 fluxer_api/src/api/channel/services/message/CrosspostSourceService.ts create mode 100644 fluxer_api/src/api/channel/services/message/MessageCrosspostService.ts create mode 100644 fluxer_api/src/api/channel/services/message/MessageHistoryCutoff.ts create mode 100644 fluxer_api/src/api/channel/services/message/MessageWriteLock.ts create mode 100644 fluxer_api/src/api/channel/tests/AnnouncementChannelConversion.test.ts create mode 100644 fluxer_api/src/api/channel/tests/AnnouncementChannelType.test.ts create mode 100644 fluxer_api/src/api/channel/tests/AnnouncementTestUtils.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostConcurrency.test.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostModeration.test.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostRateLimits.test.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostSource.test.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostTestUtils.ts create mode 100644 fluxer_api/src/api/channel/tests/CrosspostWorkerTestUtils.ts create mode 100644 fluxer_api/src/api/channel/tests/EmbedAttachmentPurge.test.ts create mode 100644 fluxer_api/src/api/channel/tests/MessageCrosspost.test.ts create mode 100644 fluxer_api/src/api/channel/tests/MessageCrosspostFanout.test.ts create mode 100644 fluxer_api/src/api/channel/utils/CrosspostContentRules.ts create mode 100644 fluxer_api/src/api/guild/tests/GuildDeletionAttachmentPurge.test.ts create mode 100644 fluxer_api/src/api/middleware/ContentBlocklistCaches.ts create mode 100644 fluxer_api/src/api/report/tests/CrosspostReport.test.ts create mode 100644 fluxer_api/src/api/webhook/ChannelFollowService.ts create mode 100644 fluxer_api/src/api/webhook/WebhookRepository.test.ts create mode 100644 fluxer_api/src/api/webhook/tests/ChannelFollow.test.ts create mode 100644 fluxer_api/src/api/webhook/tests/ChannelFollowPermissions.test.ts create mode 100644 fluxer_api/src/api/webhook/tests/FollowerWebhookManagement.test.ts create mode 100644 fluxer_api/src/api/worker/tasks/CrosspostMessage.ts create mode 100644 fluxer_api/src/api/worker/tasks/CrosspostMessageChunk.ts create mode 100644 fluxer_api/src/api/worker/tasks/RemoveChannelFollowers.ts create mode 100644 fluxer_api/src/api/worker/tasks/SyncCrosspostCopies.ts create mode 100644 fluxer_api/src/api/worker/tasks/SyncCrosspostedMessage.ts create mode 100644 fluxer_api/src/api/worker/tasks/utils/CrosspostWorker.ts create mode 100644 fluxer_api/src/api/worker/tests/CrosspostTasks.test.ts create mode 100644 fluxer_app/src/features/channel/commands/ChannelFollowCommands.ts create mode 100644 fluxer_app/src/features/channel/commands/CrosspostSourceCommands.ts create mode 100644 fluxer_app/src/features/channel/components/ChannelFollowAddMessage.tsx create mode 100644 fluxer_app/src/features/channel/components/CrosspostCommunityCard.module.css create mode 100644 fluxer_app/src/features/channel/components/CrosspostCommunityCard.tsx create mode 100644 fluxer_app/src/features/channel/components/CrosspostCommunityPopout.tsx create mode 100644 fluxer_app/src/features/channel/components/CrosspostPublishNudge.module.css create mode 100644 fluxer_app/src/features/channel/components/CrosspostPublishNudge.tsx create mode 100644 fluxer_app/src/features/channel/components/channel_header_components/ChannelFollowButton.tsx create mode 100644 fluxer_app/src/features/channel/components/modals/ChannelFollowModal.module.css create mode 100644 fluxer_app/src/features/channel/components/modals/ChannelFollowModal.tsx create mode 100644 fluxer_app/src/features/channel/components/modals/ChannelFollowSuccessModal.module.css create mode 100644 fluxer_app/src/features/channel/components/modals/ChannelFollowSuccessModal.tsx create mode 100644 fluxer_app/src/features/channel/state/CrosspostSourceCards.ts create mode 100644 fluxer_app/src/features/channel/state/PublishNudge.ts create mode 100644 fluxer_app/src/features/channel/utils/ChannelFollowUtils.ts create mode 100644 fluxer_app/src/features/channel/utils/CrosspostUtils.ts create mode 100644 fluxer_app/src/features/discovery/utils/DiscoveryBannerTint.module.css create mode 100644 fluxer_app/src/features/discovery/utils/DiscoveryBannerTint.ts create mode 100644 fluxer_app/src/features/messaging/components/alerts/MessageCrosspostLimitModal.tsx create mode 100644 fluxer_app/src/features/messaging/components/alerts/PublishedMessageEditLimitModal.tsx create mode 100644 fluxer_app/src/features/messaging/components/modals/MessageCrosspostConfirmModal.tsx create mode 100644 fluxer_app/src/features/ui/components/icons/AnnouncementChannelIcon.tsx create mode 100644 fluxer_app/src/features/ui/components/icons/LockedAnnouncementChannelIcon.tsx create mode 100644 fluxer_app/src/features/ui/components/icons/MatureAnnouncementChannelIcon.tsx create mode 100644 fluxer_app/src/features/webhook/components/FollowedChannelListItem.module.css create mode 100644 fluxer_app/src/features/webhook/components/FollowedChannelListItem.tsx create mode 100644 fluxer_docs/src/content/docs/topics/announcement-channels.md create mode 100644 packages/constants/src/AnnouncementConstants.ts create mode 100644 packages/constants/src/ChannelConstants.test.ts create mode 100644 packages/errors/src/__tests__/AnnouncementErrors.test.ts create mode 100644 packages/errors/src/domains/channel/AnnouncementChannelRequiredError.ts create mode 100644 packages/errors/src/domains/channel/ChannelAlreadyFollowedError.ts create mode 100644 packages/errors/src/domains/channel/ChannelHasFollowedChannelsError.ts create mode 100644 packages/errors/src/domains/channel/ChannelTypeConversionNotSupportedError.ts create mode 100644 packages/errors/src/domains/channel/FollowTargetContentWarningRequiredError.ts create mode 100644 packages/errors/src/domains/channel/FollowTargetNotAgeRestrictedError.ts create mode 100644 packages/errors/src/domains/channel/InvalidFollowTargetChannelError.ts create mode 100644 packages/errors/src/domains/channel/MessageAlreadyCrosspostedError.ts create mode 100644 packages/errors/src/domains/channel/MessageNotCrosspostableError.ts create mode 100644 packages/limits/src/__tests__/LimitDefaults.test.ts create mode 100644 packages/schema/src/domains/channel/ChannelFollowSchemas.ts create mode 100644 packages/schema/src/domains/message/CrosspostSourceSchemas.ts create mode 100644 packages/schema/src/domains/tests/AnnouncementChannelSchemas.test.ts diff --git a/fluxer_admin/build.rs b/fluxer_admin/build.rs index 2954bab76..ca3a83840 100644 --- a/fluxer_admin/build.rs +++ b/fluxer_admin/build.rs @@ -40,6 +40,7 @@ fn generate_admin_api(manifest_dir: &Path, out_dir: &Path) { adapt_progenitor_throttled_errors(&mut spec); relax_guild_audit_log_schemas(&mut spec); relax_progenitor_schema_strictness(&mut spec); + relax_integer_enums(&mut spec); let mut settings = progenitor::GenerationSettings::new(); settings.with_interface(progenitor::InterfaceStyle::Positional); @@ -174,6 +175,23 @@ fn relax_guild_audit_log_schemas(spec: &mut openapiv3::OpenAPI) { } } +const OPEN_INTEGER_ENUMS: &[&str] = &["ChannelType", "MessageType", "WebhookType"]; + +fn relax_integer_enums(spec: &mut openapiv3::OpenAPI) { + let components = spec.components.as_mut().expect("missing API components"); + for name in OPEN_INTEGER_ENUMS { + let Some(openapiv3::ReferenceOr::Item(schema)) = components.schemas.get_mut(*name) else { + panic!("missing inline {name} schema"); + }; + let openapiv3::SchemaKind::Type(openapiv3::Type::Integer(integer)) = + &mut schema.schema_kind + else { + panic!("{name} must be an integer schema"); + }; + integer.enumeration.clear(); + } +} + fn object_schema_mut<'a>( components: &'a mut openapiv3::Components, name: &str, diff --git a/fluxer_admin/openapi-admin.json b/fluxer_admin/openapi-admin.json index e0ecd4ebb..4400a348e 100644 --- a/fluxer_admin/openapi-admin.json +++ b/fluxer_admin/openapi-admin.json @@ -13287,7 +13287,7 @@ "ChannelType": { "description": "The type of the channel", "type": "integer", - "enum": [0, 1, 2, 3, 4, 998, 999], + "enum": [0, 1, 2, 3, 4, 5, 998, 999], "format": "int32", "x-enumNames": [ "GUILD_TEXT", @@ -13295,6 +13295,7 @@ "GUILD_VOICE", "GROUP_DM", "GUILD_CATEGORY", + "GUILD_ANNOUNCEMENT", "GUILD_LINK", "DM_PERSONAL_NOTES" ], @@ -13304,6 +13305,7 @@ "A voice channel within a guild", "A group direct message between users", "A category that contains channels", + "A guild channel whose messages can be published to channels that follow it", "A link channel for external resources", "Personal notes DM channel" ] @@ -13502,7 +13504,7 @@ "enum": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] }, "GuildFeatureSchema": { - "description": "A guild feature flag Known values: ANIMATED_ICON, ANIMATED_BANNER, AUDIO_BITRATE_128_KBPS, AUDIO_BITRATE_256_KBPS, AUDIO_BITRATE_384_KBPS, BANNER, CLONE_EMOJI_DISABLED, CLONE_EMOJI_ENABLED, CLONE_STICKER_DISABLED, CLONE_STICKER_ENABLED, DETACHED_BANNER, INVITE_SPLASH, INVITES_DISABLED, RAID_DETECTED, TEXT_CHANNEL_FLEXIBLE_NAMES, HIDE_OWNER_CROWN, MORE_EMOJI, MORE_STICKERS, UNLIMITED_EMOJI, UNLIMITED_STICKERS, EXPRESSION_PURGE_ALLOWED, VANITY_URL, DISCOVERABLE, PARTNERED, VERIFIED, VIP_VOICE, VOICE_E2EE, UNAVAILABLE_FOR_EVERYONE, UNAVAILABLE_FOR_EVERYONE_BUT_STAFF, UNAVAILABLE_HIDDEN, VISIONARY, LARGE_GUILD_OVERRIDE, VERY_LARGE_GUILD (other values allowed)", + "description": "A guild feature flag Known values: ANIMATED_ICON, ANIMATED_BANNER, AUDIO_BITRATE_128_KBPS, AUDIO_BITRATE_256_KBPS, AUDIO_BITRATE_384_KBPS, BANNER, CLONE_EMOJI_DISABLED, CLONE_EMOJI_ENABLED, CLONE_STICKER_DISABLED, CLONE_STICKER_ENABLED, DETACHED_BANNER, INVITE_SPLASH, INVITES_DISABLED, RAID_DETECTED, TEXT_CHANNEL_FLEXIBLE_NAMES, HIDE_OWNER_CROWN, MORE_EMOJI, MORE_STICKERS, UNLIMITED_EMOJI, UNLIMITED_STICKERS, EXPRESSION_PURGE_ALLOWED, VANITY_URL, DISCOVERABLE, PARTNERED, VERIFIED, VIP_VOICE, VOICE_E2EE, UNAVAILABLE_FOR_EVERYONE, UNAVAILABLE_FOR_EVERYONE_BUT_STAFF, UNAVAILABLE_HIDDEN, VISIONARY, LARGE_GUILD_OVERRIDE, VERY_LARGE_GUILD, ANNOUNCEMENT_CHANNELS_DISABLED (other values allowed)", "x-enumNames": [ "ANIMATED_ICON", "ANIMATED_BANNER", @@ -13536,7 +13538,8 @@ "UNAVAILABLE_HIDDEN", "VISIONARY", "LARGE_GUILD_OVERRIDE", - "VERY_LARGE_GUILD" + "VERY_LARGE_GUILD", + "ANNOUNCEMENT_CHANNELS_DISABLED" ], "x-enumDescriptions": [ "Guild can have an animated icon", @@ -13571,7 +13574,8 @@ "Guild is hidden when it is force unavailable", "Guild is a visionary guild", "Guild has large guild overrides enabled", - "Guild has increased member capacity enabled" + "Guild has increased member capacity enabled", + "Guild cannot publish announcement messages or gain new followers" ], "type": "string" }, @@ -13738,7 +13742,8 @@ "allOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}] }, "message_id": { - "description": "The ID of the referenced message", + "description": "The ID of the referenced message, absent on a channel follow system message", + "nullable": true, "allOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}] }, "guild_id": { @@ -13748,7 +13753,7 @@ }, "type": {"allOf": [{"$ref": "#/components/schemas/MessageReferenceType"}]} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, "message_snapshots": { @@ -13883,7 +13888,8 @@ "allOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}] }, "message_id": { - "description": "The ID of the referenced message", + "description": "The ID of the referenced message, absent on a channel follow system message", + "nullable": true, "allOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}] }, "guild_id": { @@ -13893,7 +13899,7 @@ }, "type": {"allOf": [{"$ref": "#/components/schemas/MessageReferenceType"}]} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, "message_snapshots": { @@ -14384,11 +14390,26 @@ "description": "The bitwise flags of the original message", "format": "int32", "x-bitflagValues": [ + { + "name": "CROSSPOSTED", + "value": "1", + "description": "This message has been published to channels that follow this announcement channel" + }, + { + "name": "IS_CROSSPOST", + "value": "2", + "description": "This message was delivered from an announcement channel this channel follows" + }, { "name": "SUPPRESS_EMBEDS", "value": "4", "description": "Do not include embeds when serialising this message" }, + { + "name": "SOURCE_MESSAGE_DELETED", + "value": "8", + "description": "The published message this copy came from has been deleted" + }, { "name": "SUPPRESS_NOTIFICATIONS", "value": "4096", @@ -14400,7 +14421,7 @@ "MessageType": { "description": "The type of message", "type": "integer", - "enum": [0, 1, 2, 3, 4, 5, 6, 7, 19], + "enum": [0, 1, 2, 3, 4, 5, 6, 7, 12, 19], "format": "int32", "x-enumNames": [ "DEFAULT", @@ -14411,6 +14432,7 @@ "CHANNEL_ICON_CHANGE", "CHANNEL_PINNED_MESSAGE", "USER_JOIN", + "CHANNEL_FOLLOW_ADD", "REPLY" ], "x-enumDescriptions": [ @@ -14422,6 +14444,7 @@ "A system message indicating the channel icon changed", "A system message indicating a message was pinned", "A system message indicating a user joined", + "System message posted when a channel starts following an announcement channel", "A reply message" ] }, diff --git a/fluxer_admin/src/templates/pages/bulk_actions.rs b/fluxer_admin/src/templates/pages/bulk_actions.rs index 0df0e7efd..4c7f21004 100644 --- a/fluxer_admin/src/templates/pages/bulk_actions.rs +++ b/fluxer_admin/src/templates/pages/bulk_actions.rs @@ -179,6 +179,7 @@ const GUILD_FEATURES: &[&str] = &[ "VISIONARY", "LARGE_GUILD_OVERRIDE", "VERY_LARGE_GUILD", + "ANNOUNCEMENT_CHANNELS_DISABLED", ]; const DEPRECATED_GUILD_FEATURES: &[&str] = &["CLONE_EMOJI_DISABLED", "CLONE_STICKER_DISABLED"]; diff --git a/fluxer_admin/src/templates/pages/guild_detail_tabs/features.rs b/fluxer_admin/src/templates/pages/guild_detail_tabs/features.rs index 09b37c844..4588ac76c 100644 --- a/fluxer_admin/src/templates/pages/guild_detail_tabs/features.rs +++ b/fluxer_admin/src/templates/pages/guild_detail_tabs/features.rs @@ -45,6 +45,7 @@ const GUILD_FEATURES: &[&str] = &[ "VISIONARY", "LARGE_GUILD_OVERRIDE", "VERY_LARGE_GUILD", + "ANNOUNCEMENT_CHANNELS_DISABLED", ]; const HOSTED_ONLY: &[&str] = &["VISIONARY", "VIP_VOICE"]; diff --git a/fluxer_admin/src/templates/pages/guild_detail_tabs/overview.rs b/fluxer_admin/src/templates/pages/guild_detail_tabs/overview.rs index 60e07abb6..225252543 100644 --- a/fluxer_admin/src/templates/pages/guild_detail_tabs/overview.rs +++ b/fluxer_admin/src/templates/pages/guild_detail_tabs/overview.rs @@ -32,6 +32,7 @@ fn channel_type_label(channel_type: i32) -> &'static str { 0 => "Text", 2 => "Voice", 4 => "Category", + 5 => "Announcement", 13 => "Link", _ => "Unknown", } diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index 0811f06d5..97ab5c51d 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -450,6 +450,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { unfurl: apiWorkerConfig?.lane_concurrency_overrides?.unfurl, lifecycle: apiWorkerConfig?.lane_concurrency_overrides?.lifecycle, batch: apiWorkerConfig?.lane_concurrency_overrides?.batch, + crosspost: apiWorkerConfig?.lane_concurrency_overrides?.crosspost, }, }, }; diff --git a/fluxer_api/src/api/Tables.ts b/fluxer_api/src/api/Tables.ts index 07db8efa1..91ba6caf4 100644 --- a/fluxer_api/src/api/Tables.ts +++ b/fluxer_api/src/api/Tables.ts @@ -129,6 +129,10 @@ import { CHANNELS_BY_GUILD_COLUMNS, type ChannelRow, type ChannelsByGuildRow, + CROSSPOST_SOURCE_BY_CHANNEL_COLUMNS, + CROSSPOSTED_MESSAGE_COLUMNS, + type CrosspostedMessageRow, + type CrosspostSourceByChannelRow, DM_STATE_COLUMNS, type DmStateRow, INVITE_COLUMNS, @@ -136,7 +140,9 @@ import { PRIVATE_CHANNEL_COLUMNS, type PrivateChannelRow, WEBHOOK_COLUMNS, + WEBHOOKS_BY_SOURCE_CHANNEL_COLUMNS, type WebhookRow, + type WebhooksBySourceChannelRow, } from '@app/api/database/types/ChannelTypes'; import {USER_CONNECTION_STORAGE_COLUMNS, type UserConnectionStorageRow} from '@app/api/database/types/ConnectionTypes'; import { @@ -1089,6 +1095,36 @@ export const WebhooksByGuild = defineTable({ + name: 'webhooks_by_source_channel_id', + columns: WEBHOOKS_BY_SOURCE_CHANNEL_COLUMNS, + primaryKey: ['source_channel_id', 'webhook_id'], + partitionKey: ['source_channel_id'], +}); +export const CrosspostedMessages = defineTable< + CrosspostedMessageRow, + 'source_message_id' | 'webhook_id', + 'source_message_id' +>({ + name: 'crossposted_messages', + columns: CROSSPOSTED_MESSAGE_COLUMNS, + primaryKey: ['source_message_id', 'webhook_id'], + partitionKey: ['source_message_id'], +}); +export const CrosspostSourcesByChannel = defineTable< + CrosspostSourceByChannelRow, + 'source_channel_id' | 'source_message_id', + 'source_channel_id' +>({ + name: 'crosspost_sources_by_channel', + columns: CROSSPOST_SOURCE_BY_CHANNEL_COLUMNS, + primaryKey: ['source_channel_id', 'source_message_id'], + partitionKey: ['source_channel_id'], +}); export const InstanceConfiguration = defineTable({ name: 'instance_configuration', columns: INSTANCE_CONFIGURATION_COLUMNS, diff --git a/fluxer_api/src/api/admin/services/AdminMessageService.ts b/fluxer_api/src/api/admin/services/AdminMessageService.ts index b36f1961d..770d4ed52 100644 --- a/fluxer_api/src/api/admin/services/AdminMessageService.ts +++ b/fluxer_api/src/api/admin/services/AdminMessageService.ts @@ -13,6 +13,10 @@ import { type UserID, } from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import { + enqueueCrosspostFamilyPurgeFromCopies, + enqueueCrosspostSourceRemoval, +} from '@app/api/channel/services/message/CrosspostPropagation'; import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; import { createMessageResponseDataService, @@ -127,15 +131,13 @@ export class AdminMessageService { async deleteMessage(data: DeleteMessageRequest, adminUserId: UserID, auditLogReason: string | null) { const {channelRepository, auditService} = this.deps; - const {gateway: gatewayService} = this.deps.apiContext.services; + const {gateway: gatewayService, worker: workerService} = this.deps.apiContext.services; const channelId = createChannelID(data.channel_id); const messageId = createMessageID(data.message_id); const channel = await channelRepository.findUnique(channelId); const message = await channelRepository.getMessage(channelId, messageId); if (message) { - if (message.attachments.length > 0) { - await purgeMessageAttachments(message, getStorageService(), getPurgeQueue()); - } + await purgeMessageAttachments(message, getStorageService(), getPurgeQueue()); await channelRepository.deleteMessage( channelId, messageId, @@ -166,6 +168,8 @@ export class AdminMessageService { } } await deleteMessageSearchDocuments([messageId], {context: {source: 'admin_message_delete'}}); + await enqueueCrosspostSourceRemoval(workerService, {messages: [message], mode: 'purge'}); + await enqueueCrosspostFamilyPurgeFromCopies(workerService, {messages: [message]}); } await auditService.createAuditLog({ adminUserId, diff --git a/fluxer_api/src/api/admin/tests/AdminSyntheticUserServiceGuards.test.ts b/fluxer_api/src/api/admin/tests/AdminSyntheticUserServiceGuards.test.ts index 3606b7ae0..7a7a13c5a 100644 --- a/fluxer_api/src/api/admin/tests/AdminSyntheticUserServiceGuards.test.ts +++ b/fluxer_api/src/api/admin/tests/AdminSyntheticUserServiceGuards.test.ts @@ -2,6 +2,7 @@ import {createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import type {CrosspostWorkerService} from '@app/api/channel/services/message/CrosspostPropagation'; import {UserMessageDeletionService} from '@app/api/channel/services/message/UserMessageDeletionService'; import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate'; import {GuildMemberOperationsService} from '@app/api/guild/services/member/GuildMemberOperationsService'; @@ -37,6 +38,7 @@ function createMessageDeletionService(): UserMessageDeletionService { gatewayService: unusable as IGatewayService, storageService: unusable as IStorageService, purgeQueue: unusable as IPurgeQueue, + workerService: unusable as CrosspostWorkerService, }); } diff --git a/fluxer_api/src/api/channel/ChannelMappers.ts b/fluxer_api/src/api/channel/ChannelMappers.ts index 41d43c23f..1561e2b31 100644 --- a/fluxer_api/src/api/channel/ChannelMappers.ts +++ b/fluxer_api/src/api/channel/ChannelMappers.ts @@ -196,6 +196,7 @@ export async function mapChannelToResponse(params: MapChannelToResponseParams): let response: ChannelResponse; switch (channel.type) { case ChannelTypes.GUILD_TEXT: + case ChannelTypes.GUILD_ANNOUNCEMENT: response = serializeGuildTextChannel(channel, ctx); break; case ChannelTypes.GUILD_VOICE: diff --git a/fluxer_api/src/api/channel/ChannelRepository.ts b/fluxer_api/src/api/channel/ChannelRepository.ts index 1986ed7fd..eae81b9e3 100644 --- a/fluxer_api/src/api/channel/ChannelRepository.ts +++ b/fluxer_api/src/api/channel/ChannelRepository.ts @@ -30,6 +30,10 @@ export class ChannelRepository extends IChannelRepository { return this.repository.messageInteractions; } + get crossposts() { + return this.repository.crossposts; + } + async findUnique(channelId: ChannelID): Promise { return this.repository.channelData.findUnique(channelId); } diff --git a/fluxer_api/src/api/channel/controllers/ChannelController.ts b/fluxer_api/src/api/channel/controllers/ChannelController.ts index 4516ac60e..c8f96866e 100644 --- a/fluxer_api/src/api/channel/controllers/ChannelController.ts +++ b/fluxer_api/src/api/channel/controllers/ChannelController.ts @@ -11,6 +11,8 @@ import {RateLimitConfigs} from '@app/api/RateLimitConfig'; import type {HonoApp, HonoEnv} from '@app/api/types/HonoEnv'; import {CLIENT_FEATURES_HEADER, parseClientFeaturesHeader} from '@app/api/utils/featureUtils'; import {Validator} from '@app/api/Validator'; +import {ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES} from '@fluxer/constants/src/ChannelConstants'; +import {ChannelTypeConversionNotSupportedError} from '@fluxer/errors/src/domains/channel/ChannelTypeConversionNotSupportedError'; import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError'; import {SudoVerificationSchema} from '@fluxer/schema/src/domains/auth/AuthSchemas'; import { @@ -136,7 +138,19 @@ export function ChannelController(app: HonoApp) { throw new UnknownChannelError(); } const body = isPlainObject(raw) ? raw : {}; - return {...body, type: channelType}; + const requestedType = body.type; + if ( + requestedType === undefined || + requestedType === null || + requestedType === channelType || + !ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES.has(channelType) + ) { + return {...body, type: channelType}; + } + if (typeof requestedType !== 'number' || !ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES.has(requestedType)) { + throw new ChannelTypeConversionNotSupportedError(); + } + return {...body, type: requestedType}; }, }), OpenAPI({ @@ -154,6 +168,9 @@ export function ChannelController(app: HonoApp) { const userId = ctx.get('user').id; const channelId = createChannelID(ctx.req.valid('param').channel_id); const data = ctx.req.valid('json'); + const existingType = ctx.get('channelUpdateType'); + const typeConversion = + existingType !== undefined && data.type !== existingType ? {from: existingType, to: data.type} : null; const clientFeatures = parseClientFeaturesHeader(ctx.req.header(CLIENT_FEATURES_HEADER)); const requestCache = ctx.get('requestCache'); const auditLogReason = ctx.get('auditLogReason') ?? null; @@ -166,6 +183,7 @@ export function ChannelController(app: HonoApp) { clientFeatures, requestCache, auditLogReason, + typeConversion, }), ); }, diff --git a/fluxer_api/src/api/channel/controllers/ChannelFollowController.ts b/fluxer_api/src/api/channel/controllers/ChannelFollowController.ts new file mode 100644 index 000000000..1de81696b --- /dev/null +++ b/fluxer_api/src/api/channel/controllers/ChannelFollowController.ts @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID} from '@app/api/BrandedTypes'; +import {LoginRequired} from '@app/api/middleware/AuthMiddleware'; +import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware'; +import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware'; +import {RateLimitConfigs} from '@app/api/RateLimitConfig'; +import type {HonoApp} from '@app/api/types/HonoEnv'; +import {Validator} from '@app/api/Validator'; +import { + ChannelFollowerStatsResponse, + ChannelFollowRequest, + FollowedChannelResponse, +} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; +import {ChannelIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas'; + +export function ChannelFollowController(app: HonoApp) { + app.post( + '/channels/:channel_id/followers', + RateLimitMiddleware(RateLimitConfigs.CHANNEL_FOLLOW), + LoginRequired, + Validator('param', ChannelIdParam), + Validator('json', ChannelFollowRequest), + OpenAPI({ + operationId: 'follow_channel', + summary: 'Follow an announcement channel', + description: + 'Follows an announcement channel into a text channel. Creates a channel follower webhook in the target channel that receives every message published in the announcement channel. Requires Manage Webhooks in the target channel and View Channel on the announcement channel.', + requestSchema: ChannelFollowRequest, + responseSchema: FollowedChannelResponse, + statusCode: 200, + security: ['botToken', 'bearerToken', 'sessionToken'], + tags: 'Channels', + }), + async (ctx) => { + const followed = await ctx.get('channelFollowService').followChannel({ + userId: ctx.get('user').id, + channelId: createChannelID(ctx.req.valid('param').channel_id), + webhookChannelId: createChannelID(ctx.req.valid('json').webhook_channel_id), + requestCache: ctx.get('requestCache'), + auditLogReason: ctx.get('auditLogReason') ?? null, + }); + return ctx.json({ + channel_id: followed.channelId.toString(), + webhook_id: followed.webhookId.toString(), + } satisfies FollowedChannelResponse); + }, + ); + app.get( + '/channels/:channel_id/follower-stats', + RateLimitMiddleware(RateLimitConfigs.CHANNEL_FOLLOWER_STATS), + LoginRequired, + Validator('param', ChannelIdParam), + OpenAPI({ + operationId: 'get_channel_follower_stats', + summary: 'Get announcement channel follower stats', + description: + 'Returns how many channels and distinct guilds follow an announcement channel. Requires View Channel on the announcement channel.', + responseSchema: ChannelFollowerStatsResponse, + statusCode: 200, + security: ['botToken', 'bearerToken', 'sessionToken'], + tags: 'Channels', + }), + async (ctx) => { + const stats = await ctx.get('channelFollowService').getFollowerStats({ + userId: ctx.get('user').id, + channelId: createChannelID(ctx.req.valid('param').channel_id), + }); + return ctx.json(stats); + }, + ); +} diff --git a/fluxer_api/src/api/channel/controllers/MessageController.ts b/fluxer_api/src/api/channel/controllers/MessageController.ts index 3833a8c9d..d2e130f5d 100644 --- a/fluxer_api/src/api/channel/controllers/MessageController.ts +++ b/fluxer_api/src/api/channel/controllers/MessageController.ts @@ -29,6 +29,7 @@ import { PresignedAttachmentUploadRequest, PresignedAttachmentUploadResponse, } from '@fluxer/schema/src/domains/message/AttachmentUploadSchemas'; +import {CrosspostSourceResponse} from '@fluxer/schema/src/domains/message/CrosspostSourceSchemas'; import { BulkDeleteMessagesRequest, BulkMessageFetchRequest, @@ -503,6 +504,60 @@ export function MessageController(app: HonoApp) { return ctx.body(null, 204); }, ); + app.post( + '/channels/:channel_id/messages/:message_id/crosspost', + RateLimitMiddleware(RateLimitConfigs.CHANNEL_MESSAGE_CROSSPOST), + LoginRequired, + Validator('param', ChannelIdMessageIdParam), + OpenAPI({ + operationId: 'crosspost_message', + summary: 'Publish a message to following channels', + responseSchema: MessageResponseSchema, + statusCode: 200, + security: ['botToken', 'bearerToken', 'sessionToken'], + tags: ['Channels', 'Messages'], + description: + 'Publishes a message in an announcement channel to every channel that follows it. The author needs Send Messages. Anyone else needs Send Messages and Manage Messages. Only default messages that are not replies, forwards or copies can be published, and each message can be published once. Copies are delivered asynchronously. Publishing is limited per channel (10 in a row, then one every 6 minutes) and per community (30 in a row, then one every 2 minutes). Returns the updated message with the CROSSPOSTED flag set.', + }), + async (ctx) => { + const {channel_id, message_id} = ctx.req.valid('param'); + return ctx.json( + await ctx.get('messageRequestService').crosspostMessage({ + userId: ctx.get('user').id, + channelId: createChannelID(channel_id), + messageId: createMessageID(message_id), + requestCache: ctx.get('requestCache'), + }), + ); + }, + ); + app.get( + '/channels/:channel_id/messages/:message_id/crosspost-source', + RateLimitMiddleware(RateLimitConfigs.CHANNEL_MESSAGE_CROSSPOST_SOURCE), + LoginRequired, + Validator('param', ChannelIdMessageIdParam), + OpenAPI({ + operationId: 'get_message_crosspost_source', + summary: 'Get the source community of a published message copy', + responseSchema: CrosspostSourceResponse, + statusCode: 200, + security: ['botToken', 'bearerToken', 'sessionToken'], + tags: ['Channels', 'Messages'], + description: + 'Returns the public profile of the community a message copy was published from. Works on copies delivered to a following channel and on the system message posted when a channel starts following. Needs the same access as fetching the message. The response holds the community name, icon, banner, badge features, approximate counts and whether it can be joined through discovery.', + }), + async (ctx) => { + const {channel_id, message_id} = ctx.req.valid('param'); + return ctx.json( + await ctx.get('messageRequestService').getCrosspostSource({ + userId: ctx.get('user').id, + channelId: createChannelID(channel_id), + messageId: createMessageID(message_id), + requestCache: ctx.get('requestCache'), + }), + ); + }, + ); app.post( '/channels/:channel_id/messages/:message_id/ack', RateLimitMiddleware(RateLimitConfigs.CHANNEL_MESSAGE_ACK), diff --git a/fluxer_api/src/api/channel/controllers/index.ts b/fluxer_api/src/api/channel/controllers/index.ts index a3da8b989..6d9b5b755 100644 --- a/fluxer_api/src/api/channel/controllers/index.ts +++ b/fluxer_api/src/api/channel/controllers/index.ts @@ -2,6 +2,7 @@ import {CallController} from '@app/api/channel/controllers/CallController'; import {ChannelController} from '@app/api/channel/controllers/ChannelController'; +import {ChannelFollowController} from '@app/api/channel/controllers/ChannelFollowController'; import {MessageController} from '@app/api/channel/controllers/MessageController'; import {MessageInteractionController} from '@app/api/channel/controllers/MessageInteractionController'; import {StreamController} from '@app/api/channel/controllers/StreamController'; @@ -9,6 +10,7 @@ import type {HonoApp} from '@app/api/types/HonoEnv'; export function registerChannelControllers(app: HonoApp) { ChannelController(app); + ChannelFollowController(app); MessageInteractionController(app); MessageController(app); CallController(app); diff --git a/fluxer_api/src/api/channel/repositories/ChannelRepository.ts b/fluxer_api/src/api/channel/repositories/ChannelRepository.ts index 4ec4dc490..2336a55f4 100644 --- a/fluxer_api/src/api/channel/repositories/ChannelRepository.ts +++ b/fluxer_api/src/api/channel/repositories/ChannelRepository.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {ChannelDataRepository} from '@app/api/channel/repositories/ChannelDataRepository'; +import {CrosspostedMessageRepository} from '@app/api/channel/repositories/CrosspostedMessageRepository'; import {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; import {MessageInteractionRepository} from '@app/api/channel/repositories/MessageInteractionRepository'; import {MessageRepository} from '@app/api/channel/repositories/MessageRepository'; @@ -10,11 +11,13 @@ export class ChannelRepository extends IChannelRepositoryAggregate { readonly channelData: ChannelDataRepository; readonly messages: MessageRepository; readonly messageInteractions: MessageInteractionRepository; + readonly crossposts: CrosspostedMessageRepository; constructor(requestCache?: RequestCache) { super(); this.channelData = new ChannelDataRepository(requestCache); this.messages = new MessageRepository(this.channelData); this.messageInteractions = new MessageInteractionRepository(this.messages); + this.crossposts = new CrosspostedMessageRepository(); } } diff --git a/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.test.ts b/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.test.ts new file mode 100644 index 000000000..675b44620 --- /dev/null +++ b/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.test.ts @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createGuildID, createMessageID, createWebhookID, type MessageID} from '@app/api/BrandedTypes'; +import {ChannelRepository} from '@app/api/channel/repositories/ChannelRepository'; +import {CrosspostedMessageRepository} from '@app/api/channel/repositories/CrosspostedMessageRepository'; +import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution'; +import type {CrosspostedMessageRow} from '@app/api/database/types/ChannelTypes'; +import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor'; +import {afterEach, beforeEach, describe, expect, it} from 'vitest'; + +const SOURCE_CHANNEL = createChannelID(10n); +const SOURCE_MESSAGE = createMessageID(100n); +const WEBHOOK = createWebhookID(500n); +const KEY = {sourceMessageId: SOURCE_MESSAGE, webhookId: WEBHOOK}; + +let executor: InMemoryCassandraQueryExecutor; +let repository: CrosspostedMessageRepository; + +function pendingRow(overrides: Partial = {}): CrosspostedMessageRow { + return { + source_message_id: SOURCE_MESSAGE, + webhook_id: WEBHOOK, + source_channel_id: SOURCE_CHANNEL, + target_guild_id: createGuildID(20n), + target_channel_id: createChannelID(30n), + target_message_id: createMessageID(1000n), + state: 'pending', + reserved_at: new Date('2026-09-30T12:00:00.000Z'), + source_fingerprint: null, + created_at: new Date('2026-09-30T12:00:00.000Z'), + ...overrides, + }; +} + +describe('CrosspostedMessageRepository', () => { + beforeEach(() => { + executor = new InMemoryCassandraQueryExecutor(); + setCassandraQueryExecutorForTesting(executor); + repository = new CrosspostedMessageRepository(); + }); + afterEach(() => { + executor.reset(); + setCassandraQueryExecutorForTesting(null); + }); + + it('is exposed on the channel repository aggregate', () => { + expect(new ChannelRepository().crossposts).toBeInstanceOf(CrosspostedMessageRepository); + }); + + it('returns null for a pair that was never reserved', async () => { + expect(await repository.get(SOURCE_MESSAGE, WEBHOOK)).toBeNull(); + }); + + it('inserts a pending row once and refuses a second reservation', async () => { + expect(await repository.insertPending(pendingRow())).toBe(true); + expect(await repository.insertPending(pendingRow({target_message_id: createMessageID(2000n)}))).toBe(false); + const row = await repository.get(SOURCE_MESSAGE, WEBHOOK); + expect(row?.state).toBe('pending'); + expect(row?.target_message_id).toBe(1000n); + expect(row?.target_guild_id).toBe(20n); + expect(row?.target_channel_id).toBe(30n); + expect(row?.source_channel_id).toBe(10n); + }); + + it('always stores a reservation as pending', async () => { + expect(await repository.insertPending(pendingRow({state: 'delivered'}))).toBe(true); + expect((await repository.get(SOURCE_MESSAGE, WEBHOOK))?.state).toBe('pending'); + }); + + it('reclaims a pending row only from the expected target id', async () => { + await repository.insertPending(pendingRow()); + const reservedAt = new Date('2026-09-30T12:05:00.000Z'); + expect( + await repository.reclaimPending(KEY, { + fromTargetMessageId: createMessageID(999n), + toTargetMessageId: createMessageID(2000n), + reservedAt, + }), + ).toBe(false); + expect( + await repository.reclaimPending(KEY, { + fromTargetMessageId: createMessageID(1000n), + toTargetMessageId: createMessageID(2000n), + reservedAt, + }), + ).toBe(true); + const row = await repository.get(SOURCE_MESSAGE, WEBHOOK); + expect(row?.target_message_id).toBe(2000n); + expect(row?.reserved_at.getTime()).toBe(reservedAt.getTime()); + expect(row?.state).toBe('pending'); + }); + + it('does not reclaim a delivered row', async () => { + await repository.insertPending(pendingRow()); + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp0', + }); + expect( + await repository.reclaimPending(KEY, { + fromTargetMessageId: createMessageID(1000n), + toTargetMessageId: createMessageID(2000n), + reservedAt: new Date(), + }), + ).toBe(false); + expect((await repository.get(SOURCE_MESSAGE, WEBHOOK))?.target_message_id).toBe(1000n); + }); + + it('marks delivered only when the target id still matches', async () => { + await repository.insertPending(pendingRow()); + await repository.reclaimPending(KEY, { + fromTargetMessageId: createMessageID(1000n), + toTargetMessageId: createMessageID(2000n), + reservedAt: new Date(), + }); + expect( + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'stale', + }), + ).toBe(false); + expect( + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(2000n), + sourceFingerprint: 'fp0', + }), + ).toBe(true); + const row = await repository.get(SOURCE_MESSAGE, WEBHOOK); + expect(row?.state).toBe('delivered'); + expect(row?.source_fingerprint).toBe('fp0'); + }); + + it('marks delivered with a null fingerprint for a recovered pending copy', async () => { + await repository.insertPending(pendingRow({source_fingerprint: 'reserved'})); + expect( + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: null, + }), + ).toBe(true); + const row = await repository.get(SOURCE_MESSAGE, WEBHOOK); + expect(row?.state).toBe('delivered'); + expect(row?.source_fingerprint ?? null).toBeNull(); + }); + + it('does not mark a missing row delivered', async () => { + expect( + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp0', + }), + ).toBe(false); + expect(await repository.get(SOURCE_MESSAGE, WEBHOOK)).toBeNull(); + }); + + it('updates sync state only on a delivered row with the same target id', async () => { + await repository.insertPending(pendingRow()); + expect( + await repository.updateSynced(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp1', + }), + ).toBe(false); + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp0', + }); + expect( + await repository.updateSynced(KEY, { + targetMessageId: createMessageID(1001n), + sourceFingerprint: 'fp1', + }), + ).toBe(false); + expect( + await repository.updateSynced(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp1', + }), + ).toBe(true); + const row = await repository.get(SOURCE_MESSAGE, WEBHOOK); + expect(row?.source_fingerprint).toBe('fp1'); + expect(row?.state).toBe('delivered'); + }); + + it('pages rows for a source message in webhook id order', async () => { + const webhookIds = [505n, 501n, 503n, 502n, 504n]; + for (const id of webhookIds) { + await repository.insertPending(pendingRow({webhook_id: createWebhookID(id)})); + } + await repository.insertPending( + pendingRow({source_message_id: createMessageID(101n), webhook_id: createWebhookID(506n)}), + ); + const first = await repository.listBySourceMessage(SOURCE_MESSAGE, {limit: 2}); + expect(first.map((row) => row.webhook_id)).toEqual([501n, 502n]); + const second = await repository.listBySourceMessage(SOURCE_MESSAGE, { + afterWebhookId: first[first.length - 1]!.webhook_id, + limit: 2, + }); + expect(second.map((row) => row.webhook_id)).toEqual([503n, 504n]); + const third = await repository.listBySourceMessage(SOURCE_MESSAGE, { + afterWebhookId: second[second.length - 1]!.webhook_id, + limit: 2, + }); + expect(third.map((row) => row.webhook_id)).toEqual([505n]); + }); + + it('deletes unconditionally without expected values', async () => { + await repository.insertPending(pendingRow()); + expect(await repository.delete(KEY)).toBe(true); + expect(await repository.get(SOURCE_MESSAGE, WEBHOOK)).toBeNull(); + }); + + it('deletes conditionally only when the expected state and target match', async () => { + await repository.insertPending(pendingRow()); + expect(await repository.delete(KEY, {state: 'delivered', target_message_id: createMessageID(1000n)})).toBe(false); + expect(await repository.get(SOURCE_MESSAGE, WEBHOOK)).not.toBeNull(); + await repository.markDelivered(KEY, { + targetMessageId: createMessageID(1000n), + sourceFingerprint: 'fp0', + }); + expect(await repository.delete(KEY, {state: 'delivered', target_message_id: createMessageID(999n)})).toBe(false); + expect(await repository.delete(KEY, {state: 'delivered', target_message_id: createMessageID(1000n)})).toBe(true); + expect(await repository.get(SOURCE_MESSAGE, WEBHOOK)).toBeNull(); + }); + + it('pages published sources by channel and removes them', async () => { + const messageIds: Array = [103n, 101n, 102n].map((id) => createMessageID(id)); + for (const sourceMessageId of messageIds) { + await repository.addSource({sourceChannelId: SOURCE_CHANNEL, sourceMessageId}); + } + await repository.addSource({sourceChannelId: SOURCE_CHANNEL, sourceMessageId: createMessageID(101n)}); + await repository.addSource({sourceChannelId: createChannelID(11n), sourceMessageId: createMessageID(104n)}); + expect(await repository.listSourcesByChannel(SOURCE_CHANNEL, {limit: 2})).toEqual([101n, 102n]); + expect( + await repository.listSourcesByChannel(SOURCE_CHANNEL, {afterMessageId: createMessageID(102n), limit: 2}), + ).toEqual([103n]); + await repository.deleteSource({sourceChannelId: SOURCE_CHANNEL, sourceMessageId: createMessageID(102n)}); + expect(await repository.listSourcesByChannel(SOURCE_CHANNEL, {limit: 10})).toEqual([101n, 103n]); + expect(await repository.listSourcesByChannel(createChannelID(11n), {limit: 10})).toEqual([104n]); + }); +}); diff --git a/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.ts b/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.ts new file mode 100644 index 000000000..79560fd12 --- /dev/null +++ b/fluxer_api/src/api/channel/repositories/CrosspostedMessageRepository.ts @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, MessageID, WebhookID} from '@app/api/BrandedTypes'; +import { + type CrosspostedMessageKey, + type CrosspostSource, + type CrosspostSyncState, + ICrosspostedMessageRepository, +} from '@app/api/channel/repositories/ICrosspostedMessageRepository'; +import { + deleteOneOrMany, + executeConditional, + fetchMany, + fetchOne, + upsertOne, +} from '@app/api/database/CassandraQueryExecution'; +import {Db} from '@app/api/database/CassandraTypes'; +import type {CrosspostedMessageRow, CrosspostSourceByChannelRow} from '@app/api/database/types/ChannelTypes'; +import {CrosspostedMessages, CrosspostSourcesByChannel} from '@app/api/Tables'; + +const FETCH_CROSSPOSTED_MESSAGE_CQL = CrosspostedMessages.selectCql({ + where: [CrosspostedMessages.where.eq('source_message_id'), CrosspostedMessages.where.eq('webhook_id')], + limit: 1, +}); + +function createBySourceMessageFirstPageQuery(limit: number) { + return CrosspostedMessages.select({ + where: CrosspostedMessages.where.eq('source_message_id'), + orderBy: {col: 'webhook_id', direction: 'ASC'}, + limit, + }); +} + +function createBySourceMessagePageQuery(limit: number) { + return CrosspostedMessages.select({ + where: [CrosspostedMessages.where.eq('source_message_id'), CrosspostedMessages.where.gt('webhook_id')], + orderBy: {col: 'webhook_id', direction: 'ASC'}, + limit, + }); +} + +function createSourcesByChannelFirstPageQuery(limit: number) { + return CrosspostSourcesByChannel.select({ + columns: ['source_message_id'], + where: CrosspostSourcesByChannel.where.eq('source_channel_id'), + orderBy: {col: 'source_message_id', direction: 'ASC'}, + limit, + }); +} + +function createSourcesByChannelPageQuery(limit: number) { + return CrosspostSourcesByChannel.select({ + columns: ['source_message_id'], + where: [ + CrosspostSourcesByChannel.where.eq('source_channel_id'), + CrosspostSourcesByChannel.where.gt('source_message_id'), + ], + orderBy: {col: 'source_message_id', direction: 'ASC'}, + limit, + }); +} + +function toPk(key: CrosspostedMessageKey): Pick { + return {source_message_id: key.sourceMessageId, webhook_id: key.webhookId}; +} + +export class CrosspostedMessageRepository extends ICrosspostedMessageRepository { + async get(sourceMessageId: MessageID, webhookId: WebhookID): Promise { + return fetchOne(FETCH_CROSSPOSTED_MESSAGE_CQL, { + source_message_id: sourceMessageId, + webhook_id: webhookId, + }); + } + + async insertPending(row: CrosspostedMessageRow): Promise { + return executeConditional(CrosspostedMessages.insertIfNotExists({...row, state: 'pending'})); + } + + async reclaimPending( + key: CrosspostedMessageKey, + data: { + fromTargetMessageId: MessageID; + toTargetMessageId: MessageID; + reservedAt: Date; + }, + ): Promise { + return executeConditional( + CrosspostedMessages.conditionalPatchByPk( + toPk(key), + { + target_message_id: Db.set(data.toTargetMessageId), + reserved_at: Db.set(data.reservedAt), + }, + {state: 'pending', target_message_id: data.fromTargetMessageId}, + ), + ); + } + + async markDelivered(key: CrosspostedMessageKey, data: CrosspostSyncState): Promise { + return executeConditional( + CrosspostedMessages.conditionalPatchByPk( + toPk(key), + { + state: Db.set('delivered'), + source_fingerprint: Db.set(data.sourceFingerprint), + }, + {target_message_id: data.targetMessageId}, + ), + ); + } + + async updateSynced(key: CrosspostedMessageKey, data: CrosspostSyncState): Promise { + return executeConditional( + CrosspostedMessages.conditionalPatchByPk( + toPk(key), + {source_fingerprint: Db.set(data.sourceFingerprint)}, + {state: 'delivered', target_message_id: data.targetMessageId}, + ), + ); + } + + async listBySourceMessage( + sourceMessageId: MessageID, + options: {afterWebhookId?: WebhookID; limit: number}, + ): Promise> { + if (options.afterWebhookId !== undefined) { + return fetchMany( + createBySourceMessagePageQuery(options.limit).bind({ + source_message_id: sourceMessageId, + webhook_id: options.afterWebhookId, + }), + ); + } + return fetchMany( + createBySourceMessageFirstPageQuery(options.limit).bind({source_message_id: sourceMessageId}), + ); + } + + async delete( + key: CrosspostedMessageKey, + expected?: Partial>, + ): Promise { + if (expected && Object.keys(expected).length > 0) { + return executeConditional(CrosspostedMessages.conditionalDeleteByPk(toPk(key), expected)); + } + await deleteOneOrMany(CrosspostedMessages.deleteByPk(toPk(key))); + return true; + } + + async addSource(source: CrosspostSource): Promise { + await upsertOne( + CrosspostSourcesByChannel.upsertAll({ + source_channel_id: source.sourceChannelId, + source_message_id: source.sourceMessageId, + }), + ); + } + + async listSourcesByChannel( + sourceChannelId: ChannelID, + options: {afterMessageId?: MessageID; limit: number}, + ): Promise> { + const rows = + options.afterMessageId !== undefined + ? await fetchMany>( + createSourcesByChannelPageQuery(options.limit).bind({ + source_channel_id: sourceChannelId, + source_message_id: options.afterMessageId, + }), + ) + : await fetchMany>( + createSourcesByChannelFirstPageQuery(options.limit).bind({source_channel_id: sourceChannelId}), + ); + return rows.map((row) => row.source_message_id); + } + + async deleteSource(source: CrosspostSource): Promise { + await deleteOneOrMany( + CrosspostSourcesByChannel.deleteByPk({ + source_channel_id: source.sourceChannelId, + source_message_id: source.sourceMessageId, + }), + ); + } +} diff --git a/fluxer_api/src/api/channel/repositories/IChannelRepositoryAggregate.ts b/fluxer_api/src/api/channel/repositories/IChannelRepositoryAggregate.ts index d69a4517c..28d3494ad 100644 --- a/fluxer_api/src/api/channel/repositories/IChannelRepositoryAggregate.ts +++ b/fluxer_api/src/api/channel/repositories/IChannelRepositoryAggregate.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import type {IChannelDataRepository} from '@app/api/channel/repositories/IChannelDataRepository'; +import type {ICrosspostedMessageRepository} from '@app/api/channel/repositories/ICrosspostedMessageRepository'; import type {IMessageInteractionRepository} from '@app/api/channel/repositories/IMessageInteractionRepository'; import type {IMessageRepository} from '@app/api/channel/repositories/IMessageRepository'; @@ -8,4 +9,5 @@ export abstract class IChannelRepositoryAggregate { abstract readonly channelData: IChannelDataRepository; abstract readonly messages: IMessageRepository; abstract readonly messageInteractions: IMessageInteractionRepository; + abstract readonly crossposts: ICrosspostedMessageRepository; } diff --git a/fluxer_api/src/api/channel/repositories/ICrosspostedMessageRepository.ts b/fluxer_api/src/api/channel/repositories/ICrosspostedMessageRepository.ts new file mode 100644 index 000000000..c8a968e53 --- /dev/null +++ b/fluxer_api/src/api/channel/repositories/ICrosspostedMessageRepository.ts @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, MessageID, WebhookID} from '@app/api/BrandedTypes'; +import type {CrosspostedMessageRow} from '@app/api/database/types/ChannelTypes'; + +export interface CrosspostedMessageKey { + sourceMessageId: MessageID; + webhookId: WebhookID; +} + +export interface CrosspostSyncState { + targetMessageId: MessageID; + sourceFingerprint: string | null; +} + +export interface CrosspostSource { + sourceChannelId: ChannelID; + sourceMessageId: MessageID; +} + +export abstract class ICrosspostedMessageRepository { + abstract get(sourceMessageId: MessageID, webhookId: WebhookID): Promise; + + abstract insertPending(row: CrosspostedMessageRow): Promise; + + abstract reclaimPending( + key: CrosspostedMessageKey, + data: { + fromTargetMessageId: MessageID; + toTargetMessageId: MessageID; + reservedAt: Date; + }, + ): Promise; + + abstract markDelivered(key: CrosspostedMessageKey, data: CrosspostSyncState): Promise; + + abstract updateSynced(key: CrosspostedMessageKey, data: CrosspostSyncState): Promise; + + abstract listBySourceMessage( + sourceMessageId: MessageID, + options: {afterWebhookId?: WebhookID; limit: number}, + ): Promise>; + + abstract delete( + key: CrosspostedMessageKey, + expected?: Partial>, + ): Promise; + + abstract addSource(source: CrosspostSource): Promise; + + abstract listSourcesByChannel( + sourceChannelId: ChannelID, + options: {afterMessageId?: MessageID; limit: number}, + ): Promise>; + + abstract deleteSource(source: CrosspostSource): Promise; +} diff --git a/fluxer_api/src/api/channel/services/AttachmentUploadService.ts b/fluxer_api/src/api/channel/services/AttachmentUploadService.ts index 4afe4d660..41b0d832d 100644 --- a/fluxer_api/src/api/channel/services/AttachmentUploadService.ts +++ b/fluxer_api/src/api/channel/services/AttachmentUploadService.ts @@ -13,6 +13,7 @@ import type {MessageService} from '@app/api/channel/services/MessageService'; import { assertAttachmentFileSizesWithinLimit, getContentType, + isCrosspostCopy, isMessageEmpty, isOperationDisabled, makeAttachmentCdnKey, @@ -385,19 +386,45 @@ export class AttachmentUploadService { }); return; } - const cdnKey = makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename); - await this.storageService.deleteObject(Config.s3.buckets.cdn, cdnKey); - const cdnUrl = makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename); - await this.purgeQueue.addUrls([cdnUrl]); - const updatedAttachments = message.attachments.filter((a: Attachment) => a.id !== attachmentId); - const updatedRowData = { - ...message.toRow(), - edited_timestamp: new Date(), - attachments: - updatedAttachments.length > 0 ? updatedAttachments.map((a: Attachment) => a.toMessageAttachment()) : null, - }; - const updatedMessage = await this.channelRepository.messages.upsertMessage(updatedRowData, message.toRow()); + const updatedMessage = await this.messageService.writeLock.withFreshMessage(channelId, messageId, async (fresh) => { + if (!fresh || fresh.authorId !== userId) { + throw new UnknownMessageError(); + } + const freshAttachment = fresh.attachments.find((a: Attachment) => a.id === attachmentId); + if (!freshAttachment) { + throw new UnknownMessageError(); + } + const updatedAttachments = fresh.attachments.filter((a: Attachment) => a.id !== attachmentId); + if (updatedAttachments.length === 0 && isMessageEmpty(fresh, true)) { + return null; + } + const updatedRowData = { + ...fresh.toRow(), + edited_timestamp: new Date(), + attachments: + updatedAttachments.length > 0 ? updatedAttachments.map((a: Attachment) => a.toMessageAttachment()) : null, + }; + return this.messageService.crosspostPropagation.withPublishedEditBudget({fresh, actor: 'author'}, () => + this.channelRepository.messages.upsertMessage(updatedRowData, fresh.toRow()), + ); + }); + if (!updatedMessage) { + await this.messageService.deletion.deleteMessage({ + userId, + channelId, + messageId, + requestCache, + }); + return; + } + if (!isCrosspostCopy(updatedMessage)) { + const cdnKey = makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename); + await this.storageService.deleteObject(Config.s3.buckets.cdn, cdnKey); + const cdnUrl = makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename); + await this.purgeQueue.addUrls([cdnUrl]); + } await this.messageInteractionService.dispatchMessageUpdate({channel, message: updatedMessage, requestCache}); + await this.messageService.crosspostPropagation.propagateEdit(updatedMessage); } async purgeChannelAttachments(channel: Channel): Promise { diff --git a/fluxer_api/src/api/channel/services/BaseChannelAuthService.ts b/fluxer_api/src/api/channel/services/BaseChannelAuthService.ts index b004566cd..c8767269f 100644 --- a/fluxer_api/src/api/channel/services/BaseChannelAuthService.ts +++ b/fluxer_api/src/api/channel/services/BaseChannelAuthService.ts @@ -224,6 +224,7 @@ export abstract class BaseChannelAuthService { this.options.validateNsfw && !skipNsfwValidation && (channel.type === ChannelTypes.GUILD_TEXT || + channel.type === ChannelTypes.GUILD_ANNOUNCEMENT || channel.type === ChannelTypes.GUILD_VOICE || channel.type === ChannelTypes.GUILD_LINK) && requiresAgeVerification diff --git a/fluxer_api/src/api/channel/services/ChannelDataService.ts b/fluxer_api/src/api/channel/services/ChannelDataService.ts index c71d8bda1..93dfacb7c 100644 --- a/fluxer_api/src/api/channel/services/ChannelDataService.ts +++ b/fluxer_api/src/api/channel/services/ChannelDataService.ts @@ -4,7 +4,10 @@ import type {ChannelID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; import {ChannelAuthService} from '@app/api/channel/services/channel_data/ChannelAuthService'; -import type {ChannelUpdateData} from '@app/api/channel/services/channel_data/ChannelOperationsService'; +import type { + ChannelTypeConversion, + ChannelUpdateData, +} from '@app/api/channel/services/channel_data/ChannelOperationsService'; import {ChannelOperationsService} from '@app/api/channel/services/channel_data/ChannelOperationsService'; import {ChannelUtilsService} from '@app/api/channel/services/channel_data/ChannelUtilsService'; import {GroupDmUpdateService} from '@app/api/channel/services/channel_data/GroupDmUpdateService'; @@ -28,6 +31,7 @@ import type {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilitySer import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; import type {ChannelUpdateRequest} from '@fluxer/schema/src/domains/channel/ChannelRequestSchemas'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; import type {IRateLimitService} from '@pkgs/rate_limit/src/IRateLimitService'; type GuildChannelUpdateRequest = Exclude< @@ -63,6 +67,7 @@ export class ChannelDataService { webhookRepository: IWebhookRepository, limitConfigService: LimitConfigService, rateLimitService: IRateLimitService, + cacheService: ICacheService, ) { this.utils = new ChannelUtilsService( channelRepository, @@ -87,6 +92,7 @@ export class ChannelDataService { guildRepository, limitConfigService, rateLimitService, + cacheService, ); this.groupDmUpdate = new GroupDmUpdateService( channelRepository, @@ -105,6 +111,7 @@ export class ChannelDataService { clientFeatures, requestCache, auditLogReason, + typeConversion, }: { userId: UserID; channelId: ChannelID; @@ -112,6 +119,7 @@ export class ChannelDataService { clientFeatures: ReadonlySet; requestCache: RequestCache; auditLogReason: string | null; + typeConversion?: ChannelTypeConversion | null; }): Promise { const {channel} = await this.auth.getChannelAuthenticated({userId, channelId, skipNsfwValidation: true}); if (channel.type === ChannelTypes.GROUP_DM) { @@ -185,6 +193,7 @@ export class ChannelDataService { clientFeatures, requestCache, auditLogReason, + typeConversion, }); } } diff --git a/fluxer_api/src/api/channel/services/ChannelFollowers.ts b/fluxer_api/src/api/channel/services/ChannelFollowers.ts new file mode 100644 index 000000000..9b3d41f44 --- /dev/null +++ b/fluxer_api/src/api/channel/services/ChannelFollowers.ts @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID} from '@app/api/BrandedTypes'; +import type {ICrosspostedMessageRepository} from '@app/api/channel/repositories/ICrosspostedMessageRepository'; +import {Logger} from '@app/api/Logger'; +import {getSnowflakeService, getWorkerService} from '@app/api/middleware/ServiceRegistry'; +import type {Channel} from '@app/api/models/Channel'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; +import {ThrottledError} from '@fluxer/errors/src/domains/core/ThrottledError'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; + +const CHANNEL_FOLLOW_LOCK_TTL_SECONDS = 5; +const CHANNEL_FOLLOW_LOCK_ACQUIRE_ATTEMPTS = 6; +const CHANNEL_FOLLOW_LOCK_RETRY_DELAY_MS = 50; + +export type ChannelFollowerRemovalReason = 'deleted' | 'converted'; +export type ChannelFollowerRemovalCopyMode = 'source_deleted' | 'purge'; + +export async function withChannelFollowLock( + cacheService: ICacheService, + channelId: ChannelID, + fn: () => Promise, +): Promise { + const lockKey = `channel-follow:${channelId}`; + let lockToken: string | null = null; + for (let attempt = 0; attempt < CHANNEL_FOLLOW_LOCK_ACQUIRE_ATTEMPTS; attempt++) { + lockToken = await cacheService.acquireLock(lockKey, CHANNEL_FOLLOW_LOCK_TTL_SECONDS); + if (lockToken) break; + await new Promise((resolve) => setTimeout(resolve, CHANNEL_FOLLOW_LOCK_RETRY_DELAY_MS * (attempt + 1))); + } + if (!lockToken) { + throw new ThrottledError({ + code: APIErrorCodes.RESOURCE_LOCKED, + retryAfterSeconds: 1, + data: {retry_after: 1}, + }); + } + try { + return await fn(); + } finally { + await cacheService.releaseLock(lockKey, lockToken).catch(() => {}); + } +} + +interface ChannelFollowerRemovalParams { + sourceChannelId: ChannelID; + reason: ChannelFollowerRemovalReason; + copyMode?: ChannelFollowerRemovalCopyMode; +} + +export async function addChannelFollowerRemovalJob(params: ChannelFollowerRemovalParams): Promise { + const {sourceChannelId, reason, copyMode} = params; + const uniqueSuffix = await getSnowflakeService().generate(); + await getWorkerService().addJob( + 'removeChannelFollowers', + { + sourceChannelId: sourceChannelId.toString(), + reason, + ...(copyMode ? {copyMode} : {}), + }, + {jobKey: `remove-followers:${sourceChannelId}:${reason}:${uniqueSuffix}`}, + ); +} + +export async function enqueueChannelFollowerRemoval(params: ChannelFollowerRemovalParams): Promise { + try { + await addChannelFollowerRemovalJob(params); + } catch (error) { + Logger.error( + {error, sourceChannelId: params.sourceChannelId.toString(), reason: params.reason, copyMode: params.copyMode}, + 'Failed to enqueue channel follower removal', + ); + } +} + +export async function channelMayHaveFollowerCopies( + channel: Pick, + crossposts: Pick, +): Promise { + if (channel.type === ChannelTypes.GUILD_ANNOUNCEMENT) return true; + const sources = await crossposts.listSourcesByChannel(channel.id, {limit: 1}); + return sources.length > 0; +} + +export async function scheduleDeletedChannelFollowerRemoval(params: { + channel: Pick; + crossposts: Pick; + copyMode: ChannelFollowerRemovalCopyMode; +}): Promise { + if (!(await channelMayHaveFollowerCopies(params.channel, params.crossposts))) return; + await addChannelFollowerRemovalJob({ + sourceChannelId: params.channel.id, + reason: 'deleted', + copyMode: params.copyMode, + }); +} diff --git a/fluxer_api/src/api/channel/services/ChannelRequestService.ts b/fluxer_api/src/api/channel/services/ChannelRequestService.ts index 488adf0b5..72140d42f 100644 --- a/fluxer_api/src/api/channel/services/ChannelRequestService.ts +++ b/fluxer_api/src/api/channel/services/ChannelRequestService.ts @@ -3,6 +3,7 @@ import type {ChannelID, UserID} from '@app/api/BrandedTypes'; import {mapChannelToResponse} from '@app/api/channel/ChannelMappers'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; +import type {ChannelTypeConversion} from '@app/api/channel/services/channel_data/ChannelOperationsService'; import type {UserCacheService} from '@app/api/infrastructure/UserCacheService'; import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; import type {User} from '@app/api/models/User'; @@ -62,6 +63,7 @@ export class ChannelRequestService { clientFeatures: ReadonlySet; requestCache: RequestCache; auditLogReason: string | null; + typeConversion?: ChannelTypeConversion | null; }): Promise { const channel = await this.channelService.channelData.editChannel({ userId: params.userId, @@ -70,6 +72,7 @@ export class ChannelRequestService { clientFeatures: params.clientFeatures, requestCache: params.requestCache, auditLogReason: params.auditLogReason, + typeConversion: params.typeConversion, }); return mapChannelToResponse({ channel, diff --git a/fluxer_api/src/api/channel/services/ChannelService.ts b/fluxer_api/src/api/channel/services/ChannelService.ts index e7884ac1b..7f12025a6 100644 --- a/fluxer_api/src/api/channel/services/ChannelService.ts +++ b/fluxer_api/src/api/channel/services/ChannelService.ts @@ -86,6 +86,7 @@ export class ChannelService { gatewayService, storageService, purgeQueue, + workerService, }); const messagePersistenceService = new MessagePersistenceService( channelRepository, @@ -119,6 +120,7 @@ export class ChannelService { webhookRepository, limitConfigService, rateLimitService, + cacheService, ); this.messages = new MessageService( channelRepository, diff --git a/fluxer_api/src/api/channel/services/MessageService.ts b/fluxer_api/src/api/channel/services/MessageService.ts index f9a244887..a8bc7d48e 100644 --- a/fluxer_api/src/api/channel/services/MessageService.ts +++ b/fluxer_api/src/api/channel/services/MessageService.ts @@ -2,8 +2,10 @@ import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository'; +import {CrosspostPropagation} from '@app/api/channel/services/message/CrosspostPropagation'; import {MessageAnonymizationService} from '@app/api/channel/services/message/MessageAnonymizationService'; import {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; +import {MessageCrosspostService} from '@app/api/channel/services/message/MessageCrosspostService'; import {MessageDeleteService} from '@app/api/channel/services/message/MessageDeleteService'; import {MessageDispatchService} from '@app/api/channel/services/message/MessageDispatchService'; import {MessageEditService} from '@app/api/channel/services/message/MessageEditService'; @@ -17,6 +19,7 @@ import {MessageSearchService} from '@app/api/channel/services/message/MessageSea import {MessageSendService} from '@app/api/channel/services/message/MessageSendService'; import {MessageSystemService} from '@app/api/channel/services/message/MessageSystemService'; import {MessageValidationService} from '@app/api/channel/services/message/MessageValidationService'; +import {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository'; import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService'; import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate'; @@ -48,6 +51,9 @@ export class MessageService { public readonly deletion: MessageDeleteService; public readonly retrieval: MessageRetrievalService; public readonly anonymization: MessageAnonymizationService; + public readonly writeLock: MessageWriteLock; + public readonly crosspostPropagation: CrosspostPropagation; + public readonly crosspost: MessageCrosspostService; constructor( channelRepository: IChannelRepositoryAggregate, @@ -70,6 +76,8 @@ export class MessageService { limitConfigService: LimitConfigService, ) { this.validation = new MessageValidationService(cacheService, limitConfigService); + this.writeLock = new MessageWriteLock(cacheService, channelRepository.messages); + this.crosspostPropagation = new CrosspostPropagation({rateLimitService, workerService}); this.mention = new MessageMentionService( userRepository, guildRepository, @@ -128,11 +136,12 @@ export class MessageService { attachmentUploadTraceRepository, operationsHelpers, limitConfigService, + messageWriteLock: this.writeLock, + crosspostPropagation: this.crosspostPropagation, }); this.edit = new MessageEditService({ channelRepository, userRepository, - cacheService, validationService: this.validation, persistenceService: this.persistence, channelAuthService: this.channelAuth, @@ -141,6 +150,8 @@ export class MessageService { searchService: this.search, embedAttachmentResolver: this.persistence.getEmbedAttachmentResolver(), mentionService: this.mention, + messageWriteLock: this.writeLock, + crosspostPropagation: this.crosspostPropagation, }); this.deletion = new MessageDeleteService({ channelRepository, @@ -152,6 +163,15 @@ export class MessageService { searchService: this.search, gatewayService, guildAuditLogService, + crosspostPropagation: this.crosspostPropagation, + }); + this.crosspost = new MessageCrosspostService({ + channelRepository, + channelAuthService: this.channelAuth, + dispatchService: this.dispatch, + rateLimitService, + messageWriteLock: this.writeLock, + crosspostPropagation: this.crosspostPropagation, }); this.retrieval = new MessageRetrievalService( channelRepository, diff --git a/fluxer_api/src/api/channel/services/channel_data/ChannelOperationsService.ts b/fluxer_api/src/api/channel/services/channel_data/ChannelOperationsService.ts index 580b8a906..4f3a9ece2 100644 --- a/fluxer_api/src/api/channel/services/channel_data/ChannelOperationsService.ts +++ b/fluxer_api/src/api/channel/services/channel_data/ChannelOperationsService.ts @@ -3,6 +3,11 @@ import type {ChannelID, GuildID, RoleID, UserID} from '@app/api/BrandedTypes'; import {createChannelID, createGuildID, createRoleID, createUserID} from '@app/api/BrandedTypes'; import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; +import { + enqueueChannelFollowerRemoval, + scheduleDeletedChannelFollowerRemoval, + withChannelFollowLock, +} from '@app/api/channel/services/ChannelFollowers'; import type {ChannelAuthService} from '@app/api/channel/services/channel_data/ChannelAuthService'; import type {ChannelUtilsService} from '@app/api/channel/services/channel_data/ChannelUtilsService'; import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService'; @@ -31,13 +36,17 @@ import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType'; import { ALL_PERMISSIONS, + ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES, ChannelTypes, GUILD_TEXT_BASED_CHANNEL_TYPES, Permissions, + WebhookTypes, } from '@fluxer/constants/src/ChannelConstants'; import {ContentWarningLevel, clampVoiceChannelBitrate, GuildFeatures} from '@fluxer/constants/src/GuildConstants'; import {MAX_CHANNELS_PER_CATEGORY} from '@fluxer/constants/src/LimitConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import {ChannelHasFollowedChannelsError} from '@fluxer/errors/src/domains/channel/ChannelHasFollowedChannelsError'; +import {ChannelTypeConversionNotSupportedError} from '@fluxer/errors/src/domains/channel/ChannelTypeConversionNotSupportedError'; import {InvalidChannelTypeError} from '@fluxer/errors/src/domains/channel/InvalidChannelTypeError'; import {MaxCategoryChannelsError} from '@fluxer/errors/src/domains/channel/MaxCategoryChannelsError'; import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError'; @@ -46,6 +55,7 @@ import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidat import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError'; import {resolveLimit} from '@fluxer/limits/src/LimitResolver'; import {ChannelNameType} from '@fluxer/schema/src/primitives/ChannelValidators'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; import type {IRateLimitService} from '@pkgs/rate_limit/src/IRateLimitService'; export interface ChannelUpdateData { @@ -89,6 +99,7 @@ export class ChannelOperationsService { private guildRepository: IGuildRepositoryAggregate, private limitConfigService: LimitConfigService, private rateLimitService: IRateLimitService, + private cacheService: ICacheService, ) {} async getChannel({ @@ -131,6 +142,7 @@ export class ChannelOperationsService { clientFeatures, requestCache, auditLogReason, + typeConversion, }: { userId: UserID; channelId: ChannelID; @@ -138,6 +150,7 @@ export class ChannelOperationsService { clientFeatures: ReadonlySet; requestCache: RequestCache; auditLogReason: string | null; + typeConversion?: ChannelTypeConversion | null; }): Promise { const {channel, guild, checkPermission} = await this.channelAuthService.getChannelAuthenticated({ userId, @@ -149,6 +162,7 @@ export class ChannelOperationsService { } if (!guild) throw new MissingPermissionsError(); await checkPermission(Permissions.MANAGE_CHANNELS); + const nextType = resolveNextChannelType(channel, typeConversion ?? null); const guildIdValue = createGuildID(BigInt(guild.id)); contentModerationService.scanText(data.name ?? null, { userId, @@ -165,7 +179,7 @@ export class ChannelOperationsService { surface: 'profile_field', }); let channelName = data.name ?? channel.name; - if (data.name !== undefined && channel.type === ChannelTypes.GUILD_TEXT) { + if (data.name !== undefined && isTextNamedChannelType(channel.type)) { const hasFlexibleNamesEnabled = guild.features?.includes(GuildFeatures.TEXT_CHANNEL_FLEXIBLE_NAMES) ?? false; if (!hasFlexibleNamesEnabled) { channelName = ChannelNameType.parse(data.name); @@ -262,6 +276,7 @@ export class ChannelOperationsService { } const updatedChannelData = { ...channel.toRow(), + type: nextType, name: channelName, topic: data.topic !== undefined ? data.topic : channel.topic, url: data.url !== undefined && channel.type === ChannelTypes.GUILD_LINK ? data.url : channel.url, @@ -293,7 +308,19 @@ export class ChannelOperationsService { ]), ), }; - const updatedChannel = await this.channelRepository.channelData.upsert(updatedChannelData); + const updatedChannel = + nextType === ChannelTypes.GUILD_ANNOUNCEMENT && channel.type !== ChannelTypes.GUILD_ANNOUNCEMENT + ? await withChannelFollowLock(this.cacheService, channelId, async () => { + const webhooks = await this.webhookRepository.listByChannel(channelId); + if (webhooks.some((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)) { + throw new ChannelHasFollowedChannelsError(); + } + return await this.channelRepository.channelData.upsert(updatedChannelData); + }) + : await this.channelRepository.channelData.upsert(updatedChannelData); + if (channel.type === ChannelTypes.GUILD_ANNOUNCEMENT && nextType !== ChannelTypes.GUILD_ANNOUNCEMENT) { + await enqueueChannelFollowerRemoval({sourceChannelId: channelId, reason: 'converted'}); + } if ( data.rate_limit_per_user !== undefined && GUILD_TEXT_BASED_CHANNEL_TYPES.has(channel.type) && @@ -401,6 +428,11 @@ export class ChannelOperationsService { await this.channelUtilsService.dispatchChannelUpdate({channel: updatedChild, requestCache}); } } + await scheduleDeletedChannelFollowerRemoval({ + channel, + crossposts: this.channelRepository.crossposts, + copyMode: 'source_deleted', + }); const [channelInvites, channelWebhooks] = await Promise.all([ this.inviteRepository.listChannelInvites(channelId), this.webhookRepository.listByChannel(channelId), @@ -736,9 +768,33 @@ export class ChannelOperationsService { } } +export interface ChannelTypeConversion { + from: number; + to: number; +} + +function resolveNextChannelType(channel: Channel, typeConversion: ChannelTypeConversion | null): number { + if (typeConversion === null || typeConversion.to === channel.type) { + return channel.type; + } + if ( + typeConversion.from !== channel.type || + !ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES.has(channel.type) || + !ANNOUNCEMENT_CONVERTIBLE_CHANNEL_TYPES.has(typeConversion.to) + ) { + throw new ChannelTypeConversionNotSupportedError(); + } + return typeConversion.to; +} + +function isTextNamedChannelType(type: number): boolean { + return type === ChannelTypes.GUILD_TEXT || type === ChannelTypes.GUILD_ANNOUNCEMENT; +} + function isWritableGuildChannel(type: number): boolean { return ( type === ChannelTypes.GUILD_TEXT || + type === ChannelTypes.GUILD_ANNOUNCEMENT || type === ChannelTypes.GUILD_VOICE || type === ChannelTypes.GUILD_LINK || type === ChannelTypes.GUILD_CATEGORY diff --git a/fluxer_api/src/api/channel/services/message/CrosspostDeliveryService.ts b/fluxer_api/src/api/channel/services/message/CrosspostDeliveryService.ts new file mode 100644 index 000000000..704d454bb --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/CrosspostDeliveryService.ts @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createHash} from 'node:crypto'; +import type {ChannelID, GuildID, MessageID, UserID, WebhookID} from '@app/api/BrandedTypes'; +import {createMessageID, createUserID} from '@app/api/BrandedTypes'; +import type {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import type {CrosspostedMessageKey} from '@app/api/channel/repositories/ICrosspostedMessageRepository'; +import {dispatchChannelEvent} from '@app/api/channel/services/ChannelGatewayDispatch'; +import { + collectEmbedContentHashes, + type EmbedMediaField, + forEachEmbedMedia, + parseAttachmentUrl, +} from '@app/api/channel/services/message/CrosspostEmbedObjects'; +import {isCrosspostedMessage, isCrosspostSourcePurged} from '@app/api/channel/services/message/CrosspostPropagation'; +import {MessageContentService} from '@app/api/channel/services/message/MessageContentService'; +import { + dispatchMessageCreateBroadcast, + dispatchMessageUpdateBroadcast, +} from '@app/api/channel/services/message/MessageGatewayDispatch'; +import {isOperationDisabled, purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; +import type {MessagePersistenceService} from '@app/api/channel/services/message/MessagePersistenceService'; +import type {MessageSearchService} from '@app/api/channel/services/message/MessageSearchService'; +import {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; +import {checkCrosspostContentRules} from '@app/api/channel/utils/CrosspostContentRules'; +import { + type ContentWarningChannelLike, + channelToContentWarningView, + computeEffectiveChannelNsfw, + guildResponseToContentWarningView, +} from '@app/api/channel/utils/EffectiveContentWarning'; +import type {CrosspostedMessageRow} from '@app/api/database/types/ChannelTypes'; +import type { + MessageAttachment, + MessageEmbed, + MessageEmbedChild, + MessageStickerItem, +} from '@app/api/database/types/MessageTypes'; +import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate'; +import type {AvatarService} from '@app/api/infrastructure/AvatarService'; +import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue'; +import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService'; +import type {IGatewayService} from '@app/api/infrastructure/IGatewayService'; +import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService'; +import type {IStorageService} from '@app/api/infrastructure/IStorageService'; +import {Logger} from '@app/api/Logger'; +import type {LimitConfigService} from '@app/api/limits/LimitConfigService'; +import type {Channel} from '@app/api/models/Channel'; +import type {Message} from '@app/api/models/Message'; +import type {Webhook} from '@app/api/models/Webhook'; +import {deleteMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup'; +import type {IUserRepository} from '@app/api/user/IUserRepository'; +import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; +import { + CROSSPOST_PENDING_RECLAIM_AFTER_MS, + CROSSPOST_SOURCE_DELETED_CONTENT, +} from '@fluxer/constants/src/AnnouncementConstants'; +import { + CHANNEL_FOLLOW_TARGET_TYPES, + ChannelTypes, + MessageFlags, + MessageReferenceTypes, + MessageTypes, + Permissions, + SENDABLE_MESSAGE_FLAGS, + WebhookTypes, +} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures, GuildOperations} from '@fluxer/constants/src/GuildConstants'; +import {ContentBlockedError} from '@fluxer/errors/src/domains/content/ContentBlockedError'; +import {UnknownGuildError} from '@fluxer/errors/src/domains/guild/UnknownGuildError'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; + +export type CrosspostCopySyncMode = 'update' | 'source_deleted' | 'purge'; + +export type CrosspostDeliveryOutcome = 'delivered' | 'skipped'; + +export interface CrosspostSourceContext { + message: Message; + channel: Channel; + parent: Channel | null; + guild: GuildResponse; + fingerprint: string; + accessCache: Map>; + authorAvatarCache: Map>; +} + +export type CrosspostSourceLoadResult = + | {kind: 'missing'} + | {kind: 'inactive'} + | {kind: 'ready'; context: CrosspostSourceContext}; + +interface CrosspostTarget { + channel: Channel; + parent: Channel | null; + guild: GuildResponse; +} + +interface CrosspostCopyPayload { + content: string | null; + flags: number; + attachments: Array; + embeds: Array; + stickerItems: Array; +} + +type CrosspostCopyPayloadResult = {kind: 'blocked'} | {kind: 'ready'; payload: CrosspostCopyPayload}; + +export interface CrosspostDeliveryDeps { + channelRepository: ChannelRepository; + webhookRepository: IWebhookRepository; + userRepository: IUserRepository; + guildRepository: IGuildRepositoryAggregate; + gatewayService: IGatewayService; + storageService: IStorageService; + avatarService: AvatarService; + purgeQueue: IPurgeQueue; + snowflakeService: ISnowflakeService; + cacheService: ICacheService; + limitConfigService: LimitConfigService; + persistenceService: MessagePersistenceService; + searchService: MessageSearchService; +} + +const UNAVAILABLE_GUILD_FEATURES: ReadonlyArray = [ + GuildFeatures.UNAVAILABLE_FOR_EVERYONE, + GuildFeatures.UNAVAILABLE_FOR_EVERYONE_BUT_STAFF, + GuildFeatures.UNAVAILABLE_HIDDEN, +]; + +export class CrosspostDeliveryPendingError extends Error { + constructor(sourceMessageId: MessageID, webhookId: WebhookID) { + super(`Crosspost delivery for ${sourceMessageId} to webhook ${webhookId} is still pending`); + this.name = 'CrosspostDeliveryPendingError'; + } +} + +export class CrosspostSyncConflictError extends Error { + constructor(sourceMessageId: MessageID, webhookId: WebhookID) { + super(`Crosspost copy for ${sourceMessageId} via webhook ${webhookId} changed during sync`); + this.name = 'CrosspostSyncConflictError'; + } +} + +function toStableValue(value: unknown): unknown { + if (value === null || value === undefined) return null; + if (typeof value === 'bigint') return value.toString(); + if (value instanceof Date) return value.toISOString(); + if (value instanceof Set) return [...value].map(toStableValue).sort(); + if (value instanceof Map) { + return [...value.entries()] + .map(([key, entry]) => [String(key), toStableValue(entry)] as const) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + } + if (Array.isArray(value)) return value.map(toStableValue); + if (typeof value === 'object') { + const result: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + const entry = (value as Record)[key]; + if (entry === undefined || entry === null) continue; + result[key] = toStableValue(entry); + } + return result; + } + return value; +} + +export function crosspostSourceFingerprint(source: Message): string { + const state = { + content: source.content ?? null, + flags: source.flags & SENDABLE_MESSAGE_FLAGS, + stickers: source.stickers.map((sticker) => sticker.id.toString()), + embeds: source.embeds.map((embed) => embed.toMessageEmbed()), + attachments: source.attachments.map((attachment) => ({ + id: attachment.id, + filename: attachment.filename, + title: attachment.title, + description: attachment.description, + flags: attachment.flags, + nsfw: attachment.nsfw, + })), + }; + return createHash('sha256') + .update(JSON.stringify(toStableValue(state))) + .digest('hex'); +} + +function isGuildUnavailable(guild: GuildResponse): boolean { + return guild.features.some((feature) => UNAVAILABLE_GUILD_FEATURES.includes(feature)); +} + +function withoutNsfwChildren(embed: MessageEmbed): MessageEmbed { + if (!embed.children || embed.children.length === 0) return embed; + const children = embed.children.filter((child) => !child.nsfw); + return {...embed, children: children.length > 0 ? children : null}; +} + +function cloneEmbed(embed: MessageEmbed): MessageEmbed { + const cloneChild = (child: MessageEmbedChild): MessageEmbedChild => ({ + ...child, + thumbnail: child.thumbnail ? {...child.thumbnail} : child.thumbnail, + image: child.image ? {...child.image} : child.image, + video: child.video ? {...child.video} : child.video, + audio: child.audio ? {...child.audio} : child.audio, + }); + return { + ...cloneChild(embed), + children: embed.children ? embed.children.map(cloneChild) : embed.children, + }; +} + +export class CrosspostDeliveryService { + private readonly writeLock: MessageWriteLock; + private readonly contentService: MessageContentService; + + constructor(private readonly deps: CrosspostDeliveryDeps) { + this.writeLock = new MessageWriteLock(deps.cacheService, deps.channelRepository.messages); + this.contentService = new MessageContentService(deps.userRepository, deps.guildRepository, deps.limitConfigService); + } + + async loadSource(channelId: ChannelID, messageId: MessageID): Promise { + const message = await this.deps.channelRepository.messages.getMessage(channelId, messageId); + if (!message) return null; + const channel = await this.deps.channelRepository.findUnique(channelId); + if (!channel?.guildId) return null; + const guild = await this.loadGuild(channel.guildId); + if (!guild) return null; + const parent = await this.loadParent(channel); + return { + message, + channel, + parent, + guild, + fingerprint: crosspostSourceFingerprint(message), + accessCache: new Map(), + authorAvatarCache: new Map(), + }; + } + + async loadSourceForDelivery(channelId: ChannelID, messageId: MessageID): Promise { + const context = await this.loadSource(channelId, messageId); + if (!context) return {kind: 'missing'}; + if ( + !isCrosspostedMessage(context.message) || + context.channel.type !== ChannelTypes.GUILD_ANNOUNCEMENT || + !this.isSourceGuildActive(context.guild) + ) { + return {kind: 'inactive'}; + } + return {kind: 'ready', context}; + } + + isSourceGuildActive(guild: GuildResponse): boolean { + return ( + !isGuildUnavailable(guild) && + !isOperationDisabled(guild, GuildOperations.SEND_MESSAGE) && + !guild.features.includes(GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED) + ); + } + + async deliverToWebhook(context: CrosspostSourceContext, webhookId: WebhookID): Promise { + const logContext = { + sourceMessageId: context.message.id.toString(), + webhookId: webhookId.toString(), + }; + const webhook = await this.deps.webhookRepository.findUnique(webhookId); + if ( + !webhook || + webhook.type !== WebhookTypes.CHANNEL_FOLLOWER || + webhook.sourceChannelId !== context.channel.id || + !webhook.channelId + ) { + return 'skipped'; + } + if (!(await this.creatorCanViewSource(context, webhook.creatorId))) { + Logger.info(logContext, 'Skipping crosspost delivery: follower creator cannot view the source channel'); + return 'skipped'; + } + const target = await this.loadDeliveryTarget(webhook.channelId); + if (!target) { + Logger.info(logContext, 'Skipping crosspost delivery: target channel cannot receive copies'); + return 'skipped'; + } + if (this.checkContentRules(context, target) !== 'ok') { + Logger.info(logContext, 'Skipping crosspost delivery: target does not meet the content rules'); + return 'skipped'; + } + return this.deliverToTarget(context, webhook, target, true); + } + + private async deliverToTarget( + context: CrosspostSourceContext, + webhook: Webhook, + target: CrosspostTarget, + allowRestart: boolean, + ): Promise { + const {crossposts} = this.deps.channelRepository; + const key: CrosspostedMessageKey = {sourceMessageId: context.message.id, webhookId: webhook.id}; + const existing = await crossposts.get(key.sourceMessageId, key.webhookId); + if (existing?.state === 'delivered') { + await this.closeDeliveryRace(context, key); + return 'skipped'; + } + if (existing?.state === 'pending') { + const copy = await this.deps.channelRepository.messages.getMessage( + existing.target_channel_id, + existing.target_message_id, + ); + if (copy) { + const marked = await crossposts.markDelivered(key, { + targetMessageId: existing.target_message_id, + sourceFingerprint: null, + }); + if (!marked) return 'skipped'; + const copyChannel = + existing.target_channel_id === target.channel.id + ? target.channel + : await this.deps.channelRepository.findUnique(existing.target_channel_id); + if (copyChannel) { + await this.announceCopy(copyChannel, copy); + } + await this.closeDeliveryRace(context, key); + return 'delivered'; + } + if (Date.now() - existing.reserved_at.getTime() < CROSSPOST_PENDING_RECLAIM_AFTER_MS) { + throw new CrosspostDeliveryPendingError(key.sourceMessageId, key.webhookId); + } + } + const authorAvatar = await this.resolveCopyAuthorAvatar(context, webhook); + const built = this.buildCopyPayload(context, target); + if (built.kind === 'blocked') { + Logger.warn( + {sourceMessageId: context.message.id.toString(), webhookId: webhook.id.toString()}, + 'Skipping crosspost delivery: blocked content', + ); + return 'skipped'; + } + const {payload} = built; + const messageId = createMessageID(await this.deps.snowflakeService.generateForChannel(target.channel.id)); + const now = new Date(); + const reserved = + existing?.state === 'pending' + ? await crossposts.reclaimPending(key, { + fromTargetMessageId: existing.target_message_id, + toTargetMessageId: messageId, + reservedAt: now, + }) + : await crossposts.insertPending({ + source_message_id: key.sourceMessageId, + webhook_id: key.webhookId, + source_channel_id: context.channel.id, + target_guild_id: target.channel.guildId!, + target_channel_id: target.channel.id, + target_message_id: messageId, + state: 'pending', + reserved_at: now, + source_fingerprint: null, + created_at: now, + }); + if (!reserved) { + if (allowRestart) { + return this.deliverToTarget(context, webhook, target, false); + } + throw new CrosspostDeliveryPendingError(key.sourceMessageId, key.webhookId); + } + const copy = await this.createCopy(context, webhook, target, messageId, payload, authorAvatar); + const marked = await crossposts.markDelivered(key, { + targetMessageId: messageId, + sourceFingerprint: context.fingerprint, + }); + if (!marked) { + await this.deps.channelRepository.deleteMessage(target.channel.id, messageId, createUserID(0n)); + return 'skipped'; + } + await this.announceCopy(target.channel, copy); + await this.closeDeliveryRace(context, key); + return 'delivered'; + } + + private async createCopy( + context: CrosspostSourceContext, + webhook: Webhook, + target: CrosspostTarget, + messageId: MessageID, + payload: CrosspostCopyPayload, + authorAvatar: string | null, + ): Promise { + try { + const {message} = await this.deps.persistenceService.createMessage({ + messageId, + channelId: target.channel.id, + webhookId: webhook.id, + webhookName: webhook.name, + webhookAvatar: authorAvatar, + type: MessageTypes.DEFAULT, + content: payload.content, + flags: payload.flags, + processedAttachments: payload.attachments, + processedEmbeds: payload.embeds, + processedStickerItems: payload.stickerItems, + messageReference: { + guild_id: context.channel.guildId, + channel_id: context.channel.id, + message_id: context.message.id, + type: MessageReferenceTypes.DEFAULT, + }, + mentionData: { + flags: payload.flags, + mentionUserIds: [], + mentionRoleIds: [], + mentionChannelIds: [], + mentionEveryone: false, + }, + guildId: target.channel.guildId, + skipDeferredEmbeds: true, + }); + return message; + } catch (error) { + await this.deps.channelRepository.deleteMessage(target.channel.id, messageId, createUserID(0n)); + await this.deps.channelRepository.crossposts.delete( + {sourceMessageId: context.message.id, webhookId: webhook.id}, + {state: 'pending', target_message_id: messageId}, + ); + throw error; + } + } + + private resolveCopyAuthorAvatar(context: CrosspostSourceContext, webhook: Webhook): Promise { + const iconHash = context.guild.icon ?? null; + if (!iconHash) return Promise.resolve(null); + if (iconHash === webhook.avatarHash) return Promise.resolve(iconHash); + const cacheKey = `${webhook.id}:${iconHash}`; + const cached = context.authorAvatarCache.get(cacheKey); + if (cached) return cached; + const pending = this.deps.avatarService.ensureWebhookAvatarFromGuildIcon({ + guildId: context.channel.guildId!, + iconHash, + webhookId: webhook.id, + }); + context.authorAvatarCache.set(cacheKey, pending); + pending.catch(() => context.authorAvatarCache.delete(cacheKey)); + return pending; + } + + private async announceCopy(channel: Channel, copy: Message): Promise { + await dispatchMessageCreateBroadcast({gatewayService: this.deps.gatewayService, channel, message: copy}); + if (channel.indexedAt != null) { + void this.deps.searchService.indexMessage(copy, false, {includeDefault: true}); + } + } + + private async closeDeliveryRace(context: CrosspostSourceContext, key: CrosspostedMessageKey): Promise { + const row = await this.deps.channelRepository.crossposts.get(key.sourceMessageId, key.webhookId); + if (!row) return; + const latest = await this.deps.channelRepository.messages.getMessage(context.channel.id, context.message.id); + if (!latest) { + await this.syncCopy(row, await this.removalModeFor(context.message.id), null); + return; + } + if (crosspostSourceFingerprint(latest) !== row.source_fingerprint) { + await this.syncCopy(row, 'update', await this.loadSource(context.channel.id, context.message.id)); + } + } + + async removalModeFor(sourceMessageId: MessageID): Promise<'source_deleted' | 'purge'> { + return (await isCrosspostSourcePurged(sourceMessageId)) ? 'purge' : 'source_deleted'; + } + + async syncCopy( + row: CrosspostedMessageRow, + mode: CrosspostCopySyncMode, + source: CrosspostSourceContext | null, + ): Promise { + if (mode === 'update') { + await this.syncUpdate(row, source); + return; + } + if (mode === 'source_deleted') { + await this.markCopySourceDeleted(row); + return; + } + await this.purgeCopy(row); + } + + private keyOf(row: CrosspostedMessageRow): CrosspostedMessageKey { + return {sourceMessageId: row.source_message_id, webhookId: row.webhook_id}; + } + + private async syncUpdate(initialRow: CrosspostedMessageRow, source: CrosspostSourceContext | null): Promise { + const {crossposts} = this.deps.channelRepository; + const key = this.keyOf(initialRow); + const row = await crossposts.get(key.sourceMessageId, key.webhookId); + if (row?.state !== 'delivered') return; + if (!source) return; + if (source.guild.features.includes(GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED)) { + await this.dropRemovedSourceAttachments(row, source); + return; + } + const fingerprint = source.fingerprint; + if (fingerprint === row.source_fingerprint) return; + const copy = await this.deps.channelRepository.messages.getMessage(row.target_channel_id, row.target_message_id); + if (!copy) { + await crossposts.delete(key, {state: 'delivered', target_message_id: row.target_message_id}); + return; + } + if ((copy.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0) return; + const target = await this.loadSyncTarget(row.target_channel_id); + if (!target) return; + if (!(await this.syncStillAllowed(source, row, target))) { + await this.markCopySourceDeleted(row); + return; + } + const built = this.buildCopyPayload(source, target); + if (built.kind === 'blocked') { + Logger.warn( + {sourceMessageId: row.source_message_id.toString(), webhookId: row.webhook_id.toString()}, + 'Crosspost sync found blocked content, marking the copy as source deleted', + ); + await this.markCopySourceDeleted(row); + return; + } + const {payload} = built; + const result = await this.writeLock.withFreshMessage( + row.target_channel_id, + row.target_message_id, + async (fresh) => { + const current = await crossposts.get(key.sourceMessageId, key.webhookId); + const latestSource = await this.deps.channelRepository.messages.getMessage( + source.channel.id, + source.message.id, + ); + if (current?.state === 'delivered' && current.source_fingerprint === fingerprint) { + return {kind: 'current' as const}; + } + if ( + current?.state !== 'delivered' || + current.target_message_id !== row.target_message_id || + !fresh || + (fresh.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0 || + !latestSource || + crosspostSourceFingerprint(latestSource) !== fingerprint + ) { + return {kind: 'conflict' as const}; + } + const updated = await this.deps.channelRepository.messages.upsertMessage( + { + ...fresh.toRow(), + content: payload.content, + embeds: payload.embeds.length > 0 ? payload.embeds : null, + attachments: payload.attachments.length > 0 ? payload.attachments : null, + sticker_items: payload.stickerItems.length > 0 ? payload.stickerItems : null, + flags: (fresh.flags & ~SENDABLE_MESSAGE_FLAGS) | payload.flags, + edited_timestamp: new Date(), + }, + fresh.toRow(), + ); + await crossposts.updateSynced(key, { + targetMessageId: row.target_message_id, + sourceFingerprint: fingerprint, + }); + return {kind: 'synced' as const, updated}; + }, + ); + if (result.kind === 'current') return; + if (result.kind === 'conflict') { + throw new CrosspostSyncConflictError(key.sourceMessageId, key.webhookId); + } + await dispatchMessageUpdateBroadcast({ + gatewayService: this.deps.gatewayService, + channel: target.channel, + message: result.updated, + }); + if (target.channel.indexedAt != null) { + void this.deps.searchService.updateMessageIndex(result.updated, {includeDefault: true}); + } + } + + private async dropRemovedSourceAttachments( + row: CrosspostedMessageRow, + source: CrosspostSourceContext, + ): Promise { + const updated = await this.writeLock.withFreshMessage( + row.target_channel_id, + row.target_message_id, + async (fresh) => { + if (!fresh || (fresh.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0) return null; + const latestSource = await this.deps.channelRepository.messages.getMessage( + source.channel.id, + source.message.id, + ); + if (!latestSource) return null; + const liveIds = new Set(latestSource.attachments.map((attachment) => attachment.id)); + const kept = fresh.attachments.filter((attachment) => liveIds.has(attachment.id)); + if (kept.length === fresh.attachments.length) return null; + return this.deps.channelRepository.messages.upsertMessage( + { + ...fresh.toRow(), + attachments: kept.length > 0 ? kept.map((attachment) => attachment.toMessageAttachment()) : null, + edited_timestamp: new Date(), + }, + fresh.toRow(), + ); + }, + ); + if (!updated) return; + const channel = await this.deps.channelRepository.findUnique(row.target_channel_id); + if (!channel) return; + await dispatchMessageUpdateBroadcast({gatewayService: this.deps.gatewayService, channel, message: updated}); + if (channel.indexedAt != null) { + void this.deps.searchService.updateMessageIndex(updated, {includeDefault: true}); + } + } + + private async syncStillAllowed( + source: CrosspostSourceContext, + row: CrosspostedMessageRow, + target: CrosspostTarget, + ): Promise { + if (this.checkContentRules(source, target) !== 'ok') return false; + const webhook = await this.deps.webhookRepository.findUnique(row.webhook_id); + if (!webhook || webhook.sourceChannelId !== source.channel.id) return true; + return this.creatorCanViewSource(source, webhook.creatorId); + } + + private isStaleRow(row: CrosspostedMessageRow): boolean { + return row.state === 'delivered' || Date.now() - row.reserved_at.getTime() >= CROSSPOST_PENDING_RECLAIM_AFTER_MS; + } + + private async markCopySourceDeleted(row: CrosspostedMessageRow): Promise { + const {crossposts} = this.deps.channelRepository; + const key = this.keyOf(row); + const outcome = await this.writeLock.withFreshMessage( + row.target_channel_id, + row.target_message_id, + async (fresh) => { + if (!fresh) return {kind: 'missing' as const}; + if ((fresh.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0) { + return {kind: 'already' as const}; + } + const updated = await this.deps.channelRepository.messages.upsertMessage( + { + ...fresh.toRow(), + content: CROSSPOST_SOURCE_DELETED_CONTENT, + attachments: null, + embeds: null, + sticker_items: null, + flags: MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED, + edited_timestamp: new Date(), + }, + fresh.toRow(), + ); + return {kind: 'marked' as const, updated}; + }, + ); + if (outcome.kind === 'missing') { + if (this.isStaleRow(row)) { + await crossposts.delete(key); + } + return; + } + if (outcome.kind === 'marked') { + const channel = await this.deps.channelRepository.findUnique(row.target_channel_id); + if (channel) { + await dispatchMessageUpdateBroadcast({ + gatewayService: this.deps.gatewayService, + channel, + message: outcome.updated, + }); + if (channel.indexedAt != null) { + void this.deps.searchService.updateMessageIndex(outcome.updated, {includeDefault: true}); + } + } + } + await crossposts.delete(key); + } + + private async purgeCopy(row: CrosspostedMessageRow): Promise { + const {crossposts} = this.deps.channelRepository; + const key = this.keyOf(row); + const removed = await this.writeLock.withFreshMessage( + row.target_channel_id, + row.target_message_id, + async (fresh) => { + if (!fresh) return null; + await this.deps.channelRepository.deleteMessage( + fresh.channelId, + fresh.id, + fresh.authorId ?? createUserID(0n), + fresh.pinnedTimestamp ?? undefined, + ); + return fresh; + }, + ); + if (!removed) { + if (this.isStaleRow(row)) { + await crossposts.delete(key); + } + return; + } + const channel = await this.deps.channelRepository.findUnique(row.target_channel_id); + if (channel) { + await dispatchChannelEvent({ + gatewayService: this.deps.gatewayService, + channel, + event: 'MESSAGE_DELETE', + data: {channel_id: channel.id.toString(), id: removed.id.toString()}, + }); + } + await deleteMessageSearchDocuments([removed.id], {context: {source: 'crosspost_purge'}}); + await crossposts.delete(key); + } + + async deleteSourceMessage(channelId: ChannelID, messageId: MessageID): Promise { + const removed = await this.writeLock.withFreshMessage(channelId, messageId, async (fresh) => { + if (!fresh) return null; + await this.deps.channelRepository.deleteMessage( + channelId, + messageId, + fresh.authorId ?? createUserID(0n), + fresh.pinnedTimestamp ?? undefined, + ); + return fresh; + }); + if (!removed) return; + await purgeMessageAttachments(removed, this.deps.storageService, this.deps.purgeQueue); + const channel = await this.deps.channelRepository.findUnique(channelId); + if (channel) { + await dispatchChannelEvent({ + gatewayService: this.deps.gatewayService, + channel, + event: 'MESSAGE_DELETE', + data: {channel_id: channelId.toString(), id: messageId.toString()}, + }); + } + await deleteMessageSearchDocuments([messageId], {context: {source: 'crosspost_family_purge'}}); + } + + private buildCopyPayload(source: CrosspostSourceContext, target: CrosspostTarget): CrosspostCopyPayloadResult { + const message = source.message; + if (this.isBlocked(source)) { + return {kind: 'blocked'}; + } + const nsfwAllowed = this.targetAllowsNsfw(target); + const excludedAttachmentIds = new Set(); + const attachments: Array = []; + for (const attachment of message.attachments) { + if (!nsfwAllowed && attachment.nsfw) { + excludedAttachmentIds.add(attachment.id.toString()); + continue; + } + attachments.push(attachment.toMessageAttachment()); + } + let embeds = message.embeds.map((embed) => cloneEmbed(embed.toMessageEmbed())); + if (!nsfwAllowed) { + embeds = embeds.filter((embed) => !embed.nsfw).map(withoutNsfwChildren); + } + const removedMedia: Array<{owner: MessageEmbedChild; field: EmbedMediaField}> = []; + forEachEmbedMedia(embeds, (media, owner, field) => { + const parsed = parseAttachmentUrl(media.url, source.channel.id); + if (parsed && excludedAttachmentIds.has(parsed.id)) { + removedMedia.push({owner, field}); + } + }); + for (const {owner, field} of removedMedia) { + owner[field] = null; + } + return { + kind: 'ready', + payload: { + content: message.content, + flags: MessageFlags.IS_CROSSPOST | (message.flags & SENDABLE_MESSAGE_FLAGS), + attachments, + embeds, + stickerItems: message.stickers.map((sticker) => sticker.toMessageStickerItem()), + }, + }; + } + + private isBlocked(source: CrosspostSourceContext): boolean { + const message = source.message; + const context: Omit = { + userId: message.authorId, + guildId: source.channel.guildId, + channelId: message.channelId, + messageId: message.id, + }; + try { + const textContext: ModerationContext = {...context, surface: 'message_content'}; + contentModerationService.scanText(message.content, textContext); + for (const embed of message.embeds) { + if (embed.type !== 'rich') continue; + contentModerationService.scanText(embed.title, textContext); + contentModerationService.scanText(embed.description, textContext); + for (const field of embed.fields) { + contentModerationService.scanText(field.name, textContext); + contentModerationService.scanText(field.value, textContext); + } + contentModerationService.scanText(embed.footer?.text, textContext); + contentModerationService.scanText(embed.author?.name, textContext); + } + for (const attachment of message.attachments) { + if (attachment.contentHash) { + contentModerationService.scanSha256(attachment.contentHash, {...context, surface: 'message_attachment'}); + } + } + for (const contentHash of collectEmbedContentHashes(message)) { + contentModerationService.scanSha256(contentHash, {...context, surface: 'message_attachment'}); + } + } catch (error) { + if (error instanceof ContentBlockedError) return true; + throw error; + } + return false; + } + + private targetAllowsNsfw(target: CrosspostTarget): boolean { + if ( + computeEffectiveChannelNsfw( + channelToContentWarningView(target.channel), + target.parent ? channelToContentWarningView(target.parent) : null, + guildResponseToContentWarningView(target.guild), + ) + ) { + return true; + } + return this.contentService.isNSFWContentAllowed({ + channel: target.channel, + guild: target.guild, + member: null, + isBot: false, + }); + } + + private checkContentRules(source: CrosspostSourceContext, target: CrosspostTarget) { + return checkCrosspostContentRules({ + source: channelToContentWarningView(source.channel), + sourceParent: this.parentView(source.parent), + sourceGuild: guildResponseToContentWarningView(source.guild), + target: channelToContentWarningView(target.channel), + targetParent: this.parentView(target.parent), + targetGuild: guildResponseToContentWarningView(target.guild), + }); + } + + private parentView(parent: Channel | null): ContentWarningChannelLike | null { + return parent ? channelToContentWarningView(parent) : null; + } + + private async creatorCanViewSource(source: CrosspostSourceContext, creatorId: UserID | null): Promise { + if (!creatorId || !source.channel.guildId) return false; + const cacheKey = creatorId.toString(); + let cached = source.accessCache.get(cacheKey); + if (!cached) { + cached = this.deps.gatewayService.checkPermission({ + guildId: source.channel.guildId, + userId: creatorId, + permission: Permissions.VIEW_CHANNEL, + channelId: source.channel.id, + }); + source.accessCache.set(cacheKey, cached); + } + return cached; + } + + private async loadDeliveryTarget(channelId: ChannelID): Promise { + const target = await this.loadSyncTarget(channelId); + if (!target) return null; + if (!CHANNEL_FOLLOW_TARGET_TYPES.has(target.channel.type)) return null; + if (isGuildUnavailable(target.guild) || isOperationDisabled(target.guild, GuildOperations.SEND_MESSAGE)) { + return null; + } + return target; + } + + private async loadSyncTarget(channelId: ChannelID): Promise { + const channel = await this.deps.channelRepository.findUnique(channelId); + if (!channel?.guildId) return null; + const guild = await this.loadGuild(channel.guildId); + if (!guild) return null; + return {channel, parent: await this.loadParent(channel), guild}; + } + + private async loadParent(channel: Channel): Promise { + if (!channel.parentId || channel.type === ChannelTypes.GUILD_CATEGORY) return null; + return this.deps.channelRepository.findUnique(channel.parentId); + } + + private async loadGuild(guildId: GuildID): Promise { + try { + return await this.deps.gatewayService.getGuildData({ + guildId, + userId: createUserID(0n), + skipMembershipCheck: true, + }); + } catch (error) { + if (error instanceof UnknownGuildError) { + return null; + } + throw error; + } + } +} diff --git a/fluxer_api/src/api/channel/services/message/CrosspostEmbedObjects.ts b/fluxer_api/src/api/channel/services/message/CrosspostEmbedObjects.ts new file mode 100644 index 000000000..fac328792 --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/CrosspostEmbedObjects.ts @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import type {MessageEmbed, MessageEmbedChild, MessageEmbedMedia} from '@app/api/database/types/MessageTypes'; +import type {Message} from '@app/api/models/Message'; + +const EMBED_MEDIA_FIELDS = ['image', 'thumbnail', 'video', 'audio'] as const; + +export type EmbedMediaField = (typeof EMBED_MEDIA_FIELDS)[number]; + +function attachmentPrefix(channelId: ChannelID): string { + return `${Config.endpoints.media}/attachments/${channelId}/`; +} + +export function parseAttachmentUrl( + url: string | null | undefined, + channelId: ChannelID, +): {id: string; filename: string} | null { + const prefix = attachmentPrefix(channelId); + if (!url?.startsWith(prefix)) return null; + const rest = url.slice(prefix.length); + const separator = rest.indexOf('/'); + if (separator <= 0) return null; + const id = rest.slice(0, separator); + const filename = rest.slice(separator + 1).split('?')[0] ?? ''; + if (!/^\d+$/.test(id) || filename.length === 0) return null; + return {id, filename}; +} + +export function forEachEmbedMedia( + embeds: ReadonlyArray, + visit: (media: MessageEmbedMedia, owner: MessageEmbedChild, field: EmbedMediaField) => void, +): void { + const visitOwner = (owner: MessageEmbedChild) => { + for (const field of EMBED_MEDIA_FIELDS) { + const media = owner[field]; + if (media) visit(media, owner, field); + } + }; + for (const embed of embeds) { + visitOwner(embed); + for (const child of embed.children ?? []) { + visitOwner(child); + } + } +} + +export function collectEmbedContentHashes(message: Message): Array { + const hashes: Array = []; + forEachEmbedMedia( + message.embeds.map((embed) => embed.toMessageEmbed()), + (media) => { + if (media.content_hash) hashes.push(media.content_hash); + }, + ); + return hashes; +} diff --git a/fluxer_api/src/api/channel/services/message/CrosspostPropagation.ts b/fluxer_api/src/api/channel/services/message/CrosspostPropagation.ts new file mode 100644 index 000000000..70ee87d43 --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/CrosspostPropagation.ts @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, MessageID} from '@app/api/BrandedTypes'; +import {isCrosspostCopy} from '@app/api/channel/services/message/MessageHelpers'; +import {Logger} from '@app/api/Logger'; +import {getKVClient} from '@app/api/middleware/ServiceRegistry'; +import type {Channel} from '@app/api/models/Channel'; +import type {Message} from '@app/api/models/Message'; +import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig'; +import { + CROSSPOST_SYNC_COALESCE_MS, + PUBLISHED_MESSAGE_EDIT_RATE_LIMIT, +} from '@fluxer/constants/src/AnnouncementConstants'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes, MessageFlags} from '@fluxer/constants/src/ChannelConstants'; +import {RateLimitError} from '@fluxer/errors/src/domains/core/RateLimitError'; +import type {IRateLimitService, RateLimitConfig, RateLimitResult} from '@pkgs/rate_limit/src/IRateLimitService'; +import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; +import {z} from 'zod'; + +export const CrosspostTaskNames = { + CROSSPOST_MESSAGE: 'crosspostMessage', + CROSSPOST_MESSAGE_CHUNK: 'crosspostMessageChunk', + SYNC_CROSSPOSTED_MESSAGE: 'syncCrosspostedMessage', + SYNC_CROSSPOST_COPIES: 'syncCrosspostCopies', + REMOVE_CHANNEL_FOLLOWERS: 'removeChannelFollowers', +} as const; + +export type CrosspostTaskName = (typeof CrosspostTaskNames)[keyof typeof CrosspostTaskNames]; + +export type CrosspostWorkerService = IWorkerService; + +export const CrosspostSyncModeSchema = z.enum(['update', 'source_deleted', 'purge']); +export type CrosspostSyncMode = z.infer; +export type CrosspostRemovalMode = Exclude; + +export const CrosspostMessagePayloadSchema = z.object({ + channelId: z.string(), + messageId: z.string(), + afterWebhookId: z.string().optional(), +}); +export type CrosspostMessagePayload = z.infer; + +export const CrosspostMessageChunkPayloadSchema = z.object({ + channelId: z.string(), + messageId: z.string(), + webhookIds: z.array(z.string()).min(1), + attempt: z.number().int().min(0), +}); + +export const SyncCrosspostedMessagePayloadSchema = z.object({ + channelId: z.string(), + messageId: z.string(), + mode: CrosspostSyncModeSchema, + deleteSource: z.boolean().optional(), +}); +export type SyncCrosspostedMessagePayload = z.infer; + +export const SyncCrosspostCopiesPayloadSchema = z.object({ + channelId: z.string(), + messageId: z.string(), + mode: CrosspostSyncModeSchema, + webhookIds: z.array(z.string()).min(1), +}); + +export const RemoveChannelFollowersPayloadSchema = z.object({ + sourceChannelId: z.string(), + reason: z.enum(['deleted', 'converted']), + copyMode: z.enum(['source_deleted', 'purge']).optional(), +}); + +export type PublishedEditActor = 'author' | 'webhook' | 'moderator'; + +const CROSSPOST_PURGE_MARKER_TTL_SECONDS = 86_400; + +type CrosspostMessageLike = Pick; + +export function isCrosspostedMessage(message: Pick): boolean { + return (message.flags & MessageFlags.CROSSPOSTED) !== 0; +} + +export function crosspostFanoutJobKey(messageId: string, afterWebhookId?: string): string { + return afterWebhookId ? `crosspost:${messageId}:${afterWebhookId}` : `crosspost:${messageId}`; +} + +export function crosspostChunkJobKey(messageId: string, firstWebhookId: string, attempt: number): string { + return `crosspost-chunk:${messageId}:${firstWebhookId}:${attempt}`; +} + +export function crosspostSyncBucket(nowMs: number): number { + return Math.floor(nowMs / CROSSPOST_SYNC_COALESCE_MS); +} + +export function crosspostSyncJobKey(messageId: string, mode: CrosspostSyncMode, bucket?: number): string { + return mode === 'update' ? `crosspost-sync:${messageId}:update:${bucket}` : `crosspost-sync:${messageId}:${mode}`; +} + +export function crosspostSyncChunkJobKey(params: { + messageId: string; + mode: CrosspostSyncMode; + bucketOrMode: string; + firstWebhookId: string; +}): string { + return `crosspost-sync-chunk:${params.messageId}:${params.mode}:${params.bucketOrMode}:${params.firstWebhookId}`; +} + +export function publishedEditRateLimitIdentifier(messageId: MessageID): string { + return `crosspost:edit:${messageId}`; +} + +function crosspostPurgeMarkerKey(messageId: MessageID | string): string { + return `crosspost:purged:${messageId}`; +} + +export async function isCrosspostSourcePurged(messageId: MessageID): Promise { + return (await getKVClient().exists(crosspostPurgeMarkerKey(messageId))) > 0; +} + +export function withPeekRetryAfter(result: RateLimitResult, config: RateLimitConfig): RateLimitResult { + const leakPerMs = config.maxAttempts / config.windowMs; + const retryAfterMs = Math.max(1, Math.ceil(result.resetAfterDecimal * 1000 - (config.maxAttempts - 1) / leakPerMs)); + return { + ...result, + allowed: false, + remaining: 0, + retryAfter: Math.max(1, Math.ceil(retryAfterMs / 1000)), + retryAfterDecimal: retryAfterMs / 1000, + }; +} + +export function createCrosspostRateLimitError(code: string, result: RateLimitResult): RateLimitError { + return new RateLimitError({ + code, + scope: 'shared', + retryAfter: result.retryAfter, + retryAfterDecimal: result.retryAfterDecimal, + limit: result.limit, + resetTime: result.resetTime, + }); +} + +export async function enqueueCrosspostSync( + workerService: CrosspostWorkerService, + { + channelId, + messageId, + mode, + deleteSource, + }: { + channelId: ChannelID; + messageId: MessageID; + mode: CrosspostSyncMode; + deleteSource?: boolean; + }, +): Promise { + const payload: SyncCrosspostedMessagePayload = { + channelId: channelId.toString(), + messageId: messageId.toString(), + mode, + ...(deleteSource ? {deleteSource: true} : {}), + }; + if (mode === 'purge') { + try { + await getKVClient().setex(crosspostPurgeMarkerKey(messageId), CROSSPOST_PURGE_MARKER_TTL_SECONDS, '1'); + } catch (error) { + Logger.error( + {error, channelId: payload.channelId, messageId: payload.messageId}, + 'Failed to mark crosspost purge', + ); + } + } + let jobKey: string; + let runAt: Date | undefined; + if (mode === 'update') { + const bucket = crosspostSyncBucket(Date.now()); + jobKey = crosspostSyncJobKey(payload.messageId, mode, bucket); + runAt = new Date((bucket + 1) * CROSSPOST_SYNC_COALESCE_MS + 1000); + } else { + jobKey = crosspostSyncJobKey(payload.messageId, mode); + } + try { + await workerService.addJob(CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE, payload, { + jobKey, + runAt, + skipLedger: true, + }); + } catch (error) { + Logger.error( + {error, channelId: payload.channelId, messageId: payload.messageId, mode}, + 'Failed to enqueue crosspost sync', + ); + } +} + +export interface CrosspostSourceRemovalParams { + messages: ReadonlyArray; + mode: CrosspostRemovalMode; + channel?: Pick | null; +} + +function mayHaveCrosspostCopies(message: CrosspostMessageLike, channel: Pick | null): boolean { + if (isCrosspostedMessage(message)) return true; + return channel?.type === ChannelTypes.GUILD_ANNOUNCEMENT && !isCrosspostCopy(message); +} + +export async function enqueueCrosspostSourceRemoval( + workerService: CrosspostWorkerService, + {messages, mode, channel = null}: CrosspostSourceRemovalParams, +): Promise { + for (const message of messages) { + if (!mayHaveCrosspostCopies(message, channel)) continue; + await enqueueCrosspostSync(workerService, {channelId: message.channelId, messageId: message.id, mode}); + } +} + +export async function enqueueCrosspostFamilyPurgeFromCopies( + workerService: CrosspostWorkerService, + {messages}: {messages: ReadonlyArray}, +): Promise { + const seen = new Set(); + for (const message of messages) { + if (!isCrosspostCopy(message)) continue; + const reference = message.reference; + if (!reference?.messageId) continue; + const key = `${reference.channelId}:${reference.messageId}`; + if (seen.has(key)) continue; + seen.add(key); + await enqueueCrosspostSync(workerService, { + channelId: reference.channelId, + messageId: reference.messageId, + mode: 'purge', + deleteSource: true, + }); + } +} + +interface CrosspostPropagationDeps { + rateLimitService: IRateLimitService; + workerService: CrosspostWorkerService; +} + +export class CrosspostPropagation { + constructor(private readonly deps: CrosspostPropagationDeps) {} + + async withPublishedEditBudget( + {fresh, actor}: {fresh: Message; actor: PublishedEditActor}, + write: () => Promise, + ): Promise { + if (!isCrosspostedMessage(fresh) || actor === 'moderator') { + return write(); + } + const config = {identifier: publishedEditRateLimitIdentifier(fresh.id), ...PUBLISHED_MESSAGE_EDIT_RATE_LIMIT}; + const peek = await this.deps.rateLimitService.peekLimit(config); + if (peek.remaining < 1) { + throw createCrosspostRateLimitError( + APIErrorCodes.PUBLISHED_MESSAGE_EDIT_RATE_LIMITED, + withPeekRetryAfter(peek, config), + ); + } + const result = await write(); + await this.deps.rateLimitService.checkLimit(config); + return result; + } + + async enqueueCrosspostFanout({channelId, messageId}: {channelId: ChannelID; messageId: MessageID}): Promise { + const payload: CrosspostMessagePayload = {channelId: channelId.toString(), messageId: messageId.toString()}; + await this.deps.workerService.addJob(CrosspostTaskNames.CROSSPOST_MESSAGE, payload, { + jobKey: crosspostFanoutJobKey(payload.messageId), + skipLedger: true, + }); + } + + async propagateEdit(message: Message): Promise { + if (!isCrosspostedMessage(message)) { + return; + } + await enqueueCrosspostSync(this.deps.workerService, { + channelId: message.channelId, + messageId: message.id, + mode: 'update', + }); + } + + async enqueueCrosspostSourceRemoval(params: CrosspostSourceRemovalParams): Promise { + await enqueueCrosspostSourceRemoval(this.deps.workerService, params); + } +} diff --git a/fluxer_api/src/api/channel/services/message/CrosspostSourceService.ts b/fluxer_api/src/api/channel/services/message/CrosspostSourceService.ts new file mode 100644 index 000000000..3761ba392 --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/CrosspostSourceService.ts @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createGuildID, type GuildID} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import {mapGuildToPartialResponse} from '@app/api/guild/GuildModel'; +import type {IGuildDiscoveryRepository} from '@app/api/guild/repositories/GuildDiscoveryRepository'; +import type {IGuildDataRepository} from '@app/api/guild/repositories/IGuildDataRepository'; +import type {IGatewayService} from '@app/api/infrastructure/IGatewayService'; +import {Logger} from '@app/api/Logger'; +import {MessageFlags, MessageTypes} from '@fluxer/constants/src/ChannelConstants'; +import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; +import {UnknownGuildError} from '@fluxer/errors/src/domains/guild/UnknownGuildError'; +import type {CrosspostSourceResponse} from '@fluxer/schema/src/domains/message/CrosspostSourceSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; + +const CROSSPOST_SOURCE_COUNTS_TTL_SECONDS = 60; + +const CROSSPOST_SOURCE_PUBLIC_FEATURES = new Set([ + GuildFeatures.VERIFIED, + GuildFeatures.PARTNERED, + GuildFeatures.DISCOVERABLE, +]); + +interface CrosspostSourceCounts { + memberCount: number; + presenceCount: number; +} + +export function crosspostSourceCountsCacheKey(guildId: GuildID): string { + return `crosspost-source:counts:${guildId.toString()}`; +} + +export function getCrosspostSourceGuildId(message: MessageResponse): GuildID | null { + const isCopy = (message.flags & MessageFlags.IS_CROSSPOST) !== 0; + const isFollowNotice = message.type === MessageTypes.CHANNEL_FOLLOW_ADD; + if (!isCopy && !isFollowNotice) { + return null; + } + const guildId = message.message_reference?.guild_id; + return guildId ? createGuildID(BigInt(guildId)) : null; +} + +export class CrosspostSourceService { + constructor( + private readonly guildRepository: IGuildDataRepository, + private readonly discoveryRepository: IGuildDiscoveryRepository, + private readonly gatewayService: IGatewayService, + private readonly cacheService: ICacheService, + ) {} + + async getSource(message: MessageResponse): Promise { + const guildId = getCrosspostSourceGuildId(message); + if (guildId === null) { + throw new UnknownMessageError(); + } + const guild = await this.guildRepository.findUnique(guildId); + if (!guild) { + throw new UnknownGuildError(); + } + const partial = mapGuildToPartialResponse(guild); + const isListed = Config.discovery.enabled && guild.features.has(GuildFeatures.DISCOVERABLE); + const [counts, description] = await Promise.all([ + this.getCounts(guildId), + isListed ? this.getListedDescription(guildId) : null, + ]); + return { + guild: { + id: partial.id, + name: partial.name, + icon: partial.icon ?? null, + banner: partial.banner ?? null, + features: partial.features.filter((feature) => CROSSPOST_SOURCE_PUBLIC_FEATURES.has(feature)), + approximate_member_count: counts?.memberCount ?? null, + approximate_presence_count: counts?.presenceCount ?? null, + description, + discoverable: isListed && !guild.features.has(GuildFeatures.INVITES_DISABLED), + }, + }; + } + + private async getListedDescription(guildId: GuildID): Promise { + const row = await this.discoveryRepository.findByGuildId(guildId); + if (row?.status !== DiscoveryApplicationStatus.APPROVED) { + return null; + } + return row.description || null; + } + + private async getCounts(guildId: GuildID): Promise { + try { + return await this.cacheService.getOrSet( + crosspostSourceCountsCacheKey(guildId), + () => this.gatewayService.getGuildCounts(guildId), + CROSSPOST_SOURCE_COUNTS_TTL_SECONDS, + ); + } catch (error) { + Logger.warn({error, guildId: guildId.toString()}, 'Failed to load crosspost source community counts'); + return null; + } + } +} diff --git a/fluxer_api/src/api/channel/services/message/MessageCrosspostService.ts b/fluxer_api/src/api/channel/services/message/MessageCrosspostService.ts new file mode 100644 index 000000000..d5708ebbc --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/MessageCrosspostService.ts @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; +import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; +import type {AuthenticatedChannel} from '@app/api/channel/services/AuthenticatedChannel'; +import {collectEmbedContentHashes} from '@app/api/channel/services/message/CrosspostEmbedObjects'; +import { + type CrosspostPropagation, + createCrosspostRateLimitError, + isCrosspostedMessage, + withPeekRetryAfter, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; +import type {MessageDispatchService} from '@app/api/channel/services/message/MessageDispatchService'; +import {isCrosspostCopy, isOperationDisabled} from '@app/api/channel/services/message/MessageHelpers'; +import {assertMessageWithinHistoryCutoff} from '@app/api/channel/services/message/MessageHistoryCutoff'; +import type {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; +import {contentModerationService} from '@app/api/infrastructure/ContentModerationService'; +import {Logger} from '@app/api/Logger'; +import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; +import type {Message} from '@app/api/models/Message'; +import {assertGuildMemberCanCommunicate} from '@app/api/utils/GuildCommunicationUtils'; +import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError'; +import {CROSSPOST_CHANNEL_RATE_LIMIT} from '@fluxer/constants/src/AnnouncementConstants'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes, MessageFlags, MessageTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures, GuildOperations} from '@fluxer/constants/src/GuildConstants'; +import {AnnouncementChannelRequiredError} from '@fluxer/errors/src/domains/channel/AnnouncementChannelRequiredError'; +import {MessageAlreadyCrosspostedError} from '@fluxer/errors/src/domains/channel/MessageAlreadyCrosspostedError'; +import {MessageNotCrosspostableError} from '@fluxer/errors/src/domains/channel/MessageNotCrosspostableError'; +import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError'; +import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; +import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError'; +import {ServiceUnavailableError} from '@fluxer/errors/src/HttpErrors'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import type {IRateLimitService} from '@pkgs/rate_limit/src/IRateLimitService'; + +interface MessageCrosspostServiceDeps { + channelRepository: IChannelRepositoryAggregate; + channelAuthService: MessageChannelAuthService; + dispatchService: MessageDispatchService; + rateLimitService: IRateLimitService; + messageWriteLock: MessageWriteLock; + crosspostPropagation: CrosspostPropagation; +} + +interface CrosspostMessageResult { + message: Message; + authChannel: AuthenticatedChannel; +} + +export function crosspostChannelRateLimitIdentifier(channelId: ChannelID): string { + return `crosspost:channel:${channelId}`; +} + +export class MessageCrosspostService { + constructor(private readonly deps: MessageCrosspostServiceDeps) {} + + async crosspostMessage({ + userId, + channelId, + messageId, + requestCache, + }: { + userId: UserID; + channelId: ChannelID; + messageId: MessageID; + requestCache: RequestCache; + }): Promise { + const authChannel = await this.deps.channelAuthService.getChannelAuthenticated({userId, channelId}); + const {channel, guild, member, hasPermission, checkPermission} = authChannel; + if (channel.type !== ChannelTypes.GUILD_ANNOUNCEMENT) { + throw new AnnouncementChannelRequiredError(); + } + if (!guild) { + throw new UnknownChannelError(); + } + if ( + isOperationDisabled(guild, GuildOperations.SEND_MESSAGE) || + guild.features.includes(GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED) + ) { + throw new FeatureTemporarilyDisabledError(); + } + const message = await this.deps.channelRepository.messages.getMessage(channelId, messageId); + if (!message) { + throw new UnknownMessageError(); + } + await checkPermission(Permissions.SEND_MESSAGES); + if (message.authorId === userId) { + assertGuildMemberCanCommunicate(member); + } else { + await checkPermission(Permissions.MANAGE_MESSAGES); + if (!(await hasPermission(Permissions.READ_MESSAGE_HISTORY))) { + assertMessageWithinHistoryCutoff({message, guild}); + } + } + this.assertCrosspostable(message); + this.assertNotBlocked(message, guild); + await this.assertBudgetAvailable(channelId); + const published = await this.deps.messageWriteLock.withFreshMessage(channelId, messageId, async (fresh) => { + if (!fresh) { + throw new UnknownMessageError(); + } + this.assertCrosspostable(fresh); + await this.assertBudgetAvailable(channelId); + const updated = await this.deps.channelRepository.messages.upsertMessage( + {...fresh.toRow(), flags: fresh.flags | MessageFlags.CROSSPOSTED}, + fresh.toRow(), + ); + await this.deps.channelRepository.crossposts.addSource({sourceChannelId: channelId, sourceMessageId: messageId}); + return updated; + }); + Logger.info( + { + actorId: userId.toString(), + guildId: guild.id, + channelId: channelId.toString(), + messageId: messageId.toString(), + }, + 'message published', + ); + await this.deps.dispatchService.dispatchMessageUpdate({channel, message: published, requestCache}); + try { + await this.deps.crosspostPropagation.enqueueCrosspostFanout({channelId, messageId}); + } catch (error) { + Logger.error( + {error, channelId: channelId.toString(), messageId: messageId.toString()}, + 'Failed to enqueue crosspost fan-out', + ); + const reverted = await this.revertPublish({channelId, messageId}); + if (reverted) { + await this.deps.dispatchService.dispatchMessageUpdate({channel, message: reverted, requestCache}); + } + if (error instanceof WorkerQueueOverflowError) { + throw new ServiceUnavailableError(); + } + throw error; + } + await this.deps.rateLimitService.checkLimit(this.channelBudgetConfig(channelId)); + return {message: published, authChannel}; + } + + private assertCrosspostable(message: Message): void { + if (isCrosspostCopy(message) || message.type !== MessageTypes.DEFAULT || message.messageSnapshots.length > 0) { + throw new MessageNotCrosspostableError(); + } + if (isCrosspostedMessage(message)) { + throw new MessageAlreadyCrosspostedError(); + } + } + + private assertNotBlocked(message: Message, guild: GuildResponse): void { + const context = { + userId: message.authorId, + guildId: BigInt(guild.id), + channelId: message.channelId, + messageId: message.id, + }; + const textContext = {...context, surface: 'message_content' as const}; + contentModerationService.scanText(message.content, textContext); + for (const embed of message.embeds) { + if (embed.type !== 'rich') continue; + contentModerationService.scanText(embed.title, textContext); + contentModerationService.scanText(embed.description, textContext); + for (const field of embed.fields) { + contentModerationService.scanText(field.name, textContext); + contentModerationService.scanText(field.value, textContext); + } + contentModerationService.scanText(embed.footer?.text, textContext); + contentModerationService.scanText(embed.author?.name, textContext); + } + for (const attachment of message.attachments) { + if (attachment.contentHash) { + contentModerationService.scanSha256(attachment.contentHash, {...context, surface: 'message_attachment'}); + } + } + for (const contentHash of collectEmbedContentHashes(message)) { + contentModerationService.scanSha256(contentHash, {...context, surface: 'message_attachment'}); + } + } + + private channelBudgetConfig(channelId: ChannelID) { + return {identifier: crosspostChannelRateLimitIdentifier(channelId), ...CROSSPOST_CHANNEL_RATE_LIMIT}; + } + + private async assertBudgetAvailable(channelId: ChannelID): Promise { + const config = this.channelBudgetConfig(channelId); + const peek = await this.deps.rateLimitService.peekLimit(config); + if (peek.remaining < 1) { + throw createCrosspostRateLimitError( + APIErrorCodes.MESSAGE_CROSSPOST_RATE_LIMITED, + withPeekRetryAfter(peek, config), + ); + } + } + + private async revertPublish({ + channelId, + messageId, + }: { + channelId: ChannelID; + messageId: MessageID; + }): Promise { + return this.deps.messageWriteLock.withFreshMessage(channelId, messageId, async (fresh) => { + await this.deps.channelRepository.crossposts.deleteSource({ + sourceChannelId: channelId, + sourceMessageId: messageId, + }); + if (!fresh || !isCrosspostedMessage(fresh)) { + return null; + } + return this.deps.channelRepository.messages.upsertMessage( + {...fresh.toRow(), flags: fresh.flags & ~MessageFlags.CROSSPOSTED}, + fresh.toRow(), + ); + }); + } +} diff --git a/fluxer_api/src/api/channel/services/message/MessageDeleteService.ts b/fluxer_api/src/api/channel/services/message/MessageDeleteService.ts index b3c677946..f985a4add 100644 --- a/fluxer_api/src/api/channel/services/message/MessageDeleteService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageDeleteService.ts @@ -3,6 +3,7 @@ import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createMessageID, createUserID} from '@app/api/BrandedTypes'; import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; +import type {CrosspostPropagation} from '@app/api/channel/services/message/CrosspostPropagation'; import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; import type {MessageDispatchService} from '@app/api/channel/services/message/MessageDispatchService'; import {isOperationDisabled, purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; @@ -39,6 +40,7 @@ interface MessageDeleteServiceDeps { searchService: MessageSearchService; gatewayService: IGatewayService; guildAuditLogService: GuildAuditLogService; + crosspostPropagation: CrosspostPropagation; } export class MessageDeleteService { @@ -84,6 +86,11 @@ export class MessageDeleteService { message.pinnedTimestamp || undefined, ); await this.deps.dispatchService.dispatchMessageDelete({channel, messageId, message}); + await this.deps.crosspostPropagation.enqueueCrosspostSourceRemoval({ + messages: [message], + mode: 'source_deleted', + channel, + }); if (message.pinnedTimestamp) { await this.deps.dispatchService.dispatchEvent({ channel, @@ -134,6 +141,11 @@ export class MessageDeleteService { message.pinnedTimestamp || undefined, ); await this.deps.dispatchService.dispatchMessageDelete({channel, messageId, message}); + await this.deps.crosspostPropagation.enqueueCrosspostSourceRemoval({ + messages: [message], + mode: 'source_deleted', + channel, + }); if (message.pinnedTimestamp) { await this.deps.dispatchService.dispatchEvent({ channel, @@ -182,6 +194,11 @@ export class MessageDeleteService { ); await this.deps.channelRepository.messages.bulkDeleteMessages(channelId, messageIds); await this.deps.dispatchService.dispatchMessageDeleteBulk({channel, messageIds}); + await this.deps.crosspostPropagation.enqueueCrosspostSourceRemoval({ + messages: existingMessages, + mode: 'source_deleted', + channel, + }); if (channel.guildId && existingMessages.length > 0) { await this.guildAuditLogService .createBuilder(channel.guildId, userId) @@ -260,6 +277,11 @@ export class MessageDeleteService { ); await this.deps.channelRepository.messages.bulkDeleteMessages(channel.id, messageIds); await this.deps.dispatchService.dispatchMessageDeleteBulk({channel, messageIds}); + await this.deps.crosspostPropagation.enqueueCrosspostSourceRemoval({ + messages: userMessages, + mode: 'source_deleted', + channel, + }); await this.deps.searchService.deleteMessagesIndex(messageIds); } if (inWindow.length < messages.length || messages.length < batchSize) break; diff --git a/fluxer_api/src/api/channel/services/message/MessageEditService.ts b/fluxer_api/src/api/channel/services/message/MessageEditService.ts index 0087f37cb..3a5a07585 100644 --- a/fluxer_api/src/api/channel/services/message/MessageEditService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageEditService.ts @@ -4,6 +4,7 @@ import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; import type {MessageUpdateRequest} from '@app/api/channel/MessageTypes'; import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; import type {AuthenticatedChannel} from '@app/api/channel/services/AuthenticatedChannel'; +import type {CrosspostPropagation} from '@app/api/channel/services/message/CrosspostPropagation'; import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; import type {MessageDispatchService} from '@app/api/channel/services/message/MessageDispatchService'; import type {MessageEmbedAttachmentResolver} from '@app/api/channel/services/message/MessageEmbedAttachmentResolver'; @@ -13,25 +14,19 @@ import type {MessagePersistenceService} from '@app/api/channel/services/message/ import type {MessageProcessingService} from '@app/api/channel/services/message/MessageProcessingService'; import type {MessageSearchService} from '@app/api/channel/services/message/MessageSearchService'; import type {MessageValidationService} from '@app/api/channel/services/message/MessageValidationService'; +import type {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; import {Logger} from '@app/api/Logger'; import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; import type {Message} from '@app/api/models/Message'; import type {IUserRepository} from '@app/api/user/IUserRepository'; import {assertGuildMemberCanCommunicate} from '@app/api/utils/GuildCommunicationUtils'; -import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; import {Permissions} from '@fluxer/constants/src/ChannelConstants'; import {GuildOperations} from '@fluxer/constants/src/GuildConstants'; import {UserFlags} from '@fluxer/constants/src/UserConstants'; import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError'; import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError'; -import {ThrottledError} from '@fluxer/errors/src/domains/core/ThrottledError'; import type {AllowedMentionsRequest} from '@fluxer/schema/src/domains/message/SharedMessageSchemas'; -import type {ICacheService} from '@pkgs/cache/src/ICacheService'; - -const MESSAGE_LOCK_TTL_SECONDS = 5; -const MESSAGE_LOCK_ACQUIRE_ATTEMPTS = 6; -const MESSAGE_LOCK_RETRY_DELAY_MS = 50; interface EditMessageResult { message: Message; @@ -41,7 +36,6 @@ interface EditMessageResult { interface MessageEditServiceDeps { channelRepository: IChannelRepositoryAggregate; userRepository: IUserRepository; - cacheService: ICacheService; validationService: MessageValidationService; persistenceService: MessagePersistenceService; channelAuthService: MessageChannelAuthService; @@ -50,6 +44,8 @@ interface MessageEditServiceDeps { searchService: MessageSearchService; embedAttachmentResolver: MessageEmbedAttachmentResolver; mentionService: MessageMentionService; + messageWriteLock: MessageWriteLock; + crosspostPropagation: CrosspostPropagation; } export class MessageEditService { @@ -113,7 +109,7 @@ export class MessageEditService { attachments: data.attachments, existingAttachments: message.attachments.map((att) => ({filename: att.filename})), }); - const referencedMessage = message.reference + const referencedMessage = message.reference?.messageId ? await this.deps.channelRepository.messages.getMessage(channelId, message.reference.messageId) : null; const effectiveAllowedMentions = this.getEffectiveAllowedMentionsForEdit({message, referencedMessage, data}); @@ -138,9 +134,10 @@ export class MessageEditService { }); } if (message.authorId !== userId) { - const editedMessage = await this.withMessageLock(channelId, messageId, () => - this.deps.processingService.handleNonAuthorEdit({ - message, + const editedMessage = await this.deps.messageWriteLock.withFreshMessage(channelId, messageId, (fresh) => { + if (!fresh) throw new UnknownMessageError(); + return this.deps.processingService.handleNonAuthorEdit({ + message: fresh, messageId, data, guild, @@ -149,26 +146,30 @@ export class MessageEditService { requestCache, persistenceService: this.deps.persistenceService, dispatchService: this.deps.dispatchService, - }), - ); + }); + }); + await this.deps.crosspostPropagation.propagateEdit(editedMessage); return {message: editedMessage, authChannel}; } const isBugHunterBot = !!user?.isBot && (user.flags & UserFlags.BUG_HUNTER) !== 0n; - const updateResult = await this.withMessageLock(channelId, messageId, () => - this.deps.persistenceService.updateMessage({ - message, - messageId, - data, - channel, - guild, - member, - attachmentUploadUserId: userId, - allowEmbeds: canEmbedLinks, - isBot: user?.isBot, - isBugHunterBot, - locale: user?.locale, - }), - ); + const updateResult = await this.deps.messageWriteLock.withFreshMessage(channelId, messageId, async (fresh) => { + if (!fresh) throw new UnknownMessageError(); + return this.deps.crosspostPropagation.withPublishedEditBudget({fresh, actor: 'author'}, () => + this.deps.persistenceService.updateMessage({ + message: fresh, + messageId, + data, + channel, + guild, + member, + attachmentUploadUserId: userId, + allowEmbeds: canEmbedLinks, + isBot: user?.isBot, + isBugHunterBot, + locale: user?.locale, + }), + ); + }); let updatedMessage = updateResult.message; if (data.content !== undefined || data.allowed_mentions !== undefined || data.embeds !== undefined) { const mentionResult = await this.deps.processingService.handleMentions({ @@ -186,6 +187,7 @@ export class MessageEditService { } } await this.deps.dispatchService.dispatchMessageUpdate({channel, message: updatedMessage, requestCache}); + await this.deps.crosspostPropagation.propagateEdit(updatedMessage); void updateResult.enqueueDeferredEmbeds().catch((error) => { Logger.warn({error, messageId: messageId.toString()}, 'Failed to enqueue deferred embed extraction after edit'); }); @@ -218,26 +220,4 @@ export class MessageEditService { } return {replied_user: false}; } - - private async withMessageLock(channelId: ChannelID, messageId: MessageID, fn: () => Promise): Promise { - const lockKey = `message:${channelId}:${messageId}:write`; - let lockToken: string | null = null; - for (let attempt = 0; attempt < MESSAGE_LOCK_ACQUIRE_ATTEMPTS; attempt++) { - lockToken = await this.deps.cacheService.acquireLock(lockKey, MESSAGE_LOCK_TTL_SECONDS); - if (lockToken) break; - await new Promise((resolve) => setTimeout(resolve, MESSAGE_LOCK_RETRY_DELAY_MS * (attempt + 1))); - } - if (!lockToken) { - throw new ThrottledError({ - code: APIErrorCodes.RESOURCE_LOCKED, - retryAfterSeconds: 1, - data: {retry_after: 1}, - }); - } - try { - return await fn(); - } finally { - await this.deps.cacheService.releaseLock(lockKey, lockToken).catch(() => {}); - } - } } diff --git a/fluxer_api/src/api/channel/services/message/MessageEmbedAttachmentResolver.ts b/fluxer_api/src/api/channel/services/message/MessageEmbedAttachmentResolver.ts index 7e0efe454..4f9b91f39 100644 --- a/fluxer_api/src/api/channel/services/message/MessageEmbedAttachmentResolver.ts +++ b/fluxer_api/src/api/channel/services/message/MessageEmbedAttachmentResolver.ts @@ -3,7 +3,11 @@ import type {AttachmentID, ChannelID} from '@app/api/BrandedTypes'; import type {AttachmentRequestData} from '@app/api/channel/AttachmentDTOs'; import type {RichEmbedMediaWithMetadata} from '@app/api/channel/EmbedTypes'; -import {getContentType, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; +import { + EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, + getContentType, + makeAttachmentCdnUrl, +} from '@app/api/channel/services/message/MessageHelpers'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError'; import type {RichEmbedRequest} from '@fluxer/schema/src/domains/message/MessageRequestSchemas'; @@ -158,7 +162,7 @@ export class MessageEmbedAttachmentResolver { content_type: metadata.content_type, content_hash: metadata.content_hash, placeholder: metadata.placeholder, - flags: metadata.flags, + flags: metadata.flags | EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, duration: metadata.duration, nsfw: metadata.nsfw, }, @@ -176,7 +180,7 @@ export class MessageEmbedAttachmentResolver { content_type: metadata.content_type, content_hash: metadata.content_hash, placeholder: metadata.placeholder, - flags: metadata.flags, + flags: metadata.flags | EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, duration: metadata.duration, nsfw: metadata.nsfw, }, diff --git a/fluxer_api/src/api/channel/services/message/MessageHelpers.test.ts b/fluxer_api/src/api/channel/services/message/MessageHelpers.test.ts index 74113ca10..b4ef2a17b 100644 --- a/fluxer_api/src/api/channel/services/message/MessageHelpers.test.ts +++ b/fluxer_api/src/api/channel/services/message/MessageHelpers.test.ts @@ -2,7 +2,10 @@ import {createAttachmentID, createChannelID, createMessageID, createUserID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; -import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; +import { + EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, + purgeMessageAttachments, +} from '@app/api/channel/services/message/MessageHelpers'; import type {MessageEmbed} from '@app/api/database/types/MessageTypes'; import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue'; import type {IStorageService} from '@app/api/infrastructure/IStorageService'; @@ -14,7 +17,7 @@ const CHANNEL_ID = createChannelID(10n); const ATTACHMENT_KEY = 'attachments/10/200/ação.png'; const OTHER_MESSAGE_ATTACHMENT_KEY = 'attachments/11/300/photo.jpg'; -function imageEmbed(key: string): MessageEmbed { +function imageEmbed(key: string, flags = 0): MessageEmbed { return { type: 'image', title: null, @@ -33,7 +36,7 @@ function imageEmbed(key: string): MessageEmbed { content_type: 'image/png', content_hash: null, placeholder: null, - flags: 0, + flags, duration: null, }, video: null, @@ -43,7 +46,9 @@ function imageEmbed(key: string): MessageEmbed { }; } -function makeMessageWithMedia(): Message { +function makeMessageWithMedia( + embeds: Array = [imageEmbed(ATTACHMENT_KEY), imageEmbed(OTHER_MESSAGE_ATTACHMENT_KEY)], +): Message { return new Message({ channel_id: CHANNEL_ID, bucket: 0, @@ -79,7 +84,7 @@ function makeMessageWithMedia(): Message { waveform: null, }, ], - embeds: [imageEmbed(ATTACHMENT_KEY), imageEmbed(OTHER_MESSAGE_ATTACHMENT_KEY)], + embeds, sticker_items: null, message_reference: null, message_snapshots: null, @@ -109,4 +114,27 @@ describe('purgeMessageAttachments', () => { expect(deletedObjects).toEqual([`${Config.s3.buckets.cdn}/${ATTACHMENT_KEY}`]); expect(queuedUrls).toEqual([`${Config.endpoints.media}/${ATTACHMENT_KEY}`]); }); + + it('purges embed files the message owns and leaves marked files under other channels alone', async () => { + const deletedObjects: Array = []; + const storageService = { + deleteObject: async (_bucket: string, key: string) => { + deletedObjects.push(key); + }, + } as unknown as IStorageService; + const purgeQueue: IPurgeQueue = {addUrls: async () => {}}; + const ownedEmbedKey = 'attachments/10/201/embed.png'; + + await purgeMessageAttachments( + makeMessageWithMedia([ + imageEmbed(ownedEmbedKey, EMBED_MEDIA_OWNED_ATTACHMENT_FLAG), + imageEmbed(OTHER_MESSAGE_ATTACHMENT_KEY, EMBED_MEDIA_OWNED_ATTACHMENT_FLAG), + imageEmbed('attachments/10/202/unmarked.png'), + ]), + storageService, + purgeQueue, + ); + + expect(deletedObjects).toEqual([ATTACHMENT_KEY, ownedEmbedKey]); + }); }); diff --git a/fluxer_api/src/api/channel/services/message/MessageHelpers.ts b/fluxer_api/src/api/channel/services/message/MessageHelpers.ts index fa4baa086..0c0042a8d 100644 --- a/fluxer_api/src/api/channel/services/message/MessageHelpers.ts +++ b/fluxer_api/src/api/channel/services/message/MessageHelpers.ts @@ -3,9 +3,12 @@ import type {AttachmentID, ChannelID, UserID} from '@app/api/BrandedTypes'; import {createAttachmentID, userIdToChannelId} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; +import {forEachEmbedMedia} from '@app/api/channel/services/message/CrosspostEmbedObjects'; import type { MessageSnapshot as CassandraMessageSnapshot, MessageAttachment, + MessageEmbed, + MessageEmbedMedia, } from '@app/api/database/types/MessageTypes'; import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue'; import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService'; @@ -15,14 +18,14 @@ import type {LimitConfigService} from '@app/api/limits/LimitConfigService'; import {resolveLimitSafe} from '@app/api/limits/LimitConfigUtils'; import {createLimitMatchContext} from '@app/api/limits/LimitMatchContextBuilder'; import {Attachment} from '@app/api/models/Attachment'; -import type {Embed} from '@app/api/models/Embed'; import type {Message} from '@app/api/models/Message'; import {MessageSnapshot as MessageSnapshotModel} from '@app/api/models/MessageSnapshot'; import type {User} from '@app/api/models/User'; import {S3ServiceException} from '@aws-sdk/client-s3'; -import {MessageFlags} from '@fluxer/constants/src/ChannelConstants'; +import {MessageFlags, SENDABLE_MESSAGE_FLAGS} from '@fluxer/constants/src/ChannelConstants'; import {ATTACHMENT_MAX_SIZE_NON_PREMIUM} from '@fluxer/constants/src/LimitConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeTooLargeError'; import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError'; import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; @@ -146,6 +149,17 @@ export function makeAttachmentCdnUrl( return `${Config.endpoints.media}/${makeAttachmentCdnKey(channelId, attachmentId, filename)}`; } +export function isCrosspostCopy(message: Pick): boolean { + return (message.flags & MessageFlags.IS_CROSSPOST) !== 0; +} + +export function attachmentStorageChannelId(message: Pick): ChannelID { + if (isCrosspostCopy(message) && message.reference) { + return message.reference.channelId; + } + return message.channelId; +} + function isMissingStorageObjectError(error: unknown): boolean { return ( (error instanceof S3ServiceException && (error.name === 'NoSuchKey' || error.name === 'NotFound')) || @@ -153,6 +167,30 @@ function isMissingStorageObjectError(error: unknown): boolean { ); } +async function copyCdnObject( + storageService: IStorageService, + sourceKey: string, + destinationKey: string, + contentType: string | null | undefined, +): Promise { + try { + await storageService.copyObject({ + sourceBucket: Config.s3.buckets.cdn, + sourceKey, + destinationBucket: Config.s3.buckets.cdn, + destinationKey, + newContentType: contentType ?? undefined, + }); + return true; + } catch (error) { + if (isMissingStorageObjectError(error)) { + Logger.warn({error, sourceKey, destinationKey}, 'Skipping missing attachment while cloning message'); + return false; + } + throw error; + } +} + async function cloneAttachments( attachments: Array, sourceChannelId: ChannelID, @@ -163,33 +201,14 @@ async function cloneAttachments( const clonedAttachments: Array = []; for (const attachment of attachments) { const newAttachmentId = createAttachmentID(await snowflakeService.generate()); - const sourceKey = makeAttachmentCdnKey(sourceChannelId, attachment.id, attachment.filename); - const destinationKey = makeAttachmentCdnKey(destinationChannelId, newAttachmentId, attachment.filename); - try { - await storageService.copyObject({ - sourceBucket: Config.s3.buckets.cdn, - sourceKey, - destinationBucket: Config.s3.buckets.cdn, - destinationKey, - newContentType: attachment.contentType, - }); - } catch (error) { - if (isMissingStorageObjectError(error)) { - Logger.warn( - { - error, - sourceChannelId, - destinationChannelId, - sourceKey, - destinationKey, - attachmentId: attachment.id, - filename: attachment.filename, - }, - 'Skipping missing attachment while cloning forwarded message', - ); - continue; - } - throw error; + const copied = await copyCdnObject( + storageService, + makeAttachmentCdnKey(sourceChannelId, attachment.id, attachment.filename), + makeAttachmentCdnKey(destinationChannelId, newAttachmentId, attachment.filename), + attachment.contentType, + ); + if (!copied) { + continue; } clonedAttachments.push({ attachment_id: newAttachmentId, @@ -220,6 +239,9 @@ export async function createMessageSnapshotsForForward( limitConfigService: LimitConfigService, selection?: ForwardMediaSelection, ): Promise> { + if ((referencedMessage.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0) { + throw new UnknownMessageError(); + } const isMediaOnlyForward = hasForwardMediaSelection(selection); if (referencedMessage.messageSnapshots && referencedMessage.messageSnapshots.length > 0) { const snapshot = referencedMessage.messageSnapshots[0]; @@ -239,7 +261,14 @@ export async function createMessageSnapshotsForForward( ); const clonedAttachments = await cloneAttachments( attachmentsForClone, - referencedMessage.channelId, + attachmentStorageChannelId(referencedMessage), + destinationChannelId, + storageService, + snowflakeService, + ); + await cloneOwnedEmbedAttachments( + snapshotEmbeds, + attachmentStorageChannelId(referencedMessage), destinationChannelId, storageService, snowflakeService, @@ -255,7 +284,7 @@ export async function createMessageSnapshotsForForward( embeds: snapshotEmbeds.length > 0 ? snapshotEmbeds : null, sticker_items: isMediaOnlyForward ? null : snapshot.stickers.map((sticker) => sticker.toMessageStickerItem()), type: snapshot.type, - flags: snapshot.flags, + flags: snapshot.flags & SENDABLE_MESSAGE_FLAGS, }; return [new MessageSnapshotModel(snapshotData)]; } @@ -263,7 +292,7 @@ export async function createMessageSnapshotsForForward( validateTotalAttachmentSize(selectedAttachments, user, limitConfigService); const clonedAttachments = await cloneAttachments( selectedAttachments, - referencedMessage.channelId, + attachmentStorageChannelId(referencedMessage), destinationChannelId, storageService, snowflakeService, @@ -277,6 +306,13 @@ export async function createMessageSnapshotsForForward( if (isMediaOnlyForward && selectedAttachments.length === 0 && referencedMessageEmbeds.length === 0) { throw InputValidationError.fromCode('message_reference', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE); } + await cloneOwnedEmbedAttachments( + referencedMessageEmbeds, + attachmentStorageChannelId(referencedMessage), + destinationChannelId, + storageService, + snowflakeService, + ); const snapshotData: CassandraMessageSnapshot = { content: isMediaOnlyForward ? null : referencedMessage.content, timestamp: snowflakeToDate(referencedMessage.id), @@ -303,36 +339,87 @@ export async function createMessageSnapshotsForForward( ? referencedMessage.stickers.map((s) => s.toMessageStickerItem()) : null, type: referencedMessage.type, - flags: referencedMessage.flags, + flags: referencedMessage.flags & SENDABLE_MESSAGE_FLAGS, }; return [new MessageSnapshotModel(snapshotData)]; } -function collectEmbedReferencedAttachmentCdnKeys(message: Message, ownKeys: ReadonlySet): Array { +export const EMBED_MEDIA_OWNED_ATTACHMENT_FLAG = 1 << 30; + +function isOwnedEmbedAttachment(media: Pick): boolean { + return (media.flags & EMBED_MEDIA_OWNED_ATTACHMENT_FLAG) !== 0; +} + +export function keepOwnedEmbedAttachments(previous: Message, embeds: Array | null): void { + const ownedUrls = new Set(); + forEachEmbedMedia( + previous.embeds.map((embed) => embed.toMessageEmbed()), + (media) => { + if (media.url && isOwnedEmbedAttachment(media)) ownedUrls.add(media.url); + }, + ); + if (ownedUrls.size === 0 || !embeds) return; + forEachEmbedMedia(embeds, (media) => { + if (media.url && ownedUrls.has(media.url)) { + media.flags |= EMBED_MEDIA_OWNED_ATTACHMENT_FLAG; + } + }); +} + +async function cloneOwnedEmbedAttachments( + embeds: Array, + sourceChannelId: ChannelID, + destinationChannelId: ChannelID, + storageService: IStorageService, + snowflakeService: ISnowflakeService, +): Promise { const mediaPrefix = `${Config.endpoints.media}/`; - const keys = new Set(); - const consider = (url: string | null | undefined): void => { - if (!url?.startsWith(mediaPrefix)) { - return; + const sourcePrefix = `${mediaPrefix}attachments/${sourceChannelId}/`; + const owned: Array = []; + forEachEmbedMedia(embeds, (media) => { + if (isOwnedEmbedAttachment(media)) owned.push(media); + }); + const clonedUrls = new Map(); + for (const media of owned) { + media.flags &= ~EMBED_MEDIA_OWNED_ATTACHMENT_FLAG; + const url = media.url; + if (!url?.startsWith(sourcePrefix)) continue; + if (!clonedUrls.has(url)) { + const filename = url.slice(sourcePrefix.length).split('/').slice(1).join('/'); + const clonedId = createAttachmentID(await snowflakeService.generate()); + const copied = await copyCdnObject( + storageService, + url.slice(mediaPrefix.length), + makeAttachmentCdnKey(destinationChannelId, clonedId, filename), + media.content_type, + ); + clonedUrls.set(url, copied ? makeAttachmentCdnUrl(destinationChannelId, clonedId, filename) : null); } - const key = url.slice(mediaPrefix.length); - if (ownKeys.has(key)) { - keys.add(key); + const clonedUrl = clonedUrls.get(url); + if (clonedUrl) { + media.url = clonedUrl; + media.flags |= EMBED_MEDIA_OWNED_ATTACHMENT_FLAG; } - }; - const scanEmbeds = (embeds: Array): void => { - for (const embed of embeds) { - consider(embed.image?.url); - consider(embed.thumbnail?.url); - consider(embed.video?.url); - consider(embed.audio?.url); - } - }; - scanEmbeds(message.embeds); - for (const snapshot of message.messageSnapshots) { - scanEmbeds(snapshot.embeds); } - return [...keys]; +} + +export function collectOwnedEmbedAttachments(message: Message): Array<{key: string; media: MessageEmbedMedia}> { + const mediaPrefix = `${Config.endpoints.media}/`; + const ownPrefix = `${mediaPrefix}attachments/${attachmentStorageChannelId(message)}/`; + const seen = new Set(); + const owned: Array<{key: string; media: MessageEmbedMedia}> = []; + const embeds = [...message.embeds, ...message.messageSnapshots.flatMap((snapshot) => snapshot.embeds)]; + forEachEmbedMedia( + embeds.map((embed) => embed.toMessageEmbed()), + (media) => { + if (!media.url?.startsWith(ownPrefix) || !isOwnedEmbedAttachment(media)) return; + const key = media.url.slice(mediaPrefix.length); + if (seen.has(key)) return; + seen.add(key); + owned.push({key, media}); + }, + ); + return owned; } export async function purgeMessageAttachments( @@ -340,13 +427,11 @@ export async function purgeMessageAttachments( storageService: IStorageService, purgeQueue: IPurgeQueue, ): Promise { + if (isCrosspostCopy(message)) { + return; + } const cdnKeys = new Set(); const cdnUrls: Array = []; - const ownedCdnKeys = new Set( - collectMessageAttachments(message).map((attachment) => - makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename), - ), - ); for (const attachment of collectMessageAttachments(message)) { const cdnKey = makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename); if (cdnKeys.has(cdnKey)) { @@ -355,7 +440,7 @@ export async function purgeMessageAttachments( cdnKeys.add(cdnKey); cdnUrls.push(makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename)); } - for (const embedKey of collectEmbedReferencedAttachmentCdnKeys(message, ownedCdnKeys)) { + for (const {key: embedKey} of collectOwnedEmbedAttachments(message)) { if (cdnKeys.has(embedKey)) { continue; } diff --git a/fluxer_api/src/api/channel/services/message/MessageHistoryCutoff.ts b/fluxer_api/src/api/channel/services/message/MessageHistoryCutoff.ts new file mode 100644 index 000000000..d8442eb0d --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/MessageHistoryCutoff.ts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {Message} from '@app/api/models/Message'; +import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake'; + +export function assertMessageWithinHistoryCutoff(params: {message: Message | null; guild: GuildResponse}): void { + const {message, guild} = params; + if (!message) { + throw new UnknownMessageError(); + } + const cutoff = guild.message_history_cutoff; + if (!cutoff) { + throw new UnknownMessageError(); + } + const messageTimestamp = snowflakeToDate(message.id).getTime(); + const cutoffTimestamp = new Date(cutoff).getTime(); + if (messageTimestamp < cutoffTimestamp) { + throw new UnknownMessageError(); + } +} diff --git a/fluxer_api/src/api/channel/services/message/MessagePersistenceService.ts b/fluxer_api/src/api/channel/services/message/MessagePersistenceService.ts index affe1b0e0..5f2883c3d 100644 --- a/fluxer_api/src/api/channel/services/message/MessagePersistenceService.ts +++ b/fluxer_api/src/api/channel/services/message/MessagePersistenceService.ts @@ -15,6 +15,8 @@ import {MessageEmbedAttachmentResolver} from '@app/api/channel/services/message/ import { assertAttachmentFileSizesWithinLimit, collectMessageAttachments, + isCrosspostCopy, + keepOwnedEmbedAttachments, } from '@app/api/channel/services/message/MessageHelpers'; import {MessageStickerService} from '@app/api/channel/services/message/MessageStickerService'; import {getContentMessage} from '@app/api/content_i18n/ContentI18n'; @@ -111,6 +113,9 @@ interface CreateMessageParams { }; allowEmbeds?: boolean; dmNsfwContext?: DmNsfwContext; + processedEmbeds?: Array; + processedStickerItems?: Array; + skipDeferredEmbeds?: boolean; } export class MessagePersistenceService { @@ -190,8 +195,12 @@ export class MessagePersistenceService { const allowEmbeds = params.allowEmbeds ?? true; let initialEmbeds: Array | null = null; let hasUncachedUrls = false; - const referencedFilenames = this.embedAttachmentResolver.collectReferencedAttachmentFilenames(params.embeds); - if (allowEmbeds) { + const referencedFilenames = params.processedEmbeds + ? new Set() + : this.embedAttachmentResolver.collectReferencedAttachmentFilenames(params.embeds); + if (params.processedEmbeds) { + initialEmbeds = params.processedEmbeds.length > 0 ? params.processedEmbeds : null; + } else if (allowEmbeds) { const resolvedEmbeds = this.embedAttachmentResolver.resolveEmbedAttachmentUrls({ embeds: params.embeds, attachments: processedAttachments.map(mapAttachmentForEmbedResolution), @@ -299,6 +308,9 @@ export class MessagePersistenceService { params: CreateMessageParams, authorId: UserID | null, ): Promise> { + if (params.processedStickerItems) { + return params.processedStickerItems; + } if (!params.stickerIds || params.stickerIds.length === 0) { return []; } @@ -320,7 +332,7 @@ export class MessagePersistenceService { }): Promise<() => Promise> { const {message, params, authorId, allowEmbeds, hasUncachedUrls, isNSFWAllowed} = context; const operations: Array> = []; - const trackedAttachments = collectMessageAttachments(message); + const trackedAttachments = isCrosspostCopy(message) ? [] : collectMessageAttachments(message); if (trackedAttachments.length > 0) { const uploadedAt = snowflakeToDate(params.messageId); const decayPayloads = trackedAttachments.map((att) => ({ @@ -334,7 +346,7 @@ export class MessagePersistenceService { operations.push(this.attachmentDecayService.upsertMany(decayPayloads)); } let enqueueDeferredEmbeds: () => Promise = () => Promise.resolve(); - if (allowEmbeds && hasUncachedUrls) { + if (allowEmbeds && hasUncachedUrls && !params.skipDeferredEmbeds) { enqueueDeferredEmbeds = () => this.embedService.enqueueUrlEmbedExtraction( params.channelId, @@ -524,6 +536,7 @@ export class MessagePersistenceService { isBugHunterBot: params.isBugHunterBot, }); if (embedsExplicitlyProvided) { + keepOwnedEmbedAttachments(message, initialEmbeds); updatedRowData.embeds = initialEmbeds; } else { const preservedEmbeds = message.embeds diff --git a/fluxer_api/src/api/channel/services/message/MessageProcessingService.ts b/fluxer_api/src/api/channel/services/message/MessageProcessingService.ts index 02a9ab0b8..cb55e4310 100644 --- a/fluxer_api/src/api/channel/services/message/MessageProcessingService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageProcessingService.ts @@ -18,7 +18,7 @@ import type {Message} from '@app/api/models/Message'; import type {User} from '@app/api/models/User'; import type {ReadStateService} from '@app/api/read_state/ReadStateService'; import type {IUserRepository} from '@app/api/user/IUserRepository'; -import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; +import {ChannelTypes, MessageFlags} from '@fluxer/constants/src/ChannelConstants'; import {CannotEditOtherUserMessageError} from '@fluxer/errors/src/domains/channel/CannotEditOtherUserMessageError'; import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; import type {AllowedMentionsRequest} from '@fluxer/schema/src/domains/message/SharedMessageSchemas'; @@ -204,6 +204,9 @@ export class MessageProcessingService { } async repairMentionsOnRead(message: Message, sourceChannel?: Channel): Promise { + if ((message.flags & MessageFlags.IS_CROSSPOST) !== 0) { + return message; + } if ( message.mentionedUserIds.size === 0 && message.mentionedRoleIds.size === 0 && @@ -216,7 +219,7 @@ export class MessageProcessingService { } } const referencedMessage = - message.reference && message.mentionedUserIds.size > 0 + message.reference?.messageId && message.mentionedUserIds.size > 0 ? await this.channelRepository.messages.getMessage(message.reference.channelId, message.reference.messageId) : null; const repair = await this.mentionService.buildReadRepairMentionData({message, referencedMessage}); diff --git a/fluxer_api/src/api/channel/services/message/MessageRequestService.ts b/fluxer_api/src/api/channel/services/message/MessageRequestService.ts index 759b94f5b..b5c0ac1a8 100644 --- a/fluxer_api/src/api/channel/services/message/MessageRequestService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageRequestService.ts @@ -3,6 +3,7 @@ import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; import type {MessageRequest, MessageUpdateRequest} from '@app/api/channel/MessageTypes'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; +import type {CrosspostSourceService} from '@app/api/channel/services/message/CrosspostSourceService'; import {isPersonalNotesChannel} from '@app/api/channel/services/message/MessageHelpers'; import type {MessageResponseDataService} from '@app/api/channel/services/message/MessageResponseDataService'; import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; @@ -10,6 +11,7 @@ import type {User} from '@app/api/models/User'; import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils'; import {UnclaimedAccountCannotSendMessagesError} from '@fluxer/errors/src/domains/channel/UnclaimedAccountCannotSendMessagesError'; import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; +import type {CrosspostSourceResponse} from '@fluxer/schema/src/domains/message/CrosspostSourceSchemas'; import type { BulkMessageFetchResponse, MessageResponse, @@ -19,6 +21,7 @@ export class MessageRequestService { constructor( private readonly channelService: ChannelService, private readonly responseDataService: MessageResponseDataService, + private readonly crosspostSourceService: CrosspostSourceService, ) {} async listMessages(params: { @@ -95,6 +98,16 @@ export class MessageRequestService { return response; } + async getCrosspostSource(params: { + userId: UserID; + channelId: ChannelID; + messageId: MessageID; + requestCache: RequestCache; + }): Promise { + const message = await this.getMessage(params); + return this.crosspostSourceService.getSource(message); + } + async sendMessage(params: { user: User; channelId: ChannelID; @@ -127,6 +140,26 @@ export class MessageRequestService { }); } + async crosspostMessage(params: { + userId: UserID; + channelId: ChannelID; + messageId: MessageID; + requestCache: RequestCache; + }): Promise { + const {message, authChannel} = await this.channelService.messages.crosspost.crosspostMessage(params); + const access = await this.channelService.messages.retrieval.getResponseAccessContext({ + userId: params.userId, + channelId: params.channelId, + messageId: message.id, + authChannel, + }); + return this.responseDataService.buildMessage({ + userId: params.userId, + message, + access, + }); + } + async editMessage(params: { userId: UserID; channelId: ChannelID; diff --git a/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts b/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts index 16b685107..cec00d1c1 100644 --- a/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts @@ -7,7 +7,11 @@ import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IC import type {AuthenticatedChannel} from '@app/api/channel/services/AuthenticatedChannel'; import {getDmChannelIdsForScope} from '@app/api/channel/services/message/DmScopeUtils'; import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; -import {collectMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; +import { + attachmentStorageChannelId, + collectMessageAttachments, + isCrosspostCopy, +} from '@app/api/channel/services/message/MessageHelpers'; import type {MessageProcessingService} from '@app/api/channel/services/message/MessageProcessingService'; import { createMessageResponseDataService, @@ -254,11 +258,13 @@ export class MessageRetrievalService { }> { const attachments = collectMessageAttachments(message); if (attachments.length === 0) return []; - const uploadedAt = snowflakeToDate(message.id); + const ownerMessageId = isCrosspostCopy(message) ? (message.reference?.messageId ?? message.id) : message.id; + const uploadedAt = snowflakeToDate(ownerMessageId); + const storageChannelId = attachmentStorageChannelId(message); return attachments.map((attachment) => ({ attachmentId: attachment.id, - channelId: message.channelId, - messageId: message.id, + channelId: storageChannelId, + messageId: ownerMessageId, filename: attachment.filename, sizeBytes: attachment.size, uploadedAt, diff --git a/fluxer_api/src/api/channel/services/message/MessageSendService.ts b/fluxer_api/src/api/channel/services/message/MessageSendService.ts index fa63274b4..cb7dca49f 100644 --- a/fluxer_api/src/api/channel/services/message/MessageSendService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageSendService.ts @@ -15,6 +15,7 @@ import type {MessageRequest, MessageUpdateRequest} from '@app/api/channel/Messag import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IChannelRepositoryAggregate'; import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository'; import type {AuthenticatedChannel} from '@app/api/channel/services/AuthenticatedChannel'; +import type {CrosspostPropagation} from '@app/api/channel/services/message/CrosspostPropagation'; import {emitMessageCreated} from '@app/api/channel/services/message/MessageActivity'; import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; import type {DmNsfwContext} from '@app/api/channel/services/message/MessageContentService'; @@ -26,12 +27,14 @@ import { isOperationDisabled, isPersonalNotesChannel, } from '@app/api/channel/services/message/MessageHelpers'; +import {assertMessageWithinHistoryCutoff} from '@app/api/channel/services/message/MessageHistoryCutoff'; import type {MessageMentionService} from '@app/api/channel/services/message/MessageMentionService'; import type {MessageOperationsHelpers} from '@app/api/channel/services/message/MessageOperationsHelpers'; import type {MessagePersistenceService} from '@app/api/channel/services/message/MessagePersistenceService'; import type {MessageProcessingService} from '@app/api/channel/services/message/MessageProcessingService'; import type {MessageSearchService} from '@app/api/channel/services/message/MessageSearchService'; import type {MessageValidationService} from '@app/api/channel/services/message/MessageValidationService'; +import type {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; import {SYSTEM_USER_ID} from '@app/api/constants/Core'; import type {MessageAttachment, MessageReference} from '@app/api/database/types/MessageTypes'; import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository'; @@ -75,7 +78,6 @@ import {SlowmodeRateLimitError} from '@fluxer/errors/src/domains/core/SlowmodeRa import {NsfwContentRequiresAgeVerificationError} from '@fluxer/errors/src/domains/moderation/NsfwContentRequiresAgeVerificationError'; import type {GuildMemberResponse} from '@fluxer/schema/src/domains/guild/GuildMemberSchemas'; import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; -import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake'; import type {IRateLimitService} from '@pkgs/rate_limit/src/IRateLimitService'; interface MessageSendServiceDeps { @@ -97,6 +99,8 @@ interface MessageSendServiceDeps { embedAttachmentResolver: MessageEmbedAttachmentResolver; attachmentUploadTraceRepository: AttachmentUploadTraceRepository; limitConfigService: LimitConfigService; + messageWriteLock: MessageWriteLock; + crosspostPropagation: CrosspostPropagation; } interface SendMessageResult { @@ -360,8 +364,8 @@ export class MessageSendService { if (data.message_reference && guild && !isForwardMessage) { const hasReadHistory = await hasPermission(Permissions.READ_MESSAGE_HISTORY); if (!hasReadHistory) { - this.assertReferencedMessageWithinCutoff({ - referencedMessage, + assertMessageWithinHistoryCutoff({ + message: referencedMessage, guild, }); } @@ -722,27 +726,6 @@ export class MessageSendService { return {attachmentsToProcess, favoriteMemeAttachment}; } - private assertReferencedMessageWithinCutoff({ - referencedMessage, - guild, - }: { - referencedMessage: Message | null; - guild: GuildResponse; - }): void { - if (!referencedMessage) { - throw new UnknownMessageError(); - } - const cutoff = guild.message_history_cutoff; - if (!cutoff) { - throw new UnknownMessageError(); - } - const messageTimestamp = snowflakeToDate(referencedMessage.id).getTime(); - const cutoffTimestamp = new Date(cutoff).getTime(); - if (messageTimestamp < cutoffTimestamp) { - throw new UnknownMessageError(); - } - } - private getMessageTypeForRequest(data: MessageRequest): number { if (!data.message_reference) { return MessageTypes.DEFAULT; @@ -859,8 +842,8 @@ export class MessageSendService { if (data.message_reference && guild && !isForwardMessage) { const hasReadHistory = await hasPermission(Permissions.READ_MESSAGE_HISTORY); if (!hasReadHistory) { - this.assertReferencedMessageWithinCutoff({ - referencedMessage, + assertMessageWithinHistoryCutoff({ + message: referencedMessage, guild, }); } @@ -1276,16 +1259,30 @@ export class MessageSendService { existingAttachments: existingMessage.attachments.map((att) => ({filename: att.filename})), }); } - const {message: updatedMessage, enqueueDeferredEmbeds} = await this.deps.persistenceService.updateMessage({ - message: existingMessage, + const attachmentUploadUserId = await this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments); + const {message: updatedMessage, enqueueDeferredEmbeds} = await this.deps.messageWriteLock.withFreshMessage( + channelId, messageId, - data, - channel, - guild, - attachmentUploadUserId: await this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments), - allowEmbeds: true, - }); + async (fresh) => { + if (!fresh) throw new UnknownMessageError(); + if (fresh.webhookId !== webhook.id) { + throw new MissingPermissionsError(); + } + return this.deps.crosspostPropagation.withPublishedEditBudget({fresh, actor: 'webhook'}, () => + this.deps.persistenceService.updateMessage({ + message: fresh, + messageId, + data, + channel, + guild, + attachmentUploadUserId, + allowEmbeds: true, + }), + ); + }, + ); await this.deps.dispatchService.dispatchMessageUpdate({channel, message: updatedMessage, requestCache}); + await this.deps.crosspostPropagation.propagateEdit(updatedMessage); void enqueueDeferredEmbeds().catch((error) => { Logger.warn({error, messageId: messageId.toString()}, 'Failed to enqueue deferred embed extraction after edit'); }); diff --git a/fluxer_api/src/api/channel/services/message/MessageWriteLock.ts b/fluxer_api/src/api/channel/services/message/MessageWriteLock.ts new file mode 100644 index 000000000..0fd926c17 --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/MessageWriteLock.ts @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, MessageID} from '@app/api/BrandedTypes'; +import type {Message} from '@app/api/models/Message'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ThrottledError} from '@fluxer/errors/src/domains/core/ThrottledError'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; + +const MESSAGE_WRITE_LOCK_TTL_SECONDS = 5; +const MESSAGE_WRITE_LOCK_ACQUIRE_ATTEMPTS = 6; +const MESSAGE_WRITE_LOCK_RETRY_DELAY_MS = 50; + +export interface MessageWriteLockReader { + getMessage(channelId: ChannelID, messageId: MessageID): Promise; +} + +export function messageWriteLockKey(channelId: ChannelID, messageId: MessageID): string { + return `message:${channelId}:${messageId}:write`; +} + +export class MessageWriteLock { + constructor( + private readonly cacheService: ICacheService, + private readonly messages: MessageWriteLockReader, + ) {} + + async withLock(channelId: ChannelID, messageId: MessageID, fn: () => Promise): Promise { + const lockKey = messageWriteLockKey(channelId, messageId); + let lockToken: string | null = null; + for (let attempt = 0; attempt < MESSAGE_WRITE_LOCK_ACQUIRE_ATTEMPTS; attempt++) { + lockToken = await this.cacheService.acquireLock(lockKey, MESSAGE_WRITE_LOCK_TTL_SECONDS); + if (lockToken) break; + await new Promise((resolve) => setTimeout(resolve, MESSAGE_WRITE_LOCK_RETRY_DELAY_MS * (attempt + 1))); + } + if (!lockToken) { + throw new ThrottledError({ + code: APIErrorCodes.RESOURCE_LOCKED, + retryAfterSeconds: 1, + data: {retry_after: 1}, + }); + } + try { + return await fn(); + } finally { + await this.cacheService.releaseLock(lockKey, lockToken).catch(() => {}); + } + } + + async withFreshMessage( + channelId: ChannelID, + messageId: MessageID, + fn: (fresh: Message | null) => Promise, + ): Promise { + return this.withLock(channelId, messageId, async () => { + const fresh = await this.messages.getMessage(channelId, messageId); + return fn(fresh); + }); + } +} diff --git a/fluxer_api/src/api/channel/services/message/UserMessageDeletionService.ts b/fluxer_api/src/api/channel/services/message/UserMessageDeletionService.ts index b3cf03f38..d15380b50 100644 --- a/fluxer_api/src/api/channel/services/message/UserMessageDeletionService.ts +++ b/fluxer_api/src/api/channel/services/message/UserMessageDeletionService.ts @@ -3,6 +3,10 @@ import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createChannelID} from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import { + type CrosspostWorkerService, + enqueueCrosspostSourceRemoval, +} from '@app/api/channel/services/message/CrosspostPropagation'; import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; import { isChannelEligible, @@ -26,6 +30,7 @@ interface UserMessageDeletionServiceDeps { gatewayService: IGatewayService; storageService: IStorageService; purgeQueue: IPurgeQueue; + workerService: CrosspostWorkerService; } interface DeleteUserMessagesScope { @@ -232,6 +237,11 @@ export class UserMessageDeletionService { ); await this.deps.channelRepository.bulkDeleteMessages(channelId, messageIds); await this.eventDispatcher.dispatchBulkDelete(channel, messageIds); + await enqueueCrosspostSourceRemoval(this.deps.workerService, { + messages: messageObjects, + mode: 'source_deleted', + channel, + }); await deleteMessageSearchDocuments(messageIds, {context: {source: 'bulk_user_message_delete'}}); deleted += batch.length; } diff --git a/fluxer_api/src/api/channel/tests/AnnouncementChannelConversion.test.ts b/fluxer_api/src/api/channel/tests/AnnouncementChannelConversion.test.ts new file mode 100644 index 000000000..c39e143b6 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/AnnouncementChannelConversion.test.ts @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createGuildID, createUserID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes'; +import {disableCrosspostWorker, followRequest, publishRequest} from '@app/api/channel/tests/AnnouncementTestUtils'; +import { + createChannel, + createFriendship, + createGroupDmChannel, + createGuild, + deleteChannel, + getChannel, + setupTestGuildWithMembers, + updateChannel, +} from '@app/api/channel/tests/ChannelTestUtils'; +import { + copiesOf, + editRequest, + type FanoutWorld, + followInto, + postAndPublish, + sendMessage, + setupFanoutWorld, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, getChannelWebhooks} from '@app/api/webhook/tests/WebhookTestUtils'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType'; +import {ChannelTypes, WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; +import type {WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +interface RecordedJob { + taskType: string; + payload: WorkerJobPayload; + options: WorkerJobOptions | undefined; +} + +class RecordingWorkerService implements IWorkerService { + readonly jobs: Array = []; + private nextJobId = 1n; + + async addJob( + taskType: string, + payload: TPayload, + options?: WorkerJobOptions, + ): Promise { + this.jobs.push({taskType, payload, options}); + return this.nextJobId++; + } + + async cancelJob(_jobId: bigint): Promise { + return false; + } + + async retryDeadLetterJob(_jobId: bigint): Promise { + return false; + } + + followerRemovals(): Array { + return this.jobs.filter((job) => job.taskType === 'removeChannelFollowers'); + } +} + +interface AuditLogResponse { + audit_log_entries: Array<{ + action_type: number; + target_id: string | null; + changes?: Array<{key: string; old_value?: unknown; new_value?: unknown}>; + }>; +} + +async function insertFollowerWebhook(params: {guildId: string; channelId: string; creatorId: string}): Promise { + const webhookId = createWebhookID(BigInt(Date.now()) * 1000n + BigInt(Math.floor(Math.random() * 1000))); + await new WebhookRepository().create({ + webhookId, + token: createWebhookToken('f'.repeat(64)), + type: WebhookTypes.CHANNEL_FOLLOWER, + guildId: createGuildID(BigInt(params.guildId)), + channelId: createChannelID(BigInt(params.channelId)), + creatorId: createUserID(BigInt(params.creatorId)), + name: 'Source Guild #news', + avatarHash: null, + }); + return webhookId.toString(); +} + +describe('Announcement channel conversion', () => { + let harness: ApiTestHarness; + let worker: RecordingWorkerService; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + worker = new RecordingWorkerService(); + setInjectedWorkerService(worker); + }); + + afterEach(() => { + setInjectedWorkerService(new NoopWorkerService()); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('converts a text channel to an announcement channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Convert Guild'); + const text = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_TEXT); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const converted = await createBuilder(harness, owner.token) + .patch(`/channels/${text.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(converted.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect((await getChannel(harness, owner.token, text.id)).type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + const channelUpdates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'CHANNEL_UPDATE'); + expect(channelUpdates).toHaveLength(1); + expect(channelUpdates[0]?.data).toMatchObject({type: ChannelTypes.GUILD_ANNOUNCEMENT}); + const auditLog = await createBuilder(harness, owner.token) + .get(`/guilds/${guild.id}/audit-logs?action_type=${AuditLogActionType.CHANNEL_UPDATE}`) + .expect(HTTP_STATUS.OK) + .execute(); + const entry = auditLog.audit_log_entries.find((candidate) => candidate.target_id === text.id); + const typeChange = entry?.changes?.find((change) => change.key === 'type'); + expect(typeChange?.old_value).toBe(ChannelTypes.GUILD_TEXT); + expect(typeChange?.new_value).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(worker.followerRemovals()).toHaveLength(0); + }); + + test('converts an announcement channel back to text and schedules follower removal', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Revert Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const converted = await createBuilder(harness, owner.token) + .patch(`/channels/${announcement.id}`) + .body({type: ChannelTypes.GUILD_TEXT}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(converted.type).toBe(ChannelTypes.GUILD_TEXT); + const removals = worker.followerRemovals(); + expect(removals).toHaveLength(1); + expect(removals[0]?.payload).toEqual({sourceChannelId: announcement.id, reason: 'converted'}); + expect(removals[0]?.options?.jobKey).toMatch(new RegExp(`^remove-followers:${announcement.id}:converted:\\d+$`)); + expect(removals[0]?.options?.skipLedger).toBeUndefined(); + }); + + test('keeps the type when the body has no type or repeats the current type', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Keep Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const renamed = await updateChannel(harness, owner.token, announcement.id, {topic: 'release notes'}); + expect(renamed.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(renamed.topic).toBe('release notes'); + const repeated = await createBuilder(harness, owner.token) + .patch(`/channels/${announcement.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT, topic: 'still news'}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(repeated.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + const nulled = await createBuilder(harness, owner.token) + .patch(`/channels/${announcement.id}`) + .body({type: null, topic: 'null type'}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(nulled.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(worker.followerRemovals()).toHaveLength(0); + }); + + test('rejects conversions outside text and announcement', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Reject Guild'); + const text = await createChannel(harness, owner.token, guild.id, 'text', ChannelTypes.GUILD_TEXT); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const attempts: Array<{channelId: string; type: number | string}> = [ + {channelId: text.id, type: ChannelTypes.GUILD_VOICE}, + {channelId: text.id, type: ChannelTypes.GUILD_CATEGORY}, + {channelId: text.id, type: 'announcement'}, + {channelId: announcement.id, type: ChannelTypes.GUILD_VOICE}, + {channelId: announcement.id, type: ChannelTypes.GROUP_DM}, + ]; + for (const attempt of attempts) { + await createBuilder(harness, owner.token) + .patch(`/channels/${attempt.channelId}`) + .body({type: attempt.type}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CHANNEL_TYPE_CONVERSION_NOT_SUPPORTED) + .execute(); + } + expect((await getChannel(harness, owner.token, text.id)).type).toBe(ChannelTypes.GUILD_TEXT); + expect((await getChannel(harness, owner.token, announcement.id)).type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('ignores a body type on channels that cannot be converted', async () => { + const owner = await createTestAccount(harness); + const friend = await createTestAccount(harness); + await createFriendship(harness, owner, friend); + const guild = await createGuild(harness, owner.token, 'Ignore Guild'); + const voice = await createChannel(harness, owner.token, guild.id, 'voice', ChannelTypes.GUILD_VOICE); + const category = await createChannel(harness, owner.token, guild.id, 'category', ChannelTypes.GUILD_CATEGORY); + const link = await createBuilder(harness, owner.token) + .post(`/guilds/${guild.id}/channels`) + .body({name: 'link', type: ChannelTypes.GUILD_LINK, url: 'https://example.com'}) + .execute(); + const groupDm = await createGroupDmChannel(harness, owner.token, [friend.userId]); + const attempts: Array<{channelId: string; type: number; expected: number}> = [ + {channelId: voice.id, type: ChannelTypes.GUILD_TEXT, expected: ChannelTypes.GUILD_VOICE}, + {channelId: voice.id, type: ChannelTypes.GUILD_ANNOUNCEMENT, expected: ChannelTypes.GUILD_VOICE}, + {channelId: category.id, type: ChannelTypes.GUILD_ANNOUNCEMENT, expected: ChannelTypes.GUILD_CATEGORY}, + {channelId: link.id, type: ChannelTypes.GUILD_TEXT, expected: ChannelTypes.GUILD_LINK}, + {channelId: groupDm.id, type: ChannelTypes.GUILD_ANNOUNCEMENT, expected: ChannelTypes.GROUP_DM}, + ]; + for (const [index, attempt] of attempts.entries()) { + const name = `renamed-${index}`; + const updated = await createBuilder(harness, owner.token) + .patch(`/channels/${attempt.channelId}`) + .body({type: attempt.type, name}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(updated.type).toBe(attempt.expected); + expect(updated.name).toBe(name); + const stored = await getChannel(harness, owner.token, attempt.channelId); + expect(stored.type).toBe(attempt.expected); + expect(stored.name).toBe(name); + } + }); + + test('requires MANAGE_CHANNELS', async () => { + const {owner, members, guild} = await setupTestGuildWithMembers(harness, 1); + const member = members[0]!; + const text = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_TEXT); + const announcement = await createChannel(harness, owner.token, guild.id, 'ann', ChannelTypes.GUILD_ANNOUNCEMENT); + await createBuilder(harness, member.token) + .patch(`/channels/${text.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await createBuilder(harness, member.token) + .patch(`/channels/${announcement.id}`) + .body({type: ChannelTypes.GUILD_TEXT}) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + expect((await getChannel(harness, owner.token, text.id)).type).toBe(ChannelTypes.GUILD_TEXT); + expect((await getChannel(harness, owner.token, announcement.id)).type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(worker.followerRemovals()).toHaveLength(0); + }); + + test('refuses to convert a channel that receives follows until the follow is removed', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Followed Guild'); + const text = await createChannel(harness, owner.token, guild.id, 'feed', ChannelTypes.GUILD_TEXT); + await createWebhook(harness, text.id, owner.token, 'Incoming Bot'); + const followerWebhookId = await insertFollowerWebhook({ + guildId: guild.id, + channelId: text.id, + creatorId: owner.userId, + }); + const channelWebhooks = await getChannelWebhooks(harness, text.id, owner.token); + expect(channelWebhooks.some((webhook) => webhook.id === followerWebhookId)).toBe(true); + await createBuilder(harness, owner.token) + .patch(`/channels/${text.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CHANNEL_HAS_FOLLOWED_CHANNELS) + .execute(); + expect((await getChannel(harness, owner.token, text.id)).type).toBe(ChannelTypes.GUILD_TEXT); + await createBuilder(harness, owner.token) + .delete(`/webhooks/${followerWebhookId}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + const converted = await createBuilder(harness, owner.token) + .patch(`/channels/${text.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(converted.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('an incoming webhook does not block conversion', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Incoming Guild'); + const text = await createChannel(harness, owner.token, guild.id, 'feed', ChannelTypes.GUILD_TEXT); + await createWebhook(harness, text.id, owner.token, 'Incoming Bot'); + const converted = await createBuilder(harness, owner.token) + .patch(`/channels/${text.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(converted.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('records a distinct job for every conversion back to text', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Flip Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + for (const type of [ChannelTypes.GUILD_TEXT, ChannelTypes.GUILD_ANNOUNCEMENT, ChannelTypes.GUILD_TEXT]) { + await createBuilder(harness, owner.token) + .patch(`/channels/${announcement.id}`) + .body({type}) + .expect(HTTP_STATUS.OK) + .execute(); + } + const removals = worker.followerRemovals(); + expect(removals).toHaveLength(2); + const jobKeys = new Set(removals.map((job) => job.options?.jobKey)); + expect(jobKeys.size).toBe(2); + }); + + test('deleting an announcement channel schedules follower removal with source-deleted copies', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Delete Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const text = await createChannel(harness, owner.token, guild.id, 'text', ChannelTypes.GUILD_TEXT); + await deleteChannel(harness, owner.token, text.id); + expect(worker.followerRemovals()).toHaveLength(0); + await deleteChannel(harness, owner.token, announcement.id); + const removals = worker.followerRemovals(); + expect(removals).toHaveLength(1); + expect(removals[0]?.payload).toEqual({ + sourceChannelId: announcement.id, + reason: 'deleted', + copyMode: 'source_deleted', + }); + expect(removals[0]?.options?.jobKey).toMatch(new RegExp(`^remove-followers:${announcement.id}:deleted:\\d+$`)); + }); + + test('deleting a guild schedules follower removal for each announcement channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Owner Delete Guild'); + const first = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const second = await createChannel(harness, owner.token, guild.id, 'updates', ChannelTypes.GUILD_ANNOUNCEMENT); + await createBuilder(harness, owner.token) + .post(`/guilds/${guild.id}/delete`) + .body({password: owner.password}) + .expect(HTTP_STATUS.NO_CONTENT) + .executeWithResponse(); + const removals = worker.followerRemovals(); + expect(removals.map((job) => job.payload.sourceChannelId).sort()).toEqual([first.id, second.id].sort()); + for (const job of removals) { + expect(job.payload.reason).toBe('deleted'); + expect(job.payload.copyMode).toBe('source_deleted'); + } + }); + + test('admin guild deletion purges copies', async () => { + const admin = await createTestAccount(harness); + await setUserACLs(harness, admin, ['admin:authenticate', 'guild:lookup', 'guild:delete']); + const guild = await createGuild(harness, admin.token, 'Admin Delete Guild'); + const announcement = await createChannel(harness, admin.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + await createBuilder(harness, admin.token) + .delete(`/admin/guilds/${guild.id}`) + .body(null) + .expect(HTTP_STATUS.OK) + .execute(); + const removals = worker.followerRemovals(); + expect(removals).toHaveLength(1); + expect(removals[0]?.payload).toEqual({ + sourceChannelId: announcement.id, + reason: 'deleted', + copyMode: 'purge', + }); + }); +}); + +describe('Announcement channel conversion with the crosspost worker', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function convert(type: number): Promise { + await createBuilder(harness, world.a.owner.token) + .patch(`/channels/${world.a.ann.id}`) + .body({type}) + .expect(HTTP_STATUS.OK) + .execute(); + } + + async function followerWebhooks(channelId: string) { + const webhooks = await new WebhookRepository().listByChannel(createChannelID(BigInt(channelId))); + return webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER); + } + + test('converting to text removes every follower after the drain and leaves copies propagating', async () => { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + const source = await postAndPublish(harness, world, {content: 'before conversion'}); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await convert(ChannelTypes.GUILD_TEXT); + expect(world.worker.byTask('removeChannelFollowers')).toHaveLength(1); + expect(await followerWebhooks(world.b.t1.id)).toHaveLength(1); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + expect(await followerWebhooks(world.b.t1.id)).toHaveLength(0); + expect(await followerWebhooks(world.b.t2.id)).toHaveLength(0); + const updates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'WEBHOOKS_UPDATE' && params.guildId.toString() === world.b.guild.id); + expect(updates.map((params) => (params.data as {channel_id: string}).channel_id).sort()).toEqual( + [world.b.t1.id, world.b.t2.id].sort(), + ); + const audit = await createBuilder(harness, world.b.owner.token) + .get(`/guilds/${world.b.guild.id}/audit-logs?action_type=${AuditLogActionType.WEBHOOK_DELETE}`) + .execute(); + expect(audit.audit_log_entries).toHaveLength(0); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy!.content).toBe('before conversion'); + await editRequest(harness, world.a.owner.token, world.a.ann.id, source.id, {content: 'after conversion'}) + .expect(HTTP_STATUS.OK) + .execute(); + await world.worker.drain(); + for (const channelId of [world.b.t1.id, world.b.t2.id]) { + const [updated] = await copiesOf(harness, world.b.owner.token, channelId, source.id); + expect(updated!.content).toBe('after conversion'); + } + }); + + test('converting back before the drain keeps the followers', async () => { + await followInto(harness, world, world.b.t1.id); + await convert(ChannelTypes.GUILD_TEXT); + await convert(ChannelTypes.GUILD_ANNOUNCEMENT); + await world.worker.drain(); + expect(await followerWebhooks(world.b.t1.id)).toHaveLength(1); + const source = await postAndPublish(harness, world, {content: 'still followed'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(1); + }); + + test('flipping twice records two removals and ends with no followers', async () => { + await followInto(harness, world, world.b.t1.id); + await convert(ChannelTypes.GUILD_TEXT); + await convert(ChannelTypes.GUILD_ANNOUNCEMENT); + await convert(ChannelTypes.GUILD_TEXT); + const removals = world.worker.byTask('removeChannelFollowers'); + expect(new Set(removals.map((job) => job.jobKey)).size).toBe(2); + await world.worker.drain(); + expect(await followerWebhooks(world.b.t1.id)).toHaveLength(0); + }); + + test('publishing or following after converting to text is refused', async () => { + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'late'}); + await convert(ChannelTypes.GUILD_TEXT); + await publishRequest(harness, world.a.owner.token, world.a.ann.id, message.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + await followRequest(harness, world.b.owner.token, world.a.ann.id, world.b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/AnnouncementChannelType.test.ts b/fluxer_api/src/api/channel/tests/AnnouncementChannelType.test.ts new file mode 100644 index 000000000..e991859eb --- /dev/null +++ b/fluxer_api/src/api/channel/tests/AnnouncementChannelType.test.ts @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils'; +import { + acceptInvite, + createChannel, + createChannelInvite, + createGuild, + getChannel, + getGuild, + sendChannelMessage, + setupTestGuildWithMembers, + updateChannel, + updateGuild, +} from '@app/api/channel/tests/ChannelTestUtils'; +import {getGuildChannels, updateChannelPositions} from '@app/api/guild/tests/GuildTestUtils'; +import {deleteMessage, ensureSessionStarted, getMessages, pinMessage} from '@app/api/message/tests/MessageTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, executeWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import {sortChannelsForOrdering} from '@fluxer/schema/src/domains/channel/GuildChannelOrdering'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; + +describe('Announcement channel type', () => { + let harness: ApiTestHarness; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('creates a type 5 channel with the text name rules', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Announcement Guild'); + const text = await createChannel(harness, owner.token, guild.id, 'Big News', ChannelTypes.GUILD_TEXT); + const announcement = await createChannel( + harness, + owner.token, + guild.id, + 'Big News', + ChannelTypes.GUILD_ANNOUNCEMENT, + ); + expect(announcement.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(announcement.name).toBe(text.name); + expect(announcement.name).not.toBe('Big News'); + const fetched = await getChannel(harness, owner.token, announcement.id); + expect(fetched.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('keeps the name as typed when flexible names are enabled', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Flexible Guild'); + await updateGuild(harness, owner.token, guild.id, { + features: [...guild.features, GuildFeatures.TEXT_CHANNEL_FLEXIBLE_NAMES], + }); + const announcement = await createChannel( + harness, + owner.token, + guild.id, + 'Big News', + ChannelTypes.GUILD_ANNOUNCEMENT, + ); + expect(announcement.name).toBe('Big News'); + const renamed = await updateChannel(harness, owner.token, announcement.id, {name: 'Even Bigger News'}); + expect(renamed.name).toBe('Even Bigger News'); + expect(renamed.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('normalises a rename like a text channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Rename Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const renamed = await updateChannel(harness, owner.token, announcement.id, {name: 'Release Notes'}); + expect(renamed.name).not.toBe('Release Notes'); + expect(renamed.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + }); + + test('lists the channel with text before voice in a category', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Ordering Guild'); + const category = await createChannel(harness, owner.token, guild.id, 'Mixed', ChannelTypes.GUILD_CATEGORY); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const voice = await createChannel(harness, owner.token, guild.id, 'voice', ChannelTypes.GUILD_VOICE); + await updateChannelPositions(harness, owner.token, guild.id, [ + {id: voice.id, parent_id: category.id}, + {id: announcement.id, parent_id: category.id}, + ]); + const channels = await getGuildChannels(harness, owner.token, guild.id); + const listed = channels.find((channel) => channel.id === announcement.id); + expect(listed?.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + const ordered = sortChannelsForOrdering( + channels.map((channel) => ({ + id: channel.id, + parentId: channel.parent_id ?? null, + type: channel.type, + position: channel.position, + })), + ) + .filter((channel) => channel.parentId === category.id) + .map((channel) => channel.id); + expect(ordered.indexOf(announcement.id)).toBeLessThan(ordered.indexOf(voice.id)); + const response = await createBuilder<{code: string; errors: Array<{path: string; code: string}>}>( + harness, + owner.token, + ) + .patch(`/guilds/${guild.id}/channels`) + .body([{id: announcement.id, parent_id: category.id, preceding_sibling_id: voice.id}]) + .expect(HTTP_STATUS.BAD_REQUEST) + .execute(); + expect(response.errors[0]?.code).toBe(ValidationErrorCodes.VOICE_CHANNELS_CANNOT_BE_ABOVE_TEXT_CHANNELS); + }); + + test('supports sending, editing, reacting, pinning, uploading and deleting', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Messaging Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + await ensureSessionStarted(harness, owner.token); + const message = await sendChannelMessage(harness, owner.token, announcement.id, 'first announcement'); + expect(message.channel_id).toBe(announcement.id); + const edited = await createBuilder(harness, owner.token) + .patch(`/channels/${announcement.id}/messages/${message.id}`) + .body({content: 'edited announcement'}) + .execute(); + expect(edited.content).toBe('edited announcement'); + await createBuilder(harness, owner.token) + .put(`/channels/${announcement.id}/messages/${message.id}/reactions/%F0%9F%91%8D/@me`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + await pinMessage(harness, owner.token, announcement.id, message.id); + const {response: uploadResponse, json: uploaded} = await sendMessageWithAttachments( + harness, + owner.token, + announcement.id, + {content: 'with a file', attachments: [{id: 0, filename: 'notes.txt'}]}, + [{index: 0, filename: 'notes.txt', data: Buffer.from('release notes')}], + ); + expect(uploadResponse.status).toBe(HTTP_STATUS.OK); + expect(uploaded.attachments).toHaveLength(1); + const messages = await getMessages(harness, owner.token, announcement.id); + const reacted = messages.find((entry) => entry.id === message.id); + expect(reacted?.reactions?.[0]?.count).toBe(1); + expect(reacted?.pinned).toBe(true); + await deleteMessage(harness, owner.token, announcement.id, uploaded.id); + const remaining = await getMessages(harness, owner.token, announcement.id); + expect(remaining.some((entry) => entry.id === uploaded.id)).toBe(false); + }); + + test('enforces slowmode', async () => { + const {owner, members, guild} = await setupTestGuildWithMembers(harness, 1); + const member = members[0]!; + await ensureSessionStarted(harness, member.token); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const updated = await updateChannel(harness, owner.token, announcement.id, {rate_limit_per_user: 30}); + expect(updated.rate_limit_per_user).toBe(30); + await sendChannelMessage(harness, member.token, announcement.id, 'first'); + await createBuilder(harness, member.token) + .post(`/channels/${announcement.id}/messages`) + .body({content: 'second'}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.SLOWMODE_RATE_LIMITED) + .execute(); + }); + + test('creates and executes an incoming webhook', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Webhook Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const webhook = await createWebhook(harness, announcement.id, owner.token, 'Release Bot'); + expect(webhook.channel_id).toBe(announcement.id); + const {json} = await executeWebhook(harness, webhook.id, webhook.token, {content: 'shipped', wait: true}, 200); + expect(json?.channel_id).toBe(announcement.id); + expect(json?.webhook_id).toBe(webhook.id); + }); + + test('applies the age gate to an age-restricted announcement channel', async () => { + const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'}); + const minor = await createTestAccount(harness, {dateOfBirth: '2012-01-01'}); + const guild = await createGuild(harness, owner.token, 'Age Gate Guild'); + const invite = await createChannelInvite(harness, owner.token, guild.system_channel_id!); + await acceptInvite(harness, minor.token, invite.code); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + await sendChannelMessage(harness, owner.token, announcement.id, 'before the gate'); + const minorView = await getMessages(harness, minor.token, announcement.id); + expect(minorView).toHaveLength(1); + const restricted = await updateChannel(harness, owner.token, announcement.id, {nsfw: true}); + expect(restricted.nsfw).toBe(true); + await createBuilder(harness, minor.token) + .get(`/channels/${announcement.id}/messages`) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.NSFW_CONTENT_AGE_RESTRICTED) + .execute(); + const adultView = await getMessages(harness, owner.token, announcement.id); + expect(adultView).toHaveLength(1); + }); + + test('can be the system channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'System Guild'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const updated = await updateGuild(harness, owner.token, guild.id, {system_channel_id: announcement.id}); + expect(updated.system_channel_id).toBe(announcement.id); + const fetched = await getGuild(harness, owner.token, guild.id); + expect(fetched.system_channel_id).toBe(announcement.id); + }); + + test('still rejects a voice channel as the system channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Voice System Guild'); + const voice = await createChannel(harness, owner.token, guild.id, 'voice', ChannelTypes.GUILD_VOICE); + const response = await createBuilder<{errors: Array<{path: string; code: string}>}>(harness, owner.token) + .patch(`/guilds/${guild.id}`) + .body({system_channel_id: voice.id}) + .expect(HTTP_STATUS.BAD_REQUEST) + .execute(); + expect(response.errors[0]?.code).toBe(ValidationErrorCodes.SYSTEM_CHANNEL_MUST_BE_TEXT); + }); + + test('imports a template announcement channel as type 5 and keeps it as the system channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createBuilder<{id: string; system_channel_id: string | null}>(harness, owner.token) + .post('/guilds') + .body({ + name: 'Template Guild', + template: { + name: 'Template Source', + description: null, + verification_level: 0, + default_message_notifications: 0, + explicit_content_filter: 0, + system_channel_id: '7002', + afk_timeout: 300, + system_channel_flags: 0, + roles: [{id: '0', name: '@everyone', permissions: '0'}], + channels: [ + {id: '7001', type: ChannelTypes.GUILD_VOICE, name: 'lounge', position: 0}, + {id: '7002', type: ChannelTypes.GUILD_ANNOUNCEMENT, name: 'news', position: 1}, + ], + }, + }) + .execute(); + const channels = await getGuildChannels(harness, owner.token, guild.id); + const news = channels.find((channel: ChannelResponse) => channel.name === 'news'); + expect(news?.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(guild.system_channel_id).toBe(news?.id); + }); + + test('uses a template announcement channel as the fallback system channel', async () => { + const owner = await createTestAccount(harness); + const guild = await createBuilder<{id: string; system_channel_id: string | null}>(harness, owner.token) + .post('/guilds') + .body({ + name: 'Fallback Guild', + template: { + name: 'Template Source', + description: null, + verification_level: 0, + default_message_notifications: 0, + explicit_content_filter: 0, + system_channel_id: null, + afk_timeout: 300, + system_channel_flags: 0, + roles: [{id: '0', name: '@everyone', permissions: '0'}], + channels: [{id: '8001', type: ChannelTypes.GUILD_ANNOUNCEMENT, name: 'news', position: 0}], + }, + }) + .execute(); + const channels = await getGuildChannels(harness, owner.token, guild.id); + expect(channels).toHaveLength(1); + expect(channels[0]?.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); + expect(guild.system_channel_id).toBe(channels[0]?.id); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/AnnouncementTestUtils.ts b/fluxer_api/src/api/channel/tests/AnnouncementTestUtils.ts new file mode 100644 index 000000000..5dc672f3c --- /dev/null +++ b/fluxer_api/src/api/channel/tests/AnnouncementTestUtils.ts @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {createWebhookID} from '@app/api/BrandedTypes'; +import {resetApiServicesForTesting} from '@app/api/CreateApiContext'; +import { + acceptInvite, + addMemberRole, + createChannelInvite, + createGuild, + createPermissionOverwrite, + createRole, +} from '@app/api/channel/tests/ChannelTestUtils'; +import {createTestChannelService} from '@app/api/channel/tests/CrosspostTestUtils'; +import {getGatewayService, getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry'; +import { + getAvatarService, + getCacheService, + getChannelRepository, + getGuildRepository, + getLimitConfigService, + getPurgeQueue, + getStorageService, + getUserRepository, + getWebhookRepository, +} from '@app/api/middleware/ServiceSingletons'; +import type {ApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage'; +import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk'; +import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers'; +import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies'; +import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage'; +import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext'; +import type {WorkerDependencies} from '@app/api/worker/WorkerDependencies'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import {ChannelTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import {ContentWarningLevel} from '@fluxer/constants/src/GuildConstants'; +import type {LimitKey} from '@fluxer/constants/src/LimitConfigMetadata'; +import type {LimitConfigSnapshot} from '@fluxer/limits/src/LimitTypes'; +import type {FollowedChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +export interface AnnouncementSourceGuild { + owner: TestAccount; + member: TestAccount; + moderator: TestAccount; + guild: GuildResponse; + ann: ChannelResponse; +} + +export interface AnnouncementTargetGuild { + owner: TestAccount; + webhookManager: TestAccount; + guild: GuildResponse; + t1: ChannelResponse; + t2: ChannelResponse; + voice: ChannelResponse; + ageRestricted: ChannelResponse; + contentWarning: ChannelResponse; +} + +export interface AnnouncementWorld { + a: AnnouncementSourceGuild; + b: AnnouncementTargetGuild; +} + +export async function createGuildChannel( + harness: ApiTestHarness, + token: string, + guildId: string, + body: Record, +): Promise { + return createBuilder(harness, token).post(`/guilds/${guildId}/channels`).body(body).execute(); +} + +export async function addGuildMember( + harness: ApiTestHarness, + owner: TestAccount, + guild: GuildResponse, + member: TestAccount, +): Promise { + const invite = await createChannelInvite(harness, owner.token, guild.system_channel_id!); + await acceptInvite(harness, member.token, invite.code); +} + +export async function grantGuildRole( + harness: ApiTestHarness, + owner: TestAccount, + guildId: string, + member: TestAccount, + permissions: bigint, + name = 'Announcement role', +): Promise { + const role = await createRole(harness, owner.token, guildId, { + name, + permissions: (Permissions.VIEW_CHANNEL | Permissions.SEND_MESSAGES | permissions).toString(), + }); + await addMemberRole(harness, owner.token, guildId, member.userId, role.id); + return role.id; +} + +export async function createAnnouncementSourceGuild( + harness: ApiTestHarness, + name = 'Announcement Source', +): Promise { + const owner = await createTestAccount(harness); + const member = await createTestAccount(harness); + const moderator = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, name); + const ann = await createGuildChannel(harness, owner.token, guild.id, { + name: 'news', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + }); + await addGuildMember(harness, owner, guild, member); + await addGuildMember(harness, owner, guild, moderator); + await grantGuildRole(harness, owner, guild.id, moderator, Permissions.MANAGE_MESSAGES, 'Announcement moderator'); + return {owner, member, moderator, guild, ann}; +} + +export async function createAnnouncementTargetGuild( + harness: ApiTestHarness, + name = 'Announcement Target', +): Promise { + const owner = await createTestAccount(harness); + const webhookManager = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, name); + const t1 = await createGuildChannel(harness, owner.token, guild.id, {name: 't1', type: ChannelTypes.GUILD_TEXT}); + const t2 = await createGuildChannel(harness, owner.token, guild.id, {name: 't2', type: ChannelTypes.GUILD_TEXT}); + const voice = await createGuildChannel(harness, owner.token, guild.id, { + name: 'voice', + type: ChannelTypes.GUILD_VOICE, + }); + const ageRestricted = await createGuildChannel(harness, owner.token, guild.id, { + name: 'restricted', + type: ChannelTypes.GUILD_TEXT, + nsfw_override: true, + }); + const contentWarning = await createGuildChannel(harness, owner.token, guild.id, { + name: 'warned', + type: ChannelTypes.GUILD_TEXT, + content_warning_level: ContentWarningLevel.CONTENT_WARNING, + }); + await addGuildMember(harness, owner, guild, webhookManager); + await createPermissionOverwrite(harness, owner.token, t1.id, webhookManager.userId, { + type: 1, + allow: Permissions.MANAGE_WEBHOOKS.toString(), + deny: '0', + }); + return {owner, webhookManager, guild, t1, t2, voice, ageRestricted, contentWarning}; +} + +export async function announcementWorld(harness: ApiTestHarness): Promise { + const a = await createAnnouncementSourceGuild(harness); + const b = await createAnnouncementTargetGuild(harness); + await addGuildMember(harness, a.owner, a.guild, b.owner); + await addGuildMember(harness, a.owner, a.guild, b.webhookManager); + return {a, b}; +} + +export function followRequest( + harness: ApiTestHarness, + token: string, + sourceChannelId: string, + targetChannelId: string, +) { + return createBuilder(harness, token) + .post(`/channels/${sourceChannelId}/followers`) + .body({webhook_channel_id: targetChannelId}); +} + +export async function follow( + harness: ApiTestHarness, + token: string, + sourceChannelId: string, + targetChannelId: string, +): Promise { + return followRequest(harness, token, sourceChannelId, targetChannelId).execute(); +} + +export function publishRequest(harness: ApiTestHarness, token: string, channelId: string, messageId: string) { + return createBuilder(harness, token).post(`/channels/${channelId}/messages/${messageId}/crosspost`); +} + +export async function publish( + harness: ApiTestHarness, + token: string, + channelId: string, + messageId: string, +): Promise { + return publishRequest(harness, token, channelId, messageId).execute(); +} + +export async function findWebhookRow(webhookId: string) { + return new WebhookRepository().findUnique(createWebhookID(BigInt(webhookId))); +} + +export async function setGuildFeatures( + harness: ApiTestHarness, + guildId: string, + features: {add?: Array; remove?: Array}, +): Promise { + await createBuilder(harness, '') + .post(`/test/guilds/${guildId}/features`) + .body({add_features: features.add ?? [], remove_features: features.remove ?? []}) + .execute(); +} + +export async function setLimitOverride( + harness: ApiTestHarness, + limits: Partial>, +): Promise<() => Promise> { + const admin = await setUserACLs(harness, await createTestAccount(harness), [ + AdminACLs.AUTHENTICATE, + AdminACLs.INSTANCE_LIMIT_CONFIG_VIEW, + AdminACLs.INSTANCE_LIMIT_CONFIG_UPDATE, + ]); + const current = await createBuilder<{limit_config: LimitConfigSnapshot}>(harness, admin.token) + .get('/admin/limit-config') + .execute(); + const writeConfig = async (rules: LimitConfigSnapshot['rules']) => { + await createBuilder(harness, admin.token) + .put('/admin/limit-config') + .body({limit_config: {traitDefinitions: current.limit_config.traitDefinitions, rules}}) + .execute(); + }; + await writeConfig([...current.limit_config.rules, {id: 'announcement_test_override', limits}]); + return async () => { + await writeConfig(current.limit_config.rules); + }; +} + +export const crosspostTaskHandlers: Record = { + crosspostMessage, + crosspostMessageChunk, + syncCrosspostedMessage, + syncCrosspostCopies, + removeChannelFollowers, +}; + +export function enableCrosspostWorker( + extraHandlers: Record = {}, + extraDependencies: Partial = {}, +): SyncTaskWorkerService { + const worker = new SyncTaskWorkerService({...crosspostTaskHandlers, ...extraHandlers}, {deferred: true}); + setInjectedWorkerService(worker); + resetApiServicesForTesting(); + setWorkerDependenciesForTest({ + channelRepository: getChannelRepository(), + webhookRepository: getWebhookRepository(), + userRepository: getUserRepository(), + guildRepository: getGuildRepository(), + gatewayService: getGatewayService(), + storageService: getStorageService(), + avatarService: getAvatarService(), + purgeQueue: getPurgeQueue(), + snowflakeService: getSnowflakeService(), + cacheService: getCacheService(), + limitConfigService: getLimitConfigService(), + channelService: createTestChannelService(), + workerService: worker, + ...extraDependencies, + }); + return worker; +} + +export function disableCrosspostWorker(): void { + setInjectedWorkerService(new NoopWorkerService()); + resetApiServicesForTesting(); + clearWorkerDependencies(); +} diff --git a/fluxer_api/src/api/channel/tests/CrosspostConcurrency.test.ts b/fluxer_api/src/api/channel/tests/CrosspostConcurrency.test.ts new file mode 100644 index 000000000..30a91cc35 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostConcurrency.test.ts @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {MessageRepository} from '@app/api/channel/repositories/MessageRepository'; +import {disableCrosspostWorker} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {pinMessage} from '@app/api/channel/tests/ChannelTestUtils'; +import { + type AnnouncementWorld, + createAnnouncementWorld, + createStallGate, + editMessage, + postMessage, + publish, + publishedEditBudgetRemaining, + type RecordedJob, + RecordingWorkerService, + readMessageRow, +} from '@app/api/channel/tests/CrosspostTestUtils'; +import { + copiesOf, + deleteMessageRequest, + editRequest, + type FanoutWorld, + followInto, + sendMessage, + setupFanoutWorld, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {createWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import { + CROSSPOST_SOURCE_DELETED_CONTENT, + CROSSPOST_SYNC_COALESCE_MS, +} from '@fluxer/constants/src/AnnouncementConstants'; +import {MessageFlags} from '@fluxer/constants/src/ChannelConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +function stallFirstReadOf(messageId: string) { + const gate = createStallGate(); + const original = MessageRepository.prototype.getMessage; + let armed = true; + vi.spyOn(MessageRepository.prototype, 'getMessage').mockImplementation(async function ( + this: MessageRepository, + channelId, + id, + ) { + const result = await original.call(this, channelId, id); + if (armed && id.toString() === messageId) { + armed = false; + await gate.hit(); + } + return result; + }); + return gate; +} + +function expectUpdateSyncJob(job: RecordedJob | undefined, channelId: string, messageId: string): void { + expect(job?.payload).toEqual({channelId, messageId, mode: 'update'}); + const runAt = job?.options?.runAt; + expect(runAt).toBeInstanceOf(Date); + const bucket = Math.floor((runAt!.getTime() - 1000) / CROSSPOST_SYNC_COALESCE_MS) - 1; + expect(runAt!.getTime()).toBe((bucket + 1) * CROSSPOST_SYNC_COALESCE_MS + 1000); + expect(job?.options?.jobKey).toBe(`crosspost-sync:${messageId}:update:${bucket}`); + expect(job?.options?.skipLedger).toBe(true); +} + +describe('Publishing races with other message writers', () => { + let harness: ApiTestHarness; + let worker: RecordingWorkerService; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + worker = new RecordingWorkerService(); + setInjectedWorkerService(worker); + world = await createAnnouncementWorld(harness); + }); + + afterEach(() => { + setInjectedWorkerService(new NoopWorkerService()); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('a stale author edit that sets flags keeps the published flag, counts toward the cap and propagates', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Draft'); + const gate = stallFirstReadOf(message.id); + const pendingEdit = editMessage(harness, world.author.token, world.announcement.id, message.id, { + content: 'Edited while publishing', + flags: MessageFlags.SUPPRESS_EMBEDS, + }) + .expect(200) + .execute(); + await gate.reached; + await publish(harness, world.author.token, world.announcement.id, message.id); + gate.release(); + const edited = await pendingEdit; + expect(edited.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + const row = await readMessageRow(world.announcement.id, message.id); + expect(row?.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + expect(row?.content).toBe('Edited while publishing'); + expect(await publishedEditBudgetRemaining(message.id)).toBe(2); + const syncs = worker.syncJobsFor(message.id); + expect(syncs).toHaveLength(1); + expectUpdateSyncJob(syncs[0], world.announcement.id, message.id); + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: 'Second edit'}) + .expect(200) + .execute(); + expect(await publishedEditBudgetRemaining(message.id)).toBe(1); + expect(worker.syncJobsFor(message.id)).toHaveLength(2); + await createBuilder(harness, world.author.token) + .delete(`/channels/${world.announcement.id}/messages/${message.id}`) + .expect(204) + .execute(); + const removal = worker.syncJobsFor(message.id).find((job) => job.payload.mode === 'source_deleted'); + expect(removal?.payload).toEqual({ + channelId: world.announcement.id, + messageId: message.id, + mode: 'source_deleted', + }); + expect(removal?.options?.jobKey).toBe(`crosspost-sync:${message.id}:source_deleted`); + expect(removal?.options?.runAt).toBeUndefined(); + }); + + test('a stale moderator suppress-embeds edit keeps the published flag without using the cap', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'See https://example.com'); + const gate = stallFirstReadOf(message.id); + const pendingEdit = editMessage(harness, world.mod.token, world.announcement.id, message.id, { + flags: MessageFlags.SUPPRESS_EMBEDS, + }) + .expect(200) + .execute(); + await gate.reached; + await publish(harness, world.author.token, world.announcement.id, message.id); + gate.release(); + const edited = await pendingEdit; + expect(edited.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + expect((await readMessageRow(world.announcement.id, message.id))?.flags).toBe( + MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS, + ); + expect(await publishedEditBudgetRemaining(message.id)).toBe(3); + const syncs = worker.syncJobsFor(message.id); + expect(syncs).toHaveLength(1); + expectUpdateSyncJob(syncs[0], world.announcement.id, message.id); + }); + + test('a stale webhook token edit keeps the published flag and counts toward the cap', async () => { + const webhook = await createWebhook(harness, world.announcement.id, world.owner.token, 'Release Bot'); + const executed = await createBuilderWithoutAuth(harness) + .post(`/webhooks/${webhook.id}/${webhook.token}?wait=true`) + .body({content: 'Build 42 shipped'}) + .expect(200) + .execute(); + const gate = stallFirstReadOf(executed.id); + const pendingEdit = createBuilderWithoutAuth(harness) + .patch(`/webhooks/${webhook.id}/${webhook.token}/messages/${executed.id}`) + .body({content: 'Build 42 shipped (edited)', flags: MessageFlags.SUPPRESS_EMBEDS}) + .expect(200) + .execute(); + await gate.reached; + await publish(harness, world.mod.token, world.announcement.id, executed.id); + gate.release(); + await pendingEdit; + const row = await readMessageRow(world.announcement.id, executed.id); + expect(row?.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + expect(row?.content).toBe('Build 42 shipped (edited)'); + expect(await publishedEditBudgetRemaining(executed.id)).toBe(2); + expect(worker.syncJobsFor(executed.id)).toHaveLength(1); + }); + + test('a pin concurrent with a publish keeps both', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Pin and publish'); + await Promise.all([ + pinMessage(harness, world.owner.token, world.announcement.id, message.id), + publish(harness, world.author.token, world.announcement.id, message.id), + ]); + const row = await readMessageRow(world.announcement.id, message.id); + expect(row?.flags).toBe(MessageFlags.CROSSPOSTED); + expect(row?.pinnedTimestamp).not.toBeNull(); + }); +}); + +describe('Publishing races with a follower attached', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('a stale flag edit during a publish reaches the copy and a later delete marks it', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'Draft'}); + const gate = stallFirstReadOf(message.id); + const pendingEdit = editRequest(harness, world.a.owner.token, world.a.ann.id, message.id, { + content: 'Edited while publishing', + flags: MessageFlags.SUPPRESS_EMBEDS, + }) + .expect(200) + .execute(); + await gate.reached; + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + gate.release(); + await pendingEdit; + await world.worker.drain(); + let [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('Edited while publishing'); + expect(copy!.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SUPPRESS_EMBEDS); + await editRequest(harness, world.a.owner.token, world.a.ann.id, message.id, {content: 'Second edit'}) + .expect(200) + .execute(); + await world.worker.drain(); + [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('Second edit'); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain(); + [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED); + expect(copy!.content).toBe(CROSSPOST_SOURCE_DELETED_CONTENT); + expect(world.worker.deadLetters).toHaveLength(0); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/CrosspostModeration.test.ts b/fluxer_api/src/api/channel/tests/CrosspostModeration.test.ts new file mode 100644 index 000000000..cd7e8e3b8 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostModeration.test.ts @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import { + type ChannelID, + createAttachmentID, + createChannelID, + createGuildID, + createMessageID, + createUserID, + type MessageID, + type UserID, +} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import {CrosspostTaskNames} from '@app/api/channel/services/message/CrosspostPropagation'; +import { + type AnnouncementTargetGuild, + type AnnouncementWorld, + addGuildMember, + announcementWorld, + createAnnouncementTargetGuild, + follow, + publish, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {loadFixture, sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils'; +import {sendChannelMessage} from '@app/api/channel/tests/ChannelTestUtils'; +import {readMessageRow, writeMessageRow} from '@app/api/channel/tests/CrosspostTestUtils'; +import {NcmecRepository} from '@app/api/csam/NcmecRepository'; +import type {MessageEmbed} from '@app/api/database/types/MessageTypes'; +import type {EmbedService} from '@app/api/infrastructure/EmbedService'; +import { + getGatewayService, + getKVClient, + getSnowflakeService, + setInjectedWorkerService, +} from '@app/api/middleware/ServiceRegistry'; +import { + getCacheService, + getChannelRepository, + getNcmecSubmissionService, + getPurgeQueue, + getStorageService, +} from '@app/api/middleware/ServiceSingletons'; +import {urlBlocklistCache} from '@app/api/middleware/UrlBlocklistCache'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopLogger} from '@app/api/test/mocks/NoopLogger'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, executeWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds'; +import messageShred from '@app/api/worker/tasks/MessageShred'; +import { + clearWorkerDependencies, + setWorkerDependencies, + setWorkerDependenciesForTest, +} from '@app/api/worker/WorkerContext'; +import {initializeWorkerDependencies} from '@app/api/worker/WorkerDependencies'; +import {workerTasks} from '@app/api/worker/WorkerTaskRegistry'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {MessageFlags, MessageReferenceTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; +import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask'; +import type {WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test} from 'vitest'; + +const SERVER_BITS = MessageFlags.CROSSPOSTED | MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED; +const CROSSPOST_TASKS = new Set(Object.values(CrosspostTaskNames)); +const BLOCKED_DOMAIN = 'crosspost-blocked.example'; + +interface QueuedJob { + taskType: string; + payload: WorkerJobPayload; + options: WorkerJobOptions | undefined; +} + +class CrosspostQueueWorker implements IWorkerService { + readonly recorded: Array = []; + private queue: Array = []; + private seenKeys = new Set(); + private nextJobId = 1n; + + clear(): void { + this.recorded.length = 0; + this.queue = []; + this.seenKeys.clear(); + } + + async addJob( + taskType: string, + payload: TPayload, + options?: WorkerJobOptions, + ): Promise { + const jobKey = options?.jobKey; + if (jobKey && this.seenKeys.has(jobKey)) { + return 0n; + } + if (jobKey) { + this.seenKeys.add(jobKey); + } + const job = {taskType, payload, options}; + this.recorded.push(job); + if (CROSSPOST_TASKS.has(taskType)) { + this.queue.push(job); + } + return this.nextJobId++; + } + + async cancelJob(_jobId: bigint): Promise { + return false; + } + + async retryDeadLetterJob(_jobId: bigint): Promise { + return false; + } + + syncJobs(): Array { + return this.recorded.filter((job) => job.taskType === CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE); + } + + async drain(): Promise { + const handlers = workerTasks as Partial>; + const attempts = new Map(); + let processed = 0; + while (this.queue.length > 0) { + processed += 1; + if (processed > 5_000) { + throw new Error('crosspost queue did not settle'); + } + const job = this.queue.shift()!; + const handler = handlers[job.taskType]; + if (!handler) { + throw new Error(`no worker handler registered for ${job.taskType}`); + } + const helpers: WorkerTaskHelpers = { + logger: new NoopLogger(), + jobId: 0n, + addJob: (taskType, payload, options) => this.addJob(taskType, payload, options), + reportProgress: async () => {}, + shouldCancel: async () => false, + setContextLink: async () => {}, + }; + try { + await handler(job.payload, helpers); + } catch (error) { + const count = (attempts.get(job) ?? 0) + 1; + attempts.set(job, count); + if (count >= 3) { + throw error; + } + this.queue.push(job); + } + } + } +} + +interface FabricatedCopy { + copyId: string; + attachment: {id: string; filename: string} | null; +} + +async function fabricateCopy(params: { + harness: ApiTestHarness; + owner: TestAccount; + target: string; + world: AnnouncementWorld; + sourceId: string; + extraFlags?: number; + withImage?: boolean; +}): Promise { + const webhook = await createWebhook(params.harness, params.target, params.owner.token, 'Follower'); + const {json: message} = await executeWebhook( + params.harness, + webhook.id, + webhook.token, + {content: 'copied update', wait: true}, + 200, + ); + const sourceAttachments = params.withImage + ? (await readMessageRow(params.world.a.ann.id, params.sourceId))!.attachments + : []; + const row = await readMessageRow(params.target, message!.id); + await writeMessageRow(row!, { + flags: MessageFlags.IS_CROSSPOST | (params.extraFlags ?? 0), + attachments: + sourceAttachments.length > 0 ? sourceAttachments.map((attachment) => attachment.toMessageAttachment()) : null, + message_reference: { + channel_id: createChannelID(BigInt(params.world.a.ann.id)), + guild_id: createGuildID(BigInt(params.world.a.guild.id)), + message_id: createMessageID(BigInt(params.sourceId)), + type: MessageReferenceTypes.DEFAULT, + }, + }); + const attachment = sourceAttachments[0]; + return { + copyId: message!.id, + attachment: attachment ? {id: attachment.id.toString(), filename: attachment.filename} : null, + }; +} + +async function createAdmin(harness: ApiTestHarness): Promise { + return setUserACLs(harness, await createTestAccount(harness), [ + AdminACLs.AUTHENTICATE, + AdminACLs.MESSAGE_DELETE, + AdminACLs.MESSAGE_SHRED, + AdminACLs.CSAM_SUBMIT_NCMEC, + AdminACLs.USER_DELETE, + AdminACLs.ARCHIVE_TRIGGER_USER, + ]); +} + +async function adminDeleteMessage(harness: ApiTestHarness, admin: TestAccount, channelId: string, messageId: string) { + await createBuilder(harness, admin.token) + .delete(`/admin/channels/${channelId}/messages/${messageId}`) + .expect(HTTP_STATUS.OK) + .execute(); +} + +async function deleteMessageSilently(channelId: string, messageId: string, fallbackUserId: string): Promise { + const service = getNcmecSubmissionService() as unknown as { + deleteMessageSilently(channelId: ChannelID, messageId: MessageID, fallbackUserId: UserID): Promise; + }; + await service.deleteMessageSilently( + createChannelID(BigInt(channelId)), + createMessageID(BigInt(messageId)), + createUserID(BigInt(fallbackUserId)), + ); +} + +function blockedEmbedService(): EmbedService { + return { + processUrlWithCachePolicy: async (url: string) => ({ + embeds: [{toMessageEmbed: (): MessageEmbed => ({type: 'link', url}) as MessageEmbed}], + cacheTtlSeconds: 0, + }), + cacheEmbeds: async () => {}, + } as unknown as EmbedService; +} + +function helpers(): WorkerTaskHelpers { + return { + logger: new NoopLogger(), + jobId: 1n, + addJob: async () => 0n, + reportProgress: async () => {}, + shouldCancel: async () => false, + setContextLink: async () => {}, + }; +} + +async function listCopies(channelId: string, sourceId: string): Promise> { + const messages = await new ChannelRepository().listMessages(createChannelID(BigInt(channelId)), undefined, 100); + return messages + .filter( + (message) => + (message.flags & MessageFlags.IS_CROSSPOST) !== 0 && message.reference?.messageId?.toString() === sourceId, + ) + .map((message) => message.id.toString()); +} + +describe('Crosspost moderation', () => { + let harness: ApiTestHarness; + let worker: CrosspostQueueWorker; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + worker = new CrosspostQueueWorker(); + setInjectedWorkerService(worker); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + worker.clear(); + world = await announcementWorld(harness); + }); + + afterEach(() => { + clearWorkerDependencies(); + urlBlocklistCache.removeDomain(BLOCKED_DOMAIN); + }); + + afterAll(async () => { + setInjectedWorkerService(new NoopWorkerService()); + await harness?.shutdown(); + }); + + function deletedKeys(): Array { + return harness.storageService.getDeletedObjects().map((entry) => entry.key); + } + + describe('takedown hooks', () => { + test('an admin delete of a copy enqueues a purge of the whole family', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + const {copyId} = await fabricateCopy({ + harness, + owner: world.b.owner, + target: world.b.t1.id, + world, + sourceId: source.id, + }); + const admin = await createAdmin(harness); + await adminDeleteMessage(harness, admin, world.b.t1.id, copyId); + expect(await readMessageRow(world.b.t1.id, copyId)).toBeNull(); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({ + channelId: world.a.ann.id, + messageId: source.id, + mode: 'purge', + deleteSource: true, + }); + expect(jobs[0]!.options?.jobKey).toBe(`crosspost-sync:${source.id}:purge`); + expect(jobs[0]!.options?.skipLedger).toBe(true); + }); + + test('an admin delete of a published source purges its copies and leaves others alone', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + await publish(harness, world.a.member.token, world.a.ann.id, source.id); + const plain = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'not published'); + const admin = await createAdmin(harness); + await adminDeleteMessage(harness, admin, world.a.ann.id, plain.id); + expect(worker.syncJobs()).toHaveLength(0); + await adminDeleteMessage(harness, admin, world.a.ann.id, source.id); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'}); + }); + + test('an NCMEC report on a copy records the source author and its takedown purges the family', async () => { + const {json: source} = await sendMessageWithAttachments( + harness, + world.a.member.token, + world.a.ann.id, + {content: 'update', attachments: [{id: 0, filename: 'yeah.png'}]}, + [{index: 0, filename: 'yeah.png', data: loadFixture('yeah.png')}], + ); + const {copyId, attachment} = await fabricateCopy({ + harness, + owner: world.b.owner, + target: world.b.t1.id, + world, + sourceId: source.id, + withImage: true, + }); + expect(attachment).not.toBeNull(); + const admin = await createAdmin(harness); + await createBuilder(harness, admin.token) + .post('/admin/messages/ncmec-reports') + .body({ + channel_id: world.b.t1.id, + message_id: copyId, + attachment_id: attachment!.id, + filename: attachment!.filename, + reporter_full_name: 'Crosspost Reporter', + confirmed_viewed: true, + }) + .expect(HTTP_STATUS.OK) + .execute(); + const submission = await new NcmecRepository().getAttachmentSubmission( + createAttachmentID(BigInt(attachment!.id)), + ); + expect(submission?.user_id?.toString()).toBe(world.a.member.userId); + expect(submission?.message_id.toString()).toBe(copyId); + expect(submission?.channel_id.toString()).toBe(world.b.t1.id); + worker.clear(); + const deletedBefore = deletedKeys().length; + await deleteMessageSilently(world.b.t1.id, copyId, world.a.member.userId); + expect(await readMessageRow(world.b.t1.id, copyId)).toBeNull(); + expect(deletedKeys().slice(deletedBefore)).toEqual([]); + expect( + harness.storageService.hasObject( + Config.s3.buckets.cdn, + `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`, + ), + ).toBe(true); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({ + channelId: world.a.ann.id, + messageId: source.id, + mode: 'purge', + deleteSource: true, + }); + }); + + test('an NCMEC takedown of a published source purges its copies', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + await publish(harness, world.a.member.token, world.a.ann.id, source.id); + worker.clear(); + await deleteMessageSilently(world.a.ann.id, source.id, world.a.member.userId); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'}); + }); + + test('a message shred of a published source purges its copies', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + await publish(harness, world.a.member.token, world.a.ann.id, source.id); + const plain = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'plain'); + worker.clear(); + setWorkerDependenciesForTest({ + kvClient: getKVClient(), + channelRepository: getChannelRepository(), + gatewayService: getGatewayService(), + storageService: getStorageService(), + purgeQueue: getPurgeQueue(), + workerService: worker, + }); + await messageShred( + { + job_id: 'crosspost-shred', + admin_user_id: '1', + target_user_id: world.a.member.userId, + entries: [ + {channel_id: world.a.ann.id, message_id: source.id}, + {channel_id: world.a.ann.id, message_id: plain.id}, + ], + }, + helpers(), + ); + expect(await readMessageRow(world.a.ann.id, source.id)).toBeNull(); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'}); + }); + + test('a blocked unfurl on a published source deletes it and purges its copies', async () => { + const source = await sendChannelMessage( + harness, + world.a.member.token, + world.a.ann.id, + `see https://${BLOCKED_DOMAIN}/page`, + ); + await publish(harness, world.a.member.token, world.a.ann.id, source.id); + urlBlocklistCache.addDomain(BLOCKED_DOMAIN); + worker.clear(); + setWorkerDependenciesForTest({ + channelRepository: getChannelRepository(), + gatewayService: getGatewayService(), + embedService: blockedEmbedService(), + cacheService: getCacheService(), + workerService: worker, + }); + await extractEmbeds( + {channelId: world.a.ann.id, messageId: source.id, guildId: world.a.guild.id, nsfwMode: 'allow'}, + helpers(), + ); + expect(await readMessageRow(world.a.ann.id, source.id)).toBeNull(); + const jobs = worker.syncJobs(); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'}); + }); + }); + + describe('forwarding', () => { + function forward(token: string, targetChannelId: string, channelId: string, guildId: string, messageId: string) { + return createBuilder(harness, token) + .post(`/channels/${targetChannelId}/messages`) + .body({ + message_reference: { + type: MessageReferenceTypes.FORWARD, + message_id: messageId, + channel_id: channelId, + guild_id: guildId, + }, + }); + } + + test('forwarding a published source carries none of the server bits', async () => { + const source = await sendChannelMessage(harness, world.b.owner.token, world.a.ann.id, 'forward me'); + await publish(harness, world.b.owner.token, world.a.ann.id, source.id); + const forwarded = await forward(world.b.owner.token, world.b.t2.id, world.a.ann.id, world.a.guild.id, source.id) + .expect(HTTP_STATUS.OK) + .execute(); + expect(forwarded.message_snapshots?.[0]?.flags ?? 0).toBe(0); + }); + + test('forwarding a copy carries none of the server bits', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + const {copyId} = await fabricateCopy({ + harness, + owner: world.b.owner, + target: world.b.t1.id, + world, + sourceId: source.id, + extraFlags: MessageFlags.SUPPRESS_NOTIFICATIONS, + }); + const forwarded = await forward(world.b.owner.token, world.b.t2.id, world.b.t1.id, world.b.guild.id, copyId) + .expect(HTTP_STATUS.OK) + .execute(); + const flags = forwarded.message_snapshots?.[0]?.flags ?? 0; + expect(flags & SERVER_BITS).toBe(0); + expect(flags & MessageFlags.SUPPRESS_NOTIFICATIONS).toBe(MessageFlags.SUPPRESS_NOTIFICATIONS); + }); + + test('forwarding a source-deleted copy is refused', async () => { + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update'); + const {copyId} = await fabricateCopy({ + harness, + owner: world.b.owner, + target: world.b.t1.id, + world, + sourceId: source.id, + extraFlags: MessageFlags.SOURCE_MESSAGE_DELETED, + }); + await forward(world.b.owner.token, world.b.t2.id, world.b.t1.id, world.b.guild.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + }); + }); + + describe('takedown cascade after delivery', () => { + let c: AnnouncementTargetGuild; + + beforeEach(async () => { + const deps = await initializeWorkerDependencies(getSnowflakeService()); + deps.workerService = worker; + setWorkerDependencies(deps); + c = await createAnnouncementTargetGuild(harness, 'Announcement Target C'); + await addGuildMember(harness, world.a.owner, world.a.guild, c.owner); + await follow(harness, world.b.owner.token, world.a.ann.id, world.b.t1.id); + await follow(harness, world.b.owner.token, world.a.ann.id, world.b.t2.id); + await follow(harness, c.owner.token, world.a.ann.id, c.t1.id); + }); + + async function publishAndDeliver(content: string, withImage = false): Promise { + let sourceId: string; + if (withImage) { + const filename = 'yeah.png'; + const {json} = await sendMessageWithAttachments( + harness, + world.a.member.token, + world.a.ann.id, + {content, attachments: [{id: 0, filename}]}, + [{index: 0, filename, data: loadFixture(filename)}], + ); + sourceId = json.id; + } else { + sourceId = (await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, content)).id; + } + await publish(harness, world.a.member.token, world.a.ann.id, sourceId); + await worker.drain(); + return sourceId; + } + + async function sourceAttachmentKey(sourceId: string): Promise { + const [attachment] = (await readMessageRow(world.a.ann.id, sourceId))!.attachments; + const key = `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`; + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(true); + return key; + } + + function expectNoTargetObjectsDeleted(): void { + for (const channelId of [world.b.t1.id, world.b.t2.id, c.t1.id]) { + expect(deletedKeys().some((key) => key.startsWith(`attachments/${channelId}/`))).toBe(false); + } + } + + async function expectFamilyGone(sourceId: string): Promise { + expect(await readMessageRow(world.a.ann.id, sourceId)).toBeNull(); + expect(await listCopies(world.b.t1.id, sourceId)).toEqual([]); + expect(await listCopies(world.b.t2.id, sourceId)).toEqual([]); + expect(await listCopies(c.t1.id, sourceId)).toEqual([]); + } + + test('an admin delete of one copy removes the source and every sibling copy', async () => { + const sourceId = await publishAndDeliver('family update', true); + const sourceKey = await sourceAttachmentKey(sourceId); + const [copyId] = await listCopies(world.b.t1.id, sourceId); + expect(copyId).toBeDefined(); + expect(await listCopies(world.b.t2.id, sourceId)).toHaveLength(1); + expect(await listCopies(c.t1.id, sourceId)).toHaveLength(1); + const admin = await createAdmin(harness); + await adminDeleteMessage(harness, admin, world.b.t1.id, copyId!); + expect(deletedKeys()).not.toContain(sourceKey); + await worker.drain(); + await expectFamilyGone(sourceId); + expect(deletedKeys().filter((key) => key === sourceKey)).toHaveLength(1); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(false); + expectNoTargetObjectsDeleted(); + }); + + test('an NCMEC takedown of a delivered copy records the source author and removes the family', async () => { + const sourceId = await publishAndDeliver('image update', true); + const sourceKey = await sourceAttachmentKey(sourceId); + const [copyId] = await listCopies(c.t1.id, sourceId); + const copy = await readMessageRow(c.t1.id, copyId!); + const attachment = copy!.attachments[0]!; + expect(`attachments/${world.a.ann.id}/${attachment.id}/${attachment.filename}`).toBe(sourceKey); + const admin = await createAdmin(harness); + await createBuilder(harness, admin.token) + .post('/admin/messages/ncmec-reports') + .body({ + channel_id: c.t1.id, + message_id: copyId, + attachment_id: attachment.id.toString(), + filename: attachment.filename, + reporter_full_name: 'Crosspost Reporter', + confirmed_viewed: true, + }) + .expect(HTTP_STATUS.OK) + .execute(); + const submission = await new NcmecRepository().getAttachmentSubmission(attachment.id); + expect(submission?.user_id?.toString()).toBe(world.a.member.userId); + await deleteMessageSilently(c.t1.id, copyId!, world.a.member.userId); + await worker.drain(); + await expectFamilyGone(sourceId); + expect(deletedKeys().filter((key) => key === sourceKey)).toHaveLength(1); + expectNoTargetObjectsDeleted(); + }); + + test('a blocked unfurl of a published source deletes every copy', async () => { + const sourceId = await publishAndDeliver(`see https://${BLOCKED_DOMAIN}/page`); + expect(await listCopies(c.t1.id, sourceId)).toHaveLength(1); + urlBlocklistCache.addDomain(BLOCKED_DOMAIN); + const deps = await initializeWorkerDependencies(getSnowflakeService()); + deps.workerService = worker; + deps.embedService = blockedEmbedService(); + setWorkerDependencies(deps); + await extractEmbeds( + {channelId: world.a.ann.id, messageId: sourceId, guildId: world.a.guild.id, nsfwMode: 'allow'}, + helpers(), + ); + await worker.drain(); + await expectFamilyGone(sourceId); + }); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts b/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts new file mode 100644 index 000000000..fb260ed5d --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts @@ -0,0 +1,854 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; +import {setUserACLs} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createMessageID, createUserID} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import {crosspostSyncBucket, enqueueCrosspostSync} from '@app/api/channel/services/message/CrosspostPropagation'; +import { + addGuildMember, + createAnnouncementTargetGuild, + disableCrosspostWorker, + enableCrosspostWorker, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {addGuildFeature, createTestChannelService} from '@app/api/channel/tests/CrosspostTestUtils'; +import { + copiesOf, + deleteMessageRequest, + editRequest, + type FanoutWorld, + followInto, + listCopies, + mappingRow, + mappingRows, + patchChannel, + postAndPublish, + publishAndDrain, + readRow, + sendMessage, + sendWithImage, + setupFanoutWorld, + sourceIndex, + workerHelpers, + writeRow, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import type {EmbedService} from '@app/api/infrastructure/EmbedService'; +import type {IAssetDeletionQueue} from '@app/api/infrastructure/IAssetDeletionQueue'; +import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRepository'; +import {getKVClient} from '@app/api/middleware/ServiceRegistry'; +import type {Embed} from '@app/api/models/Embed'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {getExpiryBucket} from '@app/api/utils/AttachmentDecay'; +import {createWebhook, executeWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime'; +import {processExpiredAttachments} from '@app/api/worker/tasks/ExpireAttachments'; +import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds'; +import messageShred from '@app/api/worker/tasks/MessageShred'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import { + CROSSPOST_SOURCE_DELETED_CONTENT, + CROSSPOST_SYNC_COALESCE_MS, +} from '@fluxer/constants/src/AnnouncementConstants'; +import { + ChannelTypes, + MessageAttachmentFlags, + MessageFlags, + MessageReferenceTypes, +} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +const DELETED_COPY_FLAGS = MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED; + +async function awaitFreshBucket(): Promise { + const intoBucket = Date.now() % CROSSPOST_SYNC_COALESCE_MS; + if (intoBucket > CROSSPOST_SYNC_COALESCE_MS - 2000) { + await new Promise((resolve) => setTimeout(resolve, CROSSPOST_SYNC_COALESCE_MS - intoBucket + 50)); + } +} + +describe('Crosspost propagation', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function onlyCopy(channelId: string, sourceId: string, token = world.b.owner.token): Promise { + const copies = await copiesOf(harness, token, channelId, sourceId); + expect(copies).toHaveLength(1); + return copies[0]!; + } + + async function edit(messageId: string, body: Record, token = world.a.owner.token) { + await editRequest(harness, token, world.a.ann.id, messageId, body).expect(200).execute(); + } + + async function expectDeletedCopy(channelId: string, copyId: string): Promise { + const row = (await readRow(channelId, copyId))!; + expect(row.flags).toBe(DELETED_COPY_FLAGS); + expect(row.content).toBe(CROSSPOST_SOURCE_DELETED_CONTENT); + expect(row.attachments).toHaveLength(0); + expect(row.embeds).toHaveLength(0); + expect(row.stickers).toHaveLength(0); + expect(row.editedTimestamp).not.toBeNull(); + } + + test('a content edit updates every copy through one coalesced sync job', async () => { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + const source = await postAndPublish(harness, world, {content: 'draft'}); + const copyT1 = await onlyCopy(world.b.t1.id, source.id); + const copyT2 = await onlyCopy(world.b.t2.id, source.id); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const before = crosspostSyncBucket(Date.now()); + await edit(source.id, {content: 'final'}); + const after = crosspostSyncBucket(Date.now()); + const [job] = world.worker + .byTask('syncCrosspostedMessage') + .filter((entry) => (entry.payload as {messageId: string}).messageId === source.id); + const bucket = Number(job!.jobKey!.split(':').at(-1)); + expect(job!.jobKey).toBe(`crosspost-sync:${source.id}:update:${bucket}`); + expect(bucket).toBeGreaterThanOrEqual(before); + expect(bucket).toBeLessThanOrEqual(after); + expect(job!.runAt!.getTime()).toBe((bucket + 1) * CROSSPOST_SYNC_COALESCE_MS + 1000); + expect(job!.options?.skipLedger).toBe(true); + await world.worker.drain(); + for (const [channelId, copyId] of [ + [world.b.t1.id, copyT1.id], + [world.b.t2.id, copyT2.id], + ] as const) { + const updated = await onlyCopy(channelId, source.id); + expect(updated.content).toBe('final'); + expect(updated.edited_timestamp).toBeTruthy(); + const updates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter( + (params) => + params.event === 'MESSAGE_UPDATE' && + params.guildId.toString() === world.b.guild.id && + (params.data as {id: string}).id === copyId, + ); + expect(updates).toHaveLength(1); + } + }); + + test('two edits in one bucket run a single sync and the copy ends with the second', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'first'}); + await awaitFreshBucket(); + await edit(source.id, {content: 'second'}); + await edit(source.id, {content: 'third'}); + const syncJobs = world.worker + .byTask('syncCrosspostedMessage') + .filter((entry) => (entry.payload as {messageId: string}).messageId === source.id); + expect(syncJobs).toHaveLength(1); + expect(world.worker.deduped.filter((entry) => entry.taskType === 'syncCrosspostedMessage')).toHaveLength(1); + await world.worker.drain(); + expect((await onlyCopy(world.b.t1.id, source.id)).content).toBe('third'); + }); + + test('embed edits and suppress-embeds toggles propagate, including flag-only edits', async () => { + await followInto(harness, world, world.b.t1.id); + const embedded = await postAndPublish(harness, world, {content: 'card', embeds: [{title: 'Old title'}]}); + await edit(embedded.id, {embeds: [{title: 'New title', description: 'Details'}]}); + await world.worker.drain(); + let copyRow = (await readRow(world.b.t1.id, (await onlyCopy(world.b.t1.id, embedded.id)).id))!; + expect(copyRow.embeds.map((embed) => [embed.title, embed.description])).toEqual([['New title', 'Details']]); + await edit(embedded.id, {flags: MessageFlags.SUPPRESS_EMBEDS}); + await world.worker.drain(); + copyRow = (await readRow(world.b.t1.id, copyRow.id.toString()))!; + expect(copyRow.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SUPPRESS_EMBEDS); + await edit(embedded.id, {flags: 0}, world.a.moderator.token); + await world.worker.drain(); + copyRow = (await readRow(world.b.t1.id, copyRow.id.toString()))!; + expect(copyRow.flags).toBe(MessageFlags.IS_CROSSPOST); + const plain = await postAndPublish(harness, world, {content: 'plain'}); + await awaitFreshBucket(); + await edit(plain.id, {content: 'plain edited'}); + await world.worker.drain(); + const plainCopy = await onlyCopy(world.b.t1.id, plain.id); + const editedAt = (await readRow(world.a.ann.id, plain.id))!.editedTimestamp; + await edit(plain.id, {flags: MessageFlags.SUPPRESS_EMBEDS}); + expect((await readRow(world.a.ann.id, plain.id))!.editedTimestamp).toEqual(editedAt); + await world.worker.drain(); + const plainRow = (await readRow(world.b.t1.id, plainCopy.id))!; + expect(plainRow.content).toBe('plain edited'); + expect(plainRow.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SUPPRESS_EMBEDS); + }); + + test('removing one attachment by an edit drops it from the copy and deletes no copy objects', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'two files'}, [ + 'keep.png', + 'drop.gif', + ]); + await publishAndDrain(harness, world, source.id); + const copy = await onlyCopy(world.b.t1.id, source.id); + const before = (await readRow(world.b.t1.id, copy.id))!.attachments; + expect(before).toHaveLength(2); + const sourceRow = (await readRow(world.a.ann.id, source.id))!; + await edit(source.id, {attachments: [{id: sourceRow.attachments[0]!.id.toString()}]}); + await world.worker.drain(); + expect(before.map((attachment) => attachment.id)).toEqual(sourceRow.attachments.map((attachment) => attachment.id)); + const after = (await readRow(world.b.t1.id, copy.id))!.attachments; + expect(after.map((attachment) => attachment.id)).toEqual([before[0]!.id]); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect(deleted.some((key) => key.startsWith(`attachments/${world.b.t1.id}/`))).toBe(false); + expect(deleted).not.toContain(`attachments/${world.a.ann.id}/${before[0]!.id}/${before[0]!.filename}`); + const [rendered] = (await listCopies(harness, world.b.owner.token, world.b.t1.id)).filter( + (message) => message.id === copy.id, + ); + expect(rendered!.attachments?.map((attachment) => attachment.url?.split('?')[0])).toEqual([ + expect.stringContaining(`/attachments/${world.a.ann.id}/${before[0]!.id}/${before[0]!.filename}`), + ]); + }); + + test('forwarding a copy clones the shared file into the forward channel', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'forward the file'}); + await publishAndDrain(harness, world, source.id); + const copy = await onlyCopy(world.b.t1.id, source.id); + const [attachment] = (await readRow(world.a.ann.id, source.id))!.attachments; + const sourceKey = `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`; + const copiedBefore = harness.storageService.getCopiedObjects().length; + const forwarded = await createBuilder(harness, world.b.owner.token) + .post(`/channels/${world.b.t2.id}/messages`) + .body({ + message_reference: { + type: MessageReferenceTypes.FORWARD, + message_id: copy.id, + channel_id: world.b.t1.id, + guild_id: world.b.guild.id, + }, + }) + .expect(200) + .execute(); + const clones = harness.storageService.getCopiedObjects().slice(copiedBefore); + expect(clones).toHaveLength(1); + expect(clones[0]!.sourceKey).toBe(sourceKey); + expect(clones[0]!.destinationKey.startsWith(`attachments/${world.b.t2.id}/`)).toBe(true); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, source.id); + await world.worker.drain(); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect(deleted).toContain(sourceKey); + expect(deleted).not.toContain(clones[0]!.destinationKey); + expect(forwarded.message_snapshots?.[0]?.attachments).toHaveLength(1); + }); + + test('attachment metadata edits reach the copy under the same attachment id', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'alt text'}); + await publishAndDrain(harness, world, source.id); + const copy = await onlyCopy(world.b.t1.id, source.id); + const sourceAttachment = (await readRow(world.a.ann.id, source.id))!.attachments[0]!; + const attachmentId = (await readRow(world.b.t1.id, copy.id))!.attachments[0]!.id; + expect(attachmentId).toBe(sourceAttachment.id); + const copiedBefore = harness.storageService.getCopiedObjects().length; + await edit(source.id, {attachments: [{id: sourceAttachment.id.toString(), description: 'A cat'}]}); + await world.worker.drain(); + let copied = (await readRow(world.b.t1.id, copy.id))!.attachments[0]!; + expect(copied.id).toBe(attachmentId); + expect(copied.description).toBe('A cat'); + const edited = (await readRow(world.a.ann.id, source.id))!; + await writeRow(edited, { + attachments: edited.attachments.map((attachment) => ({ + ...attachment.toMessageAttachment(), + flags: attachment.flags | MessageAttachmentFlags.IS_SPOILER, + })), + }); + await enqueueCrosspostSync(world.worker, { + channelId: createChannelID(BigInt(world.a.ann.id)), + messageId: createMessageID(BigInt(source.id)), + mode: 'update', + }); + await world.worker.drain(); + copied = (await readRow(world.b.t1.id, copy.id))!.attachments[0]!; + expect(copied.id).toBe(attachmentId); + expect(copied.flags & MessageAttachmentFlags.IS_SPOILER).toBe(MessageAttachmentFlags.IS_SPOILER); + expect(harness.storageService.getCopiedObjects()).toHaveLength(copiedBefore); + }); + + test('a late unfurl on a published source propagates its embeds', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'read https://example.com/post'}); + const embedService = { + processUrlWithCachePolicy: async (url: string) => ({ + embeds: [ + { + toMessageEmbed: () => ({ + type: 'link', + url, + title: 'Example post', + description: null, + timestamp: null, + color: null, + author: null, + provider: null, + thumbnail: null, + image: null, + video: null, + footer: null, + fields: null, + nsfw: null, + }), + } as unknown as Embed, + ], + cacheTtlSeconds: 0, + }), + cacheEmbeds: async () => {}, + } as unknown as EmbedService; + disableCrosspostWorker(); + world.worker = enableCrosspostWorker({}, {embedService}); + await extractEmbeds( + {channelId: world.a.ann.id, messageId: source.id, guildId: world.a.guild.id, nsfwMode: 'allow'}, + workerHelpers(world.worker), + ); + expect(world.worker.byTask('syncCrosspostedMessage')).toHaveLength(1); + await world.worker.drain(); + const copyRow = (await readRow(world.b.t1.id, (await onlyCopy(world.b.t1.id, source.id)).id))!; + expect(copyRow.embeds.map((embed) => embed.title)).toEqual(['Example post']); + }); + + test('a sync with nothing new writes nothing', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'steady'}); + const copy = await onlyCopy(world.b.t1.id, source.id); + const before = await readRow(world.b.t1.id, copy.id); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await enqueueCrosspostSync(world.worker, { + channelId: createChannelID(BigInt(world.a.ann.id)), + messageId: createMessageID(BigInt(source.id)), + mode: 'update', + }); + await world.worker.drain(); + expect((await readRow(world.b.t1.id, copy.id))?.version).toBe(before?.version); + expect(dispatchSpy.mock.calls.filter(([params]) => params.event === 'MESSAGE_UPDATE')).toHaveLength(0); + }); + + test('deleting the source marks every copy and cleans up the mapping and index rows', async () => { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, { + content: 'goodbye', + embeds: [{title: 'Card'}], + }); + await publishAndDrain(harness, world, source.id); + const copyT1 = await onlyCopy(world.b.t1.id, source.id); + const copyT2 = await onlyCopy(world.b.t2.id, source.id); + const sourceKeys = (await readRow(world.a.ann.id, source.id))!.attachments.map( + (attachment) => `attachments/${world.a.ann.id}/${attachment.id}/${attachment.filename}`, + ); + expect(sourceKeys).toHaveLength(1); + for (const [channelId, copyId] of [ + [world.b.t1.id, copyT1.id], + [world.b.t2.id, copyT2.id], + ] as const) { + const copyKeys = (await readRow(channelId, copyId))!.attachments.map( + (attachment) => `attachments/${world.a.ann.id}/${attachment.id}/${attachment.filename}`, + ); + expect(copyKeys).toEqual(sourceKeys); + } + expect(await sourceIndex(world.a.ann.id)).toContain(source.id); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, source.id); + await world.worker.drain(); + await expectDeletedCopy(world.b.t1.id, copyT1.id); + await expectDeletedCopy(world.b.t2.id, copyT2.id); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + for (const key of sourceKeys) { + expect(deleted.filter((entry) => entry === key)).toHaveLength(1); + } + expect(deleted.some((key) => key.startsWith(`attachments/${world.b.t1.id}/`))).toBe(false); + expect(deleted.some((key) => key.startsWith(`attachments/${world.b.t2.id}/`))).toBe(false); + expect(await mappingRows(source.id)).toHaveLength(0); + expect(await sourceIndex(world.a.ann.id)).not.toContain(source.id); + }); + + describe('other delete paths mark copies too', () => { + test('bulk delete', async () => { + await followInto(harness, world, world.b.t1.id); + const first = await postAndPublish(harness, world, {content: 'one'}); + const second = await postAndPublish(harness, world, {content: 'two'}); + await createBuilder(harness, world.a.owner.token) + .post(`/channels/${world.a.ann.id}/messages/bulk-delete`) + .body({message_ids: [first.id, second.id]}) + .expect(204) + .execute(); + await world.worker.drain(); + for (const source of [first, second]) { + await expectDeletedCopy(world.b.t1.id, (await onlyCopy(world.b.t1.id, source.id)).id); + } + }); + + test('a ban that deletes recent messages', async () => { + disableCrosspostWorker(); + world.worker = enableCrosspostWorker({deleteUserMessagesInGuildByTime}); + await followInto(harness, world, world.b.t1.id); + const source = await sendMessage(harness, world.a.member.token, world.a.ann.id, {content: 'soon banned'}); + await publishAndDrain(harness, world, source.id, world.a.member.token); + const copy = await onlyCopy(world.b.t1.id, source.id); + await createBuilder(harness, world.a.owner.token) + .put(`/guilds/${world.a.guild.id}/bans/${world.a.member.userId}`) + .body({delete_message_seconds: 3600}) + .expect(204) + .execute(); + await world.worker.drain(); + expect(await readRow(world.a.ann.id, source.id)).toBeNull(); + await expectDeletedCopy(world.b.t1.id, copy.id); + }); + + test('a self bulk delete', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendMessage(harness, world.a.member.token, world.a.ann.id, {content: 'mine'}); + await publishAndDrain(harness, world, source.id, world.a.member.token); + const copy = await onlyCopy(world.b.t1.id, source.id); + await createTestChannelService().userMessageDeletion.deleteUserMessagesInScope( + createUserID(BigInt(world.a.member.userId)), + {channelIds: [createChannelID(BigInt(world.a.ann.id))]}, + ); + await world.worker.drain(); + expect(await readRow(world.a.ann.id, source.id)).toBeNull(); + await expectDeletedCopy(world.b.t1.id, copy.id); + }); + + test('a webhook token delete of a published webhook message', async () => { + await followInto(harness, world, world.b.t1.id); + const incoming = await createWebhook(harness, world.a.ann.id, world.a.owner.token, 'Releases'); + const {json} = await executeWebhook(harness, incoming.id, incoming.token, {content: 'bot post', wait: true}, 200); + await publishAndDrain(harness, world, json!.id); + const copy = await onlyCopy(world.b.t1.id, json!.id); + await createBuilderWithoutAuth(harness) + .delete(`/webhooks/${incoming.id}/${incoming.token}/messages/${json!.id}`) + .expect(204) + .execute(); + await world.worker.drain(); + await expectDeletedCopy(world.b.t1.id, copy.id); + }); + }); + + describe('takedowns delete copies', () => { + test('an admin delete of the source', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'illegal'}); + const copy = await onlyCopy(world.b.t1.id, source.id); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const admin = await setUserACLs(harness, world.a.member, [AdminACLs.AUTHENTICATE, AdminACLs.MESSAGE_DELETE]); + await createBuilder(harness, admin.token) + .delete(`/admin/channels/${world.a.ann.id}/messages/${source.id}`) + .expect(200) + .execute(); + await world.worker.drain(); + expect(await readRow(world.b.t1.id, copy.id)).toBeNull(); + expect(await listCopies(harness, world.b.owner.token, world.b.t1.id)).toHaveLength(0); + expect( + dispatchSpy.mock.calls.some( + ([params]) => + params.event === 'MESSAGE_DELETE' && + params.guildId.toString() === world.b.guild.id && + (params.data as {id: string}).id === copy.id, + ), + ).toBe(true); + expect(await mappingRows(source.id)).toHaveLength(0); + }); + + test('a message shred of the source', async () => { + disableCrosspostWorker(); + world.worker = enableCrosspostWorker({}, {kvClient: getKVClient()}); + await followInto(harness, world, world.b.t1.id); + const source = await sendMessage(harness, world.a.member.token, world.a.ann.id, {content: 'shred me'}); + await publishAndDrain(harness, world, source.id, world.a.member.token); + const copy = await onlyCopy(world.b.t1.id, source.id); + await messageShred( + { + job_id: 'crosspost-propagation-shred', + admin_user_id: '1', + target_user_id: world.a.member.userId, + entries: [{channel_id: world.a.ann.id, message_id: source.id}], + }, + workerHelpers(world.worker), + ); + await world.worker.drain(); + expect(await readRow(world.a.ann.id, source.id)).toBeNull(); + expect(await readRow(world.b.t1.id, copy.id)).toBeNull(); + }); + }); + + describe('removing a copy keeps the source files', () => { + async function publishedCopyWithFile(): Promise<{sourceId: string; copyId: string; sourceKey: string}> { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'shared file'}); + await publishAndDrain(harness, world, source.id); + const [attachment] = (await readRow(world.a.ann.id, source.id))!.attachments; + const sourceKey = `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`; + const copy = await onlyCopy(world.b.t1.id, source.id); + expect((await readRow(world.b.t1.id, copy.id))!.attachments.map((entry) => entry.id)).toEqual([attachment!.id]); + return {sourceId: source.id, copyId: copy.id, sourceKey}; + } + + async function expectSourceFileKept(sourceId: string, sourceKey: string): Promise { + await world.worker.drain(); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect(deleted).not.toContain(sourceKey); + expect(deleted.some((key) => key.startsWith('attachments/'))).toBe(false); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(true); + expect((await readRow(world.a.ann.id, sourceId))!.attachments).toHaveLength(1); + const sibling = await onlyCopy(world.b.t2.id, sourceId); + expect(sibling.attachments?.[0]?.url).toContain(`/${sourceKey}`); + } + + test('a moderator delete in the following channel', async () => { + const {sourceId, copyId, sourceKey} = await publishedCopyWithFile(); + await deleteMessageRequest(harness, world.b.owner.token, world.b.t1.id, copyId); + expect(await readRow(world.b.t1.id, copyId)).toBeNull(); + await expectSourceFileKept(sourceId, sourceKey); + }); + + test('a bulk delete in the following channel', async () => { + const {sourceId, copyId, sourceKey} = await publishedCopyWithFile(); + const other = await sendMessage(harness, world.b.owner.token, world.b.t1.id, {content: 'local'}); + await createBuilder(harness, world.b.owner.token) + .post(`/channels/${world.b.t1.id}/messages/bulk-delete`) + .body({message_ids: [copyId, other.id]}) + .expect(204) + .execute(); + expect(await readRow(world.b.t1.id, copyId)).toBeNull(); + await expectSourceFileKept(sourceId, sourceKey); + }); + + test('deleting the following channel', async () => { + const {sourceId, sourceKey} = await publishedCopyWithFile(); + await createBuilder(harness, world.b.owner.token).delete(`/channels/${world.b.t1.id}`).expect(204).execute(); + await expectSourceFileKept(sourceId, sourceKey); + }); + + test('deleting the following guild', async () => { + const {sourceId, sourceKey} = await publishedCopyWithFile(); + await createBuilder(harness, world.b.owner.token) + .post(`/guilds/${world.b.guild.id}/delete`) + .body({password: world.b.owner.password}) + .expect(204) + .execute(); + await world.worker.drain(); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect(deleted).not.toContain(sourceKey); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(true); + expect((await readRow(world.a.ann.id, sourceId))!.attachments).toHaveLength(1); + }); + }); + + describe('deleting the source channel after it published', () => { + async function publishedWithFile(): Promise<{sourceId: string; copyId: string; sourceKey: string}> { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'channel file'}); + await publishAndDrain(harness, world, source.id); + const [attachment] = (await readRow(world.a.ann.id, source.id))!.attachments; + const copy = await onlyCopy(world.b.t1.id, source.id); + return { + sourceId: source.id, + copyId: copy.id, + sourceKey: `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`, + }; + } + + async function convertToText(): Promise { + await patchChannel(harness, world.a.owner.token, world.a.ann.id, {type: ChannelTypes.GUILD_TEXT}) + .expect(200) + .execute(); + await world.worker.drain(); + } + + async function expectCopyMarkedAndSourcePurged(sourceId: string, copyId: string, sourceKey: string) { + await world.worker.drain(); + await expectDeletedCopy(world.b.t1.id, copyId); + expect(await mappingRows(sourceId)).toHaveLength(0); + expect(await sourceIndex(world.a.ann.id)).toEqual([]); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect(deleted.filter((key) => key === sourceKey)).toHaveLength(1); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(false); + } + + test('a channel converted to text and then deleted marks its copies', async () => { + const {sourceId, copyId, sourceKey} = await publishedWithFile(); + await convertToText(); + expect((await readRow(world.b.t1.id, copyId))!.content).toBe('channel file'); + await createBuilder(harness, world.a.owner.token).delete(`/channels/${world.a.ann.id}`).expect(204).execute(); + await expectCopyMarkedAndSourcePurged(sourceId, copyId, sourceKey); + }); + + test('a guild deleted after its channel was converted to text marks the copies and purges the files', async () => { + const {sourceId, copyId, sourceKey} = await publishedWithFile(); + await convertToText(); + await createBuilder(harness, world.a.owner.token) + .post(`/guilds/${world.a.guild.id}/delete`) + .body({password: world.a.owner.password}) + .expect(204) + .execute(); + await expectCopyMarkedAndSourcePurged(sourceId, copyId, sourceKey); + }); + + test('deleting the source guild purges the source files', async () => { + const {sourceId, copyId, sourceKey} = await publishedWithFile(); + await createBuilder(harness, world.a.owner.token) + .post(`/guilds/${world.a.guild.id}/delete`) + .body({password: world.a.owner.password}) + .expect(204) + .execute(); + await expectCopyMarkedAndSourcePurged(sourceId, copyId, sourceKey); + }); + + test('a failed follower removal enqueue fails the delete before anything is purged', async () => { + const {sourceId, copyId, sourceKey} = await publishedWithFile(); + world.worker.failure = (taskType) => (taskType === 'removeChannelFollowers' ? new Error('queue down') : null); + try { + await createBuilder(harness, world.a.owner.token).delete(`/channels/${world.a.ann.id}`).expect(500).execute(); + } finally { + world.worker.failure = null; + } + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(true); + expect(await readRow(world.a.ann.id, sourceId)).not.toBeNull(); + expect((await readRow(world.b.t1.id, copyId))!.content).toBe('channel file'); + await createBuilder(harness, world.a.owner.token).delete(`/channels/${world.a.ann.id}`).expect(204).execute(); + await expectCopyMarkedAndSourcePurged(sourceId, copyId, sourceKey); + }); + }); + + test('a delete that read the source before it was flagged still marks the copies', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'raced'}); + const copy = await onlyCopy(world.b.t1.id, source.id); + const row = (await readRow(world.a.ann.id, source.id))!; + await writeRow(row, {flags: row.flags & ~MessageFlags.CROSSPOSTED}); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, source.id); + await world.worker.drain(); + await expectDeletedCopy(world.b.t1.id, copy.id); + expect(await mappingRows(source.id)).toHaveLength(0); + }); + + test('deleting unpublished messages in a text channel enqueues no sync jobs', async () => { + const message = await sendMessage(harness, world.b.owner.token, world.b.t1.id, {content: 'local only'}); + await deleteMessageRequest(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(world.worker.recorded.filter((job) => job.taskType === 'syncCrosspostedMessage')).toHaveLength(0); + }); + + test('removing a source attachment while publishing is disabled drops it from the copies', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'frozen'}, [ + 'keep.png', + 'drop.gif', + ]); + await publishAndDrain(harness, world, source.id); + const copy = await onlyCopy(world.b.t1.id, source.id); + const [kept, dropped] = (await readRow(world.a.ann.id, source.id))!.attachments; + await addGuildFeature(world.a.guild.id, GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED); + await createBuilder(harness, world.a.owner.token) + .delete(`/channels/${world.a.ann.id}/messages/${source.id}/attachments/${dropped!.id}`) + .expect(204) + .execute(); + await edit(source.id, {content: 'frozen, edited'}); + await world.worker.drain(); + const copyRow = (await readRow(world.b.t1.id, copy.id))!; + expect(copyRow.attachments.map((attachment) => attachment.id)).toEqual([kept!.id]); + expect(copyRow.content).toBe('frozen'); + const droppedKey = `attachments/${world.a.ann.id}/${dropped!.id}/${dropped!.filename}`; + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, droppedKey)).toBe(false); + }); + + test('a copy owns no decay record and its expiry purges only the source file', async () => { + const queued: Array<{s3Key: string}> = []; + disableCrosspostWorker(); + world.worker = enableCrosspostWorker( + {}, + { + assetDeletionQueue: { + async queueDeletion(item: {s3Key: string}) { + queued.push(item); + }, + } as unknown as IAssetDeletionQueue, + instanceConfigRepository: { + async getEffectiveAttachmentDecayConfig() { + return {enabled: true}; + }, + } as unknown as InstanceConfigRepository, + }, + ); + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'decaying'}); + await publishAndDrain(harness, world, source.id); + const [attachment] = (await readRow(world.a.ann.id, source.id))!.attachments; + const copy = await onlyCopy(world.b.t1.id, source.id); + await createBuilder(harness, world.b.owner.token).get(`/channels/${world.b.t1.id}/messages/${copy.id}`).execute(); + const repository = new AttachmentDecayRepository(); + const record = await repository.fetchById(attachment!.id); + expect(record?.channel_id.toString()).toBe(world.a.ann.id); + expect(record?.message_id.toString()).toBe(source.id); + const expiresAt = new Date(Date.now() - 60_000); + await repository.upsert({...record!, expires_at: expiresAt, expiry_bucket: getExpiryBucket(expiresAt)}); + await processExpiredAttachments(); + expect(queued.map((item) => item.s3Key)).toEqual([ + `attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`, + ]); + }); + + test('a copy deleted in the target is cleaned up lazily on the next edit', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'moderated'}); + const copy = await onlyCopy(world.b.t1.id, source.id); + await deleteMessageRequest(harness, world.b.owner.token, world.b.t1.id, copy.id); + expect(await mappingRow(source.id, webhookId)).not.toBeNull(); + await edit(source.id, {content: 'edited after removal'}); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + expect(await mappingRow(source.id, webhookId)).toBeNull(); + }); + + test('syncs after the source is gone are no-ops', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'twice'}); + const copy = await onlyCopy(world.b.t1.id, source.id); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, source.id); + await world.worker.drain(); + const marked = await readRow(world.b.t1.id, copy.id); + for (const mode of ['update', 'source_deleted'] as const) { + await enqueueCrosspostSync(world.worker, { + channelId: createChannelID(BigInt(world.a.ann.id)), + messageId: createMessageID(BigInt(source.id)), + mode, + }); + await world.worker.drain(); + } + expect(world.worker.deadLetters).toHaveLength(0); + expect((await readRow(world.b.t1.id, copy.id))?.version).toBe(marked?.version); + }); + + test('a copy rewritten as source deleted never takes later updates', async () => { + await followInto(harness, world, world.b.ageRestricted.id); + const source = await postAndPublish(harness, world, {content: 'rule change'}); + const copy = await onlyCopy(world.b.ageRestricted.id, source.id); + await patchChannel(harness, world.a.owner.token, world.a.ann.id, {nsfw_override: true}).expect(200).execute(); + await patchChannel(harness, world.b.owner.token, world.b.ageRestricted.id, {nsfw_override: false}) + .expect(200) + .execute(); + await edit(source.id, {content: 'second version'}); + await world.worker.drain(); + await expectDeletedCopy(world.b.ageRestricted.id, copy.id); + await patchChannel(harness, world.b.owner.token, world.b.ageRestricted.id, {nsfw_override: true}) + .expect(200) + .execute(); + await edit(source.id, {content: 'third version'}); + await world.worker.drain(); + await expectDeletedCopy(world.b.ageRestricted.id, copy.id); + }); + + test('a delivered copy keeps updating after an unfollow', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'before unfollow'}); + await createBuilder(harness, world.b.owner.token).delete(`/webhooks/${webhookId}`).expect(204).execute(); + await edit(source.id, {content: 'after unfollow'}); + await world.worker.drain(); + expect((await onlyCopy(world.b.t1.id, source.id)).content).toBe('after unfollow'); + }); + + test('old copies keep updating after the follower webhook moves', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'in t1'}); + await createBuilder(harness, world.b.owner.token) + .patch(`/webhooks/${webhookId}`) + .body({channel_id: world.b.t2.id}) + .expect(200) + .execute(); + await edit(source.id, {content: 'still in t1'}); + await world.worker.drain(); + expect((await onlyCopy(world.b.t1.id, source.id)).content).toBe('still in t1'); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, source.id)).toHaveLength(0); + const next = await postAndPublish(harness, world, {content: 'lands in t2'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, next.id)).toHaveLength(1); + }); + + test('unpublished messages enqueue no sync jobs on edit and a no-op one on delete', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'private note'}); + await edit(message.id, {content: 'edited'}); + expect(world.worker.byTask('syncCrosspostedMessage')).toHaveLength(0); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, message.id); + expect(world.worker.byTask('syncCrosspostedMessage')).toHaveLength(1); + await world.worker.drain(); + expect(world.worker.byTask('syncCrosspostCopies')).toHaveLength(0); + expect(world.worker.deadLetters).toHaveLength(0); + }); + + test('a target that loses its age restriction gets the copy rewritten on the next edit', async () => { + await patchChannel(harness, world.a.owner.token, world.a.ann.id, {nsfw_override: true}).expect(200).execute(); + await followInto(harness, world, world.b.ageRestricted.id); + const source = await postAndPublish(harness, world, {content: 'restricted news'}); + const copy = await onlyCopy(world.b.ageRestricted.id, source.id); + await patchChannel(harness, world.b.owner.token, world.b.ageRestricted.id, {nsfw_override: false}) + .expect(200) + .execute(); + await edit(source.id, {content: 'restricted news, edited'}); + await world.worker.drain(); + await expectDeletedCopy(world.b.ageRestricted.id, copy.id); + }); + + test('webhook token edits and attachment deletes propagate', async () => { + await followInto(harness, world, world.b.t1.id); + const incoming = await createWebhook(harness, world.a.ann.id, world.a.owner.token, 'Releases'); + const {json} = await executeWebhook(harness, incoming.id, incoming.token, {content: 'v1', wait: true}, 200); + await publishAndDrain(harness, world, json!.id); + await createBuilderWithoutAuth(harness) + .patch(`/webhooks/${incoming.id}/${incoming.token}/messages/${json!.id}`) + .body({content: 'v2'}) + .expect(200) + .execute(); + await world.worker.drain(); + expect((await onlyCopy(world.b.t1.id, json!.id)).content).toBe('v2'); + const withFile = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'file'}); + await publishAndDrain(harness, world, withFile.id); + const copy = await onlyCopy(world.b.t1.id, withFile.id); + const sourceAttachment = (await readRow(world.a.ann.id, withFile.id))!.attachments[0]!; + const attachmentId = sourceAttachment.id; + expect((await readRow(world.b.t1.id, copy.id))!.attachments[0]!.id).toBe(attachmentId); + await createBuilder(harness, world.a.owner.token) + .delete(`/channels/${world.a.ann.id}/messages/${withFile.id}/attachments/${attachmentId}`) + .expect(204) + .execute(); + await world.worker.drain(); + expect((await readRow(world.b.t1.id, copy.id))!.attachments).toHaveLength(0); + const deleted = harness.storageService.getDeletedObjects().map((entry) => entry.key); + expect( + deleted.filter((key) => key === `attachments/${world.a.ann.id}/${attachmentId}/${sourceAttachment.filename}`), + ).toHaveLength(1); + expect(deleted.some((key) => key.startsWith(`attachments/${world.b.t1.id}/`))).toBe(false); + }); + + test('copies in several guilds follow edits of the source', async () => { + const c = await createAnnouncementTargetGuild(harness, 'Third Community'); + await addGuildMember(harness, world.a.owner, world.a.guild, c.owner); + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, c.t1.id, c.owner.token); + const source = await postAndPublish(harness, world, {content: 'shared'}); + await edit(source.id, {content: 'shared, edited'}); + await world.worker.drain(); + expect((await onlyCopy(world.b.t1.id, source.id)).content).toBe('shared, edited'); + expect((await onlyCopy(c.t1.id, source.id, c.owner.token)).content).toBe('shared, edited'); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/CrosspostRateLimits.test.ts b/fluxer_api/src/api/channel/tests/CrosspostRateLimits.test.ts new file mode 100644 index 000000000..c68f08384 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostRateLimits.test.ts @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {loadFixture, sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils'; +import { + type AnnouncementWorld, + channelBudgetRemaining, + createAnnouncementChannel, + createAnnouncementWorld, + crosspostRequest, + editMessage, + postMessage, + publish, + publishedEditBudgetRemaining, + RecordingWorkerService, +} from '@app/api/channel/tests/CrosspostTestUtils'; +import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry'; +import {RateLimitConfigs} from '@app/api/RateLimitConfig'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {createWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import { + CROSSPOST_CHANNEL_RATE_LIMIT, + PUBLISHED_MESSAGE_EDIT_RATE_LIMIT, +} from '@fluxer/constants/src/AnnouncementConstants'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {MessageFlags} from '@fluxer/constants/src/ChannelConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {ms} from 'itty-time'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test} from 'vitest'; + +interface RateLimitBody { + code: string; + retry_after: number; + global: boolean; +} + +describe('Publishing and published-edit limits', () => { + let harness: ApiTestHarness; + let worker: RecordingWorkerService; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + worker = new RecordingWorkerService(); + setInjectedWorkerService(worker); + world = await createAnnouncementWorld(harness); + }); + + afterEach(() => { + setInjectedWorkerService(new NoopWorkerService()); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function publishMany(channelId: string, count: number, token = world.author.token): Promise { + for (let index = 0; index < count; index++) { + const message = await postMessage(harness, token, channelId, `Update ${channelId} ${index}`); + await publish(harness, token, channelId, message.id); + } + } + + test('a channel publishes 10 in a row, then the 11th is limited with a shared scope', async () => { + await publishMany(world.announcement.id, CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + const extra = await postMessage(harness, world.author.token, world.announcement.id, 'One too many'); + const {response, json} = await crosspostRequest(harness, world.author.token, world.announcement.id, extra.id) + .expect(429, APIErrorCodes.MESSAGE_CROSSPOST_RATE_LIMITED) + .executeWithResponse(); + const body = json as unknown as RateLimitBody; + expect(body.retry_after).toBeGreaterThan(300); + expect(body.retry_after).toBeLessThanOrEqual(360); + expect(body.global).toBe(false); + expect(response.headers.get('X-RateLimit-Scope')).toBe('shared'); + expect(Number(response.headers.get('Retry-After'))).toBeGreaterThan(300); + expect(worker.byTask('crosspostMessage')).toHaveLength(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + }); + + test('the channel limit is per channel and shared across members', async () => { + await publishMany(world.announcement.id, CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + const byMod = await postMessage(harness, world.mod.token, world.announcement.id, 'Moderator news'); + await crosspostRequest(harness, world.mod.token, world.announcement.id, byMod.id) + .expect(429, APIErrorCodes.MESSAGE_CROSSPOST_RATE_LIMITED) + .execute(); + const second = await createAnnouncementChannel(harness, world.owner.token, world.guild.id, 'news-two'); + const elsewhere = await postMessage(harness, world.author.token, second.id, 'Different channel'); + await publish(harness, world.author.token, second.id, elsewhere.id); + }); + + test('a community has no shared publish allowance across its announcement channels', async () => { + const channels = [ + world.announcement, + await createAnnouncementChannel(harness, world.owner.token, world.guild.id, 'news-b'), + await createAnnouncementChannel(harness, world.owner.token, world.guild.id, 'news-c'), + await createAnnouncementChannel(harness, world.owner.token, world.guild.id, 'news-d'), + ]; + for (const channel of channels) { + await publishMany(channel.id, CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + expect(await channelBudgetRemaining(channel.id)).toBe(0); + } + expect(worker.byTask('crosspostMessage')).toHaveLength(channels.length * CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + const last = channels[3]!; + const extra = await postMessage(harness, world.author.token, last.id, 'One past the channel allowance'); + const {response} = await crosspostRequest(harness, world.author.token, last.id, extra.id) + .expect(429, APIErrorCodes.MESSAGE_CROSSPOST_RATE_LIMITED) + .executeWithResponse(); + expect(response.headers.get('X-RateLimit-Scope')).toBe('shared'); + }); + + test('failed publishes do not consume budget', async () => { + const published = await postMessage(harness, world.author.token, world.announcement.id, 'Published once'); + await publish(harness, world.author.token, world.announcement.id, published.id); + const reply = await createBuilder(harness, world.author.token) + .post(`/channels/${world.announcement.id}/messages`) + .body({content: 'Reply', message_reference: {message_id: published.id}}) + .expect(200) + .execute(); + await crosspostRequest(harness, world.author.token, world.announcement.id, published.id) + .expect(400, APIErrorCodes.MESSAGE_ALREADY_CROSSPOSTED) + .execute(); + await crosspostRequest(harness, world.author.token, world.announcement.id, reply.id) + .expect(400, APIErrorCodes.MESSAGE_NOT_CROSSPOSTABLE) + .execute(); + await crosspostRequest(harness, world.member.token, world.announcement.id, reply.id) + .expect(403, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await crosspostRequest(harness, world.author.token, world.announcement.id, '123456789012345678') + .expect(404, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts - 1); + }); + + test('a published message can be edited 3 times in a row, then the 4th is limited', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Edit me'); + const other = await postMessage(harness, world.author.token, world.announcement.id, 'Edit me too'); + await publish(harness, world.author.token, world.announcement.id, message.id); + await publish(harness, world.author.token, world.announcement.id, other.id); + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts; index++) { + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: `Edit ${index}`}) + .expect(200) + .execute(); + } + const {response, json} = await editMessage(harness, world.author.token, world.announcement.id, message.id, { + content: 'Edit 4', + }) + .expect(429, APIErrorCodes.PUBLISHED_MESSAGE_EDIT_RATE_LIMITED) + .executeWithResponse(); + const body = json as unknown as RateLimitBody; + expect(body.retry_after).toBeGreaterThan(1100); + expect(body.retry_after).toBeLessThanOrEqual(1200); + expect(response.headers.get('X-RateLimit-Scope')).toBe('shared'); + expect(worker.syncJobsFor(message.id)).toHaveLength(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts); + await editMessage(harness, world.author.token, world.announcement.id, other.id, {content: 'Other edit'}) + .expect(200) + .execute(); + }); + + test('edits before publishing do not count toward the published-edit cap', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Draft'); + for (let index = 0; index < 3; index++) { + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: `Draft ${index}`}) + .expect(200) + .execute(); + } + expect(worker.syncJobsFor(message.id)).toHaveLength(0); + await publish(harness, world.author.token, world.announcement.id, message.id); + expect(await publishedEditBudgetRemaining(message.id)).toBe(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts); + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts; index++) { + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: `Final ${index}`}) + .expect(200) + .execute(); + } + }); + + test('unpublished messages are never limited by the published-edit cap', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Never published'); + for (let index = 0; index < 6; index++) { + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: `Version ${index}`}) + .expect(200) + .execute(); + } + expect(await publishedEditBudgetRemaining(message.id)).toBe(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts); + expect(worker.byTask('syncCrosspostedMessage')).toHaveLength(0); + }); + + test('a moderator suppress-embeds edit is not limited and still propagates', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Link https://example.com'); + await publish(harness, world.author.token, world.announcement.id, message.id); + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts; index++) { + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: `Link ${index}`}) + .expect(200) + .execute(); + } + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: 'Blocked'}) + .expect(429, APIErrorCodes.PUBLISHED_MESSAGE_EDIT_RATE_LIMITED) + .execute(); + const syncsBefore = worker.syncJobsFor(message.id).length; + const suppressed = await editMessage(harness, world.mod.token, world.announcement.id, message.id, { + flags: MessageFlags.SUPPRESS_EMBEDS, + }) + .expect(200) + .execute(); + expect(suppressed.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + const restored = await editMessage(harness, world.mod.token, world.announcement.id, message.id, {flags: 0}) + .expect(200) + .execute(); + expect(restored.flags).toBe(MessageFlags.CROSSPOSTED); + expect(worker.syncJobsFor(message.id).length).toBe(syncsBefore + 2); + }); + + test('webhook token edits count toward the published-edit cap', async () => { + const webhook = await createWebhook(harness, world.announcement.id, world.owner.token, 'Changelog'); + const executed = await createBuilderWithoutAuth(harness) + .post(`/webhooks/${webhook.id}/${webhook.token}?wait=true`) + .body({content: 'v1'}) + .expect(200) + .execute(); + await publish(harness, world.mod.token, world.announcement.id, executed.id); + const editPath = `/webhooks/${webhook.id}/${webhook.token}/messages/${executed.id}`; + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts; index++) { + await createBuilderWithoutAuth(harness) + .patch(editPath) + .body({content: `v1.${index}`}) + .expect(200) + .execute(); + } + await createBuilderWithoutAuth(harness) + .patch(editPath) + .body({content: 'v1.9'}) + .expect(429, APIErrorCodes.PUBLISHED_MESSAGE_EDIT_RATE_LIMITED) + .execute(); + expect(worker.syncJobsFor(executed.id)).toHaveLength(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts); + }); + + test('published edits that fail validation do not use the edit budget', async () => { + const missingUpload = { + attachments: [ + { + id: 0, + filename: 'upload.png', + upload_filename: 'missing-upload-key', + file_size: 2048, + content_type: 'image/png', + }, + ], + }; + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Edit me'); + await publish(harness, world.author.token, world.announcement.id, message.id); + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts; index++) { + const {json} = await editMessage(harness, world.author.token, world.announcement.id, message.id, { + content: `Broken ${index}`, + ...missingUpload, + }) + .expect(400) + .executeWithResponse(); + expect(JSON.stringify(json)).toContain('FILE_NOT_FOUND'); + } + expect(await publishedEditBudgetRemaining(message.id)).toBe(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts); + await editMessage(harness, world.author.token, world.announcement.id, message.id, {content: 'Valid edit'}) + .expect(200) + .execute(); + expect(await publishedEditBudgetRemaining(message.id)).toBe(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts - 1); + }); + + test('removing an attachment from a published message counts toward the cap and propagates', async () => { + const {json} = await sendMessageWithAttachments( + harness, + world.author.token, + world.announcement.id, + { + content: 'Two pictures', + attachments: [ + {id: 0, filename: 'first.png'}, + {id: 1, filename: 'second.png'}, + ], + }, + [ + {index: 0, filename: 'first.png', data: loadFixture('yeah.png')}, + {index: 1, filename: 'second.png', data: loadFixture('yeah.png')}, + ], + ); + expect(json.attachments?.length).toBe(2); + await publish(harness, world.author.token, world.announcement.id, json.id); + await createBuilder(harness, world.author.token) + .delete(`/channels/${world.announcement.id}/messages/${json.id}/attachments/${json.attachments![0]!.id}`) + .expect(204) + .execute(); + expect(await publishedEditBudgetRemaining(json.id)).toBe(PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts - 1); + expect(worker.syncJobsFor(json.id)).toHaveLength(1); + for (let index = 0; index < PUBLISHED_MESSAGE_EDIT_RATE_LIMIT.maxAttempts - 1; index++) { + await editMessage(harness, world.author.token, world.announcement.id, json.id, {content: `Caption ${index}`}) + .expect(200) + .execute(); + } + await createBuilder(harness, world.author.token) + .delete(`/channels/${world.announcement.id}/messages/${json.id}/attachments/${json.attachments![1]!.id}`) + .expect(429, APIErrorCodes.PUBLISHED_MESSAGE_EDIT_RATE_LIMITED) + .execute(); + }); + + test('the publish and follow route buckets are enforced when route limits are on', async () => { + expect(RateLimitConfigs.CHANNEL_MESSAGE_CROSSPOST).toEqual({ + bucket: 'channel:message:crosspost::channel_id', + config: {limit: 5, windowMs: ms('5 seconds')}, + }); + expect(RateLimitConfigs.CHANNEL_FOLLOW).toEqual({ + bucket: 'channel:follow::channel_id', + config: {limit: 5, windowMs: ms('10 seconds')}, + }); + expect(RateLimitConfigs.CHANNEL_FOLLOWER_STATS).toEqual({ + bucket: 'channel:follower_stats::channel_id', + config: {limit: 10, windowMs: ms('10 seconds')}, + }); + const statuses: Array = []; + for (let index = 0; index < 6; index++) { + const {response} = await crosspostRequest( + harness, + world.author.token, + world.announcement.id, + `12345678901234567${index}`, + ) + .header('x-fluxer-test-enable-rate-limits', 'true') + .executeRaw(); + statuses.push(response.status); + } + expect(statuses.slice(0, 5)).toEqual([404, 404, 404, 404, 404]); + expect(statuses[5]).toBe(429); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/CrosspostSource.test.ts b/fluxer_api/src/api/channel/tests/CrosspostSource.test.ts new file mode 100644 index 000000000..5607c247f --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostSource.test.ts @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {createGuildID} from '@app/api/BrandedTypes'; +import {addGuildMember, disableCrosspostWorker, setGuildFeatures} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {createPermissionOverwrite} from '@app/api/channel/tests/ChannelTestUtils'; +import { + copiesOf, + type FanoutWorld, + followInto, + postAndPublish, + sendMessage, + setupFanoutWorld, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {ensureSessionStarted, getMessages} from '@app/api/message/tests/MessageTestUtils'; +import {getGuildDiscoveryRepository} from '@app/api/middleware/ServiceSingletons'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {MessageTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import {DiscoveryApplicationStatus, DiscoveryCategories} from '@fluxer/constants/src/DiscoveryConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import type {CrosspostSourceResponse} from '@fluxer/schema/src/domains/message/CrosspostSourceSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +describe('Crosspost source', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + let outsider: TestAccount; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await setupFanoutWorld(harness); + outsider = await createTestAccount(harness); + await addGuildMember(harness, world.b.owner, world.b.guild, outsider); + await ensureSessionStarted(harness, outsider.token); + }); + + afterEach(() => { + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + function sourcePath(channelId: string, messageId: string): string { + return `/channels/${channelId}/messages/${messageId}/crosspost-source`; + } + + function getSource(token: string, channelId: string, messageId: string) { + return createBuilder(harness, token).get(sourcePath(channelId, messageId)); + } + + async function publishedCopy(): Promise { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'Version 2 is out'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy).toBeDefined(); + return copy!; + } + + async function followNotice(): Promise { + const messages = await getMessages(harness, world.b.owner.token, world.b.t1.id); + const notice = messages.find((message) => message.type === MessageTypes.CHANNEL_FOLLOW_ADD); + expect(notice).toBeDefined(); + return notice!; + } + + test('returns the source community card to a viewer who is not a member of it', async () => { + const copy = await publishedCopy(); + vi.spyOn(NoopGatewayService.prototype, 'getGuildCounts').mockResolvedValue({ + memberCount: 1234, + presenceCount: 56, + }); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response).toEqual({ + guild: { + id: world.a.guild.id, + name: world.a.guild.name, + icon: null, + banner: null, + description: null, + features: [], + approximate_member_count: 1234, + approximate_presence_count: 56, + discoverable: false, + }, + }); + expect(Object.keys(response.guild).sort()).toEqual( + [ + 'approximate_member_count', + 'approximate_presence_count', + 'banner', + 'description', + 'discoverable', + 'features', + 'icon', + 'id', + 'name', + ].sort(), + ); + }); + + test('returns the same card to a viewer who is a member of the source community', async () => { + const copy = await publishedCopy(); + const asMember = await getSource(world.b.owner.token, world.b.t1.id, copy.id).execute(); + const asOutsider = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(asMember).toEqual(asOutsider); + expect(asMember.guild.id).toBe(world.a.guild.id); + }); + + async function writeDiscoveryApplication(status: string, description: string): Promise { + await getGuildDiscoveryRepository().upsert({ + guild_id: createGuildID(BigInt(world.a.guild.id)), + status, + category_type: DiscoveryCategories.OTHER, + description, + primary_language: null, + custom_tags: null, + applied_at: new Date(), + reviewed_at: null, + reviewed_by: null, + review_reason: null, + removed_at: null, + removed_by: null, + removal_reason: null, + }); + } + + test('returns the community description while it is listed in discovery', async () => { + const copy = await publishedCopy(); + await writeDiscoveryApplication(DiscoveryApplicationStatus.APPROVED, 'Release notes and outage reports'); + await setGuildFeatures(harness, world.a.guild.id, {add: [GuildFeatures.DISCOVERABLE]}); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.description).toBe('Release notes and outage reports'); + }); + + test('hides the description of an application that is not approved', async () => { + const copy = await publishedCopy(); + await writeDiscoveryApplication(DiscoveryApplicationStatus.PENDING, 'Pending application text'); + await setGuildFeatures(harness, world.a.guild.id, {add: [GuildFeatures.DISCOVERABLE]}); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.description).toBeNull(); + }); + + test('hides an approved description once the community is no longer listed', async () => { + const copy = await publishedCopy(); + await writeDiscoveryApplication(DiscoveryApplicationStatus.APPROVED, 'Delisted community text'); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.description).toBeNull(); + }); + + test('reports a discoverable source community as joinable', async () => { + const copy = await publishedCopy(); + await setGuildFeatures(harness, world.a.guild.id, {add: [GuildFeatures.DISCOVERABLE]}); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.discoverable).toBe(true); + expect(response.guild.features).toEqual([GuildFeatures.DISCOVERABLE]); + }); + + test('does not report a discoverable community with invites disabled as joinable', async () => { + const copy = await publishedCopy(); + await setGuildFeatures(harness, world.a.guild.id, { + add: [GuildFeatures.DISCOVERABLE, GuildFeatures.INVITES_DISABLED], + }); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.discoverable).toBe(false); + }); + + test('only exposes the badge features of the source community', async () => { + const copy = await publishedCopy(); + await setGuildFeatures(harness, world.a.guild.id, { + add: [ + GuildFeatures.VERIFIED, + GuildFeatures.PARTNERED, + GuildFeatures.INVITES_DISABLED, + GuildFeatures.VIP_VOICE, + GuildFeatures.VANITY_URL, + ], + }); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect([...response.guild.features].sort()).toEqual([GuildFeatures.PARTNERED, GuildFeatures.VERIFIED].sort()); + }); + + test('returns null counts when the gateway cannot provide them', async () => { + const copy = await publishedCopy(); + vi.spyOn(NoopGatewayService.prototype, 'getGuildCounts').mockRejectedValue(new Error('gateway down')); + const response = await getSource(outsider.token, world.b.t1.id, copy.id).execute(); + expect(response.guild.approximate_member_count).toBeNull(); + expect(response.guild.approximate_presence_count).toBeNull(); + expect(response.guild.name).toBe(world.a.guild.name); + }); + + test('accepts the follow system message', async () => { + await followInto(harness, world, world.b.t1.id); + const notice = await followNotice(); + const response = await getSource(outsider.token, world.b.t1.id, notice.id).execute(); + expect(response.guild.id).toBe(world.a.guild.id); + }); + + test('rejects a message that is not a copy', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.b.owner.token, world.b.t1.id, {content: 'hello'}); + await getSource(outsider.token, world.b.t1.id, message.id) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + }); + + test('rejects the source message itself', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'published'}); + await getSource(world.b.owner.token, world.a.ann.id, source.id) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + }); + + test('rejects a viewer who cannot view the channel of the copy', async () => { + const copy = await publishedCopy(); + await createPermissionOverwrite(harness, world.b.owner.token, world.b.t1.id, outsider.userId, { + type: 1, + allow: '0', + deny: Permissions.VIEW_CHANNEL.toString(), + }); + const message = await createBuilder(harness, outsider.token) + .get(`/channels/${world.b.t1.id}/messages/${copy.id}`) + .executeRaw(); + expect(message.response.status).not.toBe(HTTP_STATUS.OK); + const source = await getSource(outsider.token, world.b.t1.id, copy.id).executeRaw(); + expect(source.response.status).toBe(message.response.status); + expect((source.json as {code?: string}).code).toBe((message.json as {code?: string}).code); + }); + + test('rejects a viewer who is not in the community of the copy', async () => { + const copy = await publishedCopy(); + const stranger = await createTestAccount(harness); + const message = await createBuilder(harness, stranger.token) + .get(`/channels/${world.b.t1.id}/messages/${copy.id}`) + .executeRaw(); + expect(message.response.status).not.toBe(HTTP_STATUS.OK); + const source = await getSource(stranger.token, world.b.t1.id, copy.id).executeRaw(); + expect(source.response.status).toBe(message.response.status); + expect((source.json as {code?: string}).code).toBe((message.json as {code?: string}).code); + }); + + test('returns unknown guild once the source community is deleted', async () => { + const copy = await publishedCopy(); + await createBuilder(harness, world.a.owner.token) + .post(`/guilds/${world.a.guild.id}/delete`) + .body({password: world.a.owner.password}) + .expect(HTTP_STATUS.NO_CONTENT) + .executeWithResponse(); + await world.worker.drain(); + await getSource(outsider.token, world.b.t1.id, copy.id) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_GUILD) + .execute(); + const notice = await followNotice(); + await getSource(outsider.token, world.b.t1.id, notice.id) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_GUILD) + .execute(); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/CrosspostTestUtils.ts b/fluxer_api/src/api/channel/tests/CrosspostTestUtils.ts new file mode 100644 index 000000000..3a6cb4b1c --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostTestUtils.ts @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createGuildID, createMessageID} from '@app/api/BrandedTypes'; +import {createApiContext} from '@app/api/CreateApiContext'; +import {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import type {ChannelService} from '@app/api/channel/services/ChannelService'; +import {publishedEditRateLimitIdentifier} from '@app/api/channel/services/message/CrosspostPropagation'; +import {crosspostChannelRateLimitIdentifier} from '@app/api/channel/services/message/MessageCrosspostService'; +import { + acceptInvite, + addMemberRole, + createChannel, + createChannelInvite, + createGuild, + createRole, + sendChannelMessage, +} from '@app/api/channel/tests/ChannelTestUtils'; +import type {GuildRow} from '@app/api/database/types/GuildTypes'; +import {GuildRepository} from '@app/api/guild/repositories/GuildRepository'; +import {DisabledLiveKitService} from '@app/api/infrastructure/DisabledLiveKitService'; +import {InMemoryVoiceRoomStore} from '@app/api/infrastructure/InMemoryVoiceRoomStore'; +import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils'; +import {createGuildStackServices} from '@app/api/middleware/GuildStackServiceFactory'; +import {getVoiceAvailabilityService} from '@app/api/middleware/ServiceRegistry'; +import { + getAssetDeletionQueue, + getAttachmentUploadTraceRepository, + getAvatarService, + getChannelRepository, + getEmbedService, + getEntityAssetService, + getFavoriteMemeRepository, + getGuildAuditLogService, + getGuildRepository, + getInviteRepository, + getLimitConfigService, + getPurgeQueue, + getRateLimitService, + getReadStateService, + getStorageService, + getUserCacheService, + getUserRepository, + getVirusScanServiceInstance, + getWebhookRepository, +} from '@app/api/middleware/ServiceSingletons'; +import type {Message} from '@app/api/models/Message'; +import type {ApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import { + CROSSPOST_CHANNEL_RATE_LIMIT, + PUBLISHED_MESSAGE_EDIT_RATE_LIMIT, +} from '@fluxer/constants/src/AnnouncementConstants'; +import {ChannelTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; +import type {WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; + +export interface RecordedJob { + taskType: string; + payload: WorkerJobPayload; + options: WorkerJobOptions | undefined; +} + +export class RecordingWorkerService implements IWorkerService { + readonly jobs: Array = []; + failure: ((taskType: string) => Error | null) | null = null; + private nextJobId = 1n; + + async addJob( + taskType: string, + payload: TPayload, + options?: WorkerJobOptions, + ): Promise { + const error = this.failure?.(taskType) ?? null; + if (error) { + throw error; + } + this.jobs.push({taskType, payload, options}); + return this.nextJobId++; + } + + async cancelJob(_jobId: bigint): Promise { + return false; + } + + async retryDeadLetterJob(_jobId: bigint): Promise { + return false; + } + + byTask(taskType: string): Array { + return this.jobs.filter((job) => job.taskType === taskType); + } + + syncJobsFor(messageId: string): Array { + return this.byTask('syncCrosspostedMessage').filter((job) => job.payload.messageId === messageId); + } +} + +export interface AnnouncementWorld { + owner: TestAccount; + author: TestAccount; + mod: TestAccount; + member: TestAccount; + guild: GuildResponse; + announcement: ChannelResponse; + text: ChannelResponse; + modRoleId: string; +} + +export async function createAnnouncementWorld(harness: ApiTestHarness): Promise { + const owner = await createTestAccount(harness); + const author = await createTestAccount(harness); + const mod = await createTestAccount(harness); + const member = await createTestAccount(harness); + const guild = await createGuild(harness, owner.token, 'Announcements Source'); + const announcement = await createChannel(harness, owner.token, guild.id, 'news', ChannelTypes.GUILD_ANNOUNCEMENT); + const text = await createChannel(harness, owner.token, guild.id, 'chat', ChannelTypes.GUILD_TEXT); + const invite = await createChannelInvite(harness, owner.token, announcement.id); + for (const account of [author, mod, member]) { + await acceptInvite(harness, account.token, invite.code); + await ensureSessionStarted(harness, account.token); + } + await ensureSessionStarted(harness, owner.token); + const modRole = await createRole(harness, owner.token, guild.id, { + name: 'Moderator', + permissions: Permissions.MANAGE_MESSAGES.toString(), + }); + await addMemberRole(harness, owner.token, guild.id, mod.userId, modRole.id); + return {owner, author, mod, member, guild, announcement, text, modRoleId: modRole.id}; +} + +export async function createAnnouncementChannel( + harness: ApiTestHarness, + token: string, + guildId: string, + name: string, +): Promise { + return createChannel(harness, token, guildId, name, ChannelTypes.GUILD_ANNOUNCEMENT); +} + +export async function postMessage( + harness: ApiTestHarness, + token: string, + channelId: string, + content: string, +): Promise { + return sendChannelMessage(harness, token, channelId, content); +} + +export function crosspostRequest(harness: ApiTestHarness, token: string, channelId: string, messageId: string) { + return createBuilder(harness, token).post(`/channels/${channelId}/messages/${messageId}/crosspost`); +} + +export async function publish( + harness: ApiTestHarness, + token: string, + channelId: string, + messageId: string, +): Promise { + return crosspostRequest(harness, token, channelId, messageId).expect(200).execute(); +} + +export function editMessage( + harness: ApiTestHarness, + token: string, + channelId: string, + messageId: string, + body: Record, +) { + return createBuilder(harness, token) + .patch(`/channels/${channelId}/messages/${messageId}`) + .body(body); +} + +export async function readMessageRow(channelId: string, messageId: string): Promise { + return new ChannelRepository().messages.getMessage( + createChannelID(BigInt(channelId)), + createMessageID(BigInt(messageId)), + ); +} + +export async function writeMessageRow(message: Message, patch: Partial>): Promise { + await new ChannelRepository().messages.upsertMessage({...message.toRow(), ...patch}, message.toRow()); +} + +export async function listCrosspostSources(channelId: string): Promise> { + const ids = await new ChannelRepository().crossposts.listSourcesByChannel(createChannelID(BigInt(channelId)), { + limit: 100, + }); + return ids.map((id) => id.toString()); +} + +export async function patchGuildRow(guildId: string, patch: Partial): Promise { + const repository = new GuildRepository(); + const id = createGuildID(BigInt(guildId)); + const existing = await repository.findUnique(id); + if (!existing) { + throw new Error(`guild ${guildId} not found`); + } + await repository.upsertPartial(id, patch, existing.toRow()); +} + +export async function addGuildFeature(guildId: string, feature: string): Promise { + const existing = await new GuildRepository().findUnique(createGuildID(BigInt(guildId))); + const features = new Set(existing?.features ?? []); + features.add(feature); + await patchGuildRow(guildId, {features}); +} + +export async function channelBudgetRemaining(channelId: string): Promise { + const result = await getRateLimitService().peekLimit({ + identifier: crosspostChannelRateLimitIdentifier(createChannelID(BigInt(channelId))), + ...CROSSPOST_CHANNEL_RATE_LIMIT, + }); + return result.remaining; +} + +export async function publishedEditBudgetRemaining(messageId: string): Promise { + const result = await getRateLimitService().peekLimit({ + identifier: publishedEditRateLimitIdentifier(createMessageID(BigInt(messageId))), + ...PUBLISHED_MESSAGE_EDIT_RATE_LIMIT, + }); + return result.remaining; +} + +export interface StallGate { + reached: Promise; + release: () => void; +} + +export function createStallGate(): StallGate & {hit: () => Promise} { + let markReached: () => void = () => {}; + let release: () => void = () => {}; + const reached = new Promise((resolve) => { + markReached = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + return { + reached, + release, + hit: async () => { + markReached(); + await released; + }, + }; +} + +export function createTestChannelService(): ChannelService { + return createGuildStackServices({ + apiContext: createApiContext(), + channelRepository: getChannelRepository(), + userRepository: getUserRepository(), + guildRepository: getGuildRepository(), + inviteRepository: getInviteRepository(), + webhookRepository: getWebhookRepository(), + favoriteMemeRepository: getFavoriteMemeRepository(), + avatarService: getAvatarService(), + entityAssetService: getEntityAssetService(), + assetDeletionQueue: getAssetDeletionQueue(), + userCacheService: getUserCacheService(), + limitConfigService: getLimitConfigService(), + embedService: getEmbedService(), + readStateService: getReadStateService(), + storageService: getStorageService(), + attachmentUploadTraceRepository: getAttachmentUploadTraceRepository(), + virusScanService: getVirusScanServiceInstance(), + purgeQueue: getPurgeQueue(), + guildAuditLogService: getGuildAuditLogService(), + voiceRoomStore: new InMemoryVoiceRoomStore(), + liveKitService: new DisabledLiveKitService(), + voiceAvailabilityService: getVoiceAvailabilityService(), + }).channelService; +} diff --git a/fluxer_api/src/api/channel/tests/CrosspostWorkerTestUtils.ts b/fluxer_api/src/api/channel/tests/CrosspostWorkerTestUtils.ts new file mode 100644 index 000000000..0bb5a77b3 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/CrosspostWorkerTestUtils.ts @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes'; +import {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import { + type AnnouncementWorld, + announcementWorld, + enableCrosspostWorker, + follow, + publish, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {loadFixture, sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils'; +import type {ChannelRow, CrosspostedMessageRow} from '@app/api/database/types/ChannelTypes'; +import type {MessageRow} from '@app/api/database/types/MessageTypes'; +import {ensureSessionStarted, getMessages} from '@app/api/message/tests/MessageTestUtils'; +import type {Message} from '@app/api/models/Message'; +import type {ApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopLogger} from '@app/api/test/mocks/NoopLogger'; +import type {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {MessageFlags, MessageTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask'; + +export interface FanoutWorld extends AnnouncementWorld { + worker: SyncTaskWorkerService; +} + +export async function setupFanoutWorld(harness: ApiTestHarness): Promise { + const world = await announcementWorld(harness); + for (const account of [world.a.owner, world.a.member, world.a.moderator, world.b.owner, world.b.webhookManager]) { + await ensureSessionStarted(harness, account.token); + } + const worker = enableCrosspostWorker(); + return {...world, worker}; +} + +export async function followInto( + harness: ApiTestHarness, + world: FanoutWorld, + targetChannelId: string, + token = world.b.owner.token, + sourceChannelId = world.a.ann.id, +): Promise { + const response = await follow(harness, token, sourceChannelId, targetChannelId); + await world.worker.drain(); + return response.webhook_id; +} + +export async function sendMessage( + harness: ApiTestHarness, + token: string, + channelId: string, + body: Record, +): Promise { + return createBuilder(harness, token) + .post(`/channels/${channelId}/messages`) + .body(body) + .expect(200) + .execute(); +} + +export async function sendWithImage( + harness: ApiTestHarness, + token: string, + channelId: string, + payload: Record, + filenames: Array = ['yeah.png'], +): Promise { + const {response, json, text} = await sendMessageWithAttachments( + harness, + token, + channelId, + {...payload, attachments: filenames.map((filename, index) => ({id: index, filename}))}, + filenames.map((filename, index) => ({ + index, + filename, + data: loadFixture(filename.endsWith('.gif') ? 'thisisfine.gif' : 'yeah.png'), + })), + ); + if (response.status !== 200) { + throw new Error(`upload failed with ${response.status}: ${text}`); + } + return json; +} + +export async function publishAndDrain( + harness: ApiTestHarness, + world: FanoutWorld, + messageId: string, + token = world.a.owner.token, + channelId = world.a.ann.id, +): Promise { + const published = await publish(harness, token, channelId, messageId); + await world.worker.drain(); + return published; +} + +export async function postAndPublish( + harness: ApiTestHarness, + world: FanoutWorld, + body: Record, + channelId = world.a.ann.id, +): Promise { + const message = await sendMessage(harness, world.a.owner.token, channelId, body); + await publishAndDrain(harness, world, message.id, world.a.owner.token, channelId); + return message; +} + +export async function listCopies( + harness: ApiTestHarness, + token: string, + channelId: string, +): Promise> { + const messages = await getMessages(harness, token, channelId, {limit: '100'}); + return messages.filter( + (message) => message.type === MessageTypes.DEFAULT && (message.flags & MessageFlags.IS_CROSSPOST) !== 0, + ); +} + +export async function copiesOf( + harness: ApiTestHarness, + token: string, + channelId: string, + sourceMessageId: string, +): Promise> { + const copies = await listCopies(harness, token, channelId); + return copies.filter((copy) => copy.message_reference?.message_id === sourceMessageId); +} + +export async function readRow(channelId: string, messageId: string): Promise { + return new ChannelRepository().messages.getMessage( + createChannelID(BigInt(channelId)), + createMessageID(BigInt(messageId)), + ); +} + +export async function mappingRow(sourceMessageId: string, webhookId: string): Promise { + return new ChannelRepository().crossposts.get( + createMessageID(BigInt(sourceMessageId)), + createWebhookID(BigInt(webhookId)), + ); +} + +export async function mappingRows(sourceMessageId: string): Promise> { + return new ChannelRepository().crossposts.listBySourceMessage(createMessageID(BigInt(sourceMessageId)), { + limit: 1000, + }); +} + +export async function sourceIndex(channelId: string): Promise> { + const ids = await new ChannelRepository().crossposts.listSourcesByChannel(createChannelID(BigInt(channelId)), { + limit: 1000, + }); + return ids.map((id) => id.toString()); +} + +export function editRequest( + harness: ApiTestHarness, + token: string, + channelId: string, + messageId: string, + body: Record, +) { + return createBuilder(harness, token) + .patch(`/channels/${channelId}/messages/${messageId}`) + .body(body); +} + +export async function deleteMessageRequest( + harness: ApiTestHarness, + token: string, + channelId: string, + messageId: string, +): Promise { + await createBuilder(harness, token).delete(`/channels/${channelId}/messages/${messageId}`).expect(204).execute(); +} + +export function workerHelpers(worker: SyncTaskWorkerService): WorkerTaskHelpers { + return { + logger: new NoopLogger(), + jobId: 0n, + addJob: (taskType, payload, options) => worker.addJob(taskType, payload, options), + reportProgress: async () => {}, + shouldCancel: async () => false, + setContextLink: async () => {}, + }; +} + +export async function patchChannelRow(channelId: string, patch: Partial): Promise { + const repository = new ChannelRepository(); + const channel = await repository.findUnique(createChannelID(BigInt(channelId))); + if (!channel) { + throw new Error(`channel ${channelId} not found`); + } + await repository.upsert({...channel.toRow(), ...patch}); +} + +export async function writeRow(message: Message, patch: Partial): Promise { + await new ChannelRepository().messages.upsertMessage({...message.toRow(), ...patch}, message.toRow()); +} + +export function patchChannel(harness: ApiTestHarness, token: string, channelId: string, body: Record) { + return createBuilder(harness, token).patch(`/channels/${channelId}`).body(body); +} diff --git a/fluxer_api/src/api/channel/tests/EmbedAttachmentPurge.test.ts b/fluxer_api/src/api/channel/tests/EmbedAttachmentPurge.test.ts new file mode 100644 index 000000000..b1960245b --- /dev/null +++ b/fluxer_api/src/api/channel/tests/EmbedAttachmentPurge.test.ts @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import { + type ChannelID, + createChannelID, + createMessageID, + createReportID, + createUserID, + type MessageID, + type UserID, +} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import { + createChannel, + createGuild, + loadFixture, + sendMessageWithAttachments, +} from '@app/api/channel/tests/AttachmentTestUtils'; +import {acceptInvite, createChannelInvite} from '@app/api/channel/tests/ChannelTestUtils'; +import {getNcmecSubmissionService} from '@app/api/middleware/ServiceSingletons'; +import {ReportRepository} from '@app/api/report/ReportRepository'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import {MessageReferenceTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest'; + +function storageKey(url: string | null | undefined): string { + const unsigned = (url ?? '').split('?')[0]!; + expect(unsigned.startsWith(`${Config.endpoints.media}/attachments/`)).toBe(true); + return unsigned.slice(`${Config.endpoints.media}/`.length); +} + +describe('Embed attachment purge', () => { + let harness: ApiTestHarness; + let account: TestAccount; + let channelId: string; + let guildId: string; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + account = await createTestAccount(harness); + const guild = await createGuild(harness, account.token, 'Embed Purge Guild'); + const channel = await createChannel(harness, account.token, guild.id, 'embed-purge'); + channelId = channel.id; + guildId = guild.id; + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + function deletedKeys(): Array { + return harness.storageService + .getDeletedObjects() + .filter((entry) => entry.bucket === Config.s3.buckets.cdn) + .map((entry) => entry.key); + } + + function stored(key: string): boolean { + return harness.storageService.hasObject(Config.s3.buckets.cdn, key); + } + + async function sendEmbedOnly(): Promise { + const fileData = loadFixture('yeah.png'); + const {response, json} = await sendMessageWithAttachments( + harness, + account.token, + channelId, + { + content: 'embed only', + attachments: [ + {id: 0, filename: 'image.png'}, + {id: 1, filename: 'thumb.png'}, + ], + embeds: [{title: 'Embed', image: {url: 'attachment://image.png'}, thumbnail: {url: 'attachment://thumb.png'}}], + }, + [ + {index: 0, filename: 'image.png', data: fileData}, + {index: 1, filename: 'thumb.png', data: fileData}, + ], + ); + expect(response.status).toBe(200); + expect(json.attachments ?? []).toHaveLength(0); + return json; + } + + async function sendVisible(): Promise { + const {response, json} = await sendMessageWithAttachments( + harness, + account.token, + channelId, + {content: 'visible', attachments: [{id: 0, filename: 'visible.png'}]}, + [{index: 0, filename: 'visible.png', data: loadFixture('yeah.png')}], + ); + expect(response.status).toBe(200); + expect(json.attachments).toHaveLength(1); + return json; + } + + async function deleteMessage(messageId: string): Promise { + await createBuilder(harness, account.token) + .delete(`/channels/${channelId}/messages/${messageId}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + } + + it('removes files shown only through an embed when the message is deleted', async () => { + const message = await sendEmbedOnly(); + const keys = [storageKey(message.embeds?.[0]?.image?.url), storageKey(message.embeds?.[0]?.thumbnail?.url)]; + for (const key of keys) { + expect(key.startsWith(`attachments/${channelId}/`)).toBe(true); + expect(stored(key)).toBe(true); + } + await deleteMessage(message.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(keys)); + for (const key of keys) { + expect(stored(key)).toBe(false); + } + }); + + it('keeps owning embed files after an edit that resends the embed', async () => { + const message = await sendEmbedOnly(); + const embed = message.embeds![0]!; + const keys = [storageKey(embed.image?.url), storageKey(embed.thumbnail?.url)]; + const edited = await createBuilder(harness, account.token) + .patch(`/channels/${channelId}/messages/${message.id}`) + .body({embeds: [{title: 'Edited', image: {url: embed.image!.url}, thumbnail: {url: embed.thumbnail!.url}}]}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(edited.embeds?.[0]?.title).toBe('Edited'); + await createBuilder(harness, account.token) + .patch(`/channels/${channelId}/messages/${message.id}`) + .body({content: 'content only edit'}) + .expect(HTTP_STATUS.OK) + .execute(); + await deleteMessage(message.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(keys)); + }); + + it('leaves files of other messages and external media alone', async () => { + const owner = await sendVisible(); + const ownerKey = storageKey(owner.attachments![0]!.url); + const embedOwner = await sendEmbedOnly(); + const embedOwnerKey = storageKey(embedOwner.embeds?.[0]?.image?.url); + const borrower = await createBuilder(harness, account.token) + .post(`/channels/${channelId}/messages`) + .body({ + content: `${Config.endpoints.media}/${ownerKey}`, + embeds: [ + {title: 'Borrowed', image: {url: `${Config.endpoints.media}/${ownerKey}`}}, + {title: 'Borrowed embed', image: {url: `${Config.endpoints.media}/${embedOwnerKey}`}}, + {title: 'External', image: {url: 'https://example.com/external.png'}}, + ], + }) + .expect(HTTP_STATUS.OK) + .execute(); + await deleteMessage(borrower.id); + expect(deletedKeys()).toEqual([]); + expect(stored(ownerKey)).toBe(true); + expect(stored(embedOwnerKey)).toBe(true); + }); + + it('leaves the source file alone when a forward of an embed-only message is deleted', async () => { + const source = await sendEmbedOnly(); + const sourceKey = storageKey(source.embeds?.[0]?.image?.url); + const forward = await createBuilder(harness, account.token) + .post(`/channels/${channelId}/messages`) + .body({ + message_reference: { + message_id: source.id, + channel_id: channelId, + guild_id: guildId, + type: MessageReferenceTypes.FORWARD, + }, + }) + .expect(HTTP_STATUS.OK) + .execute(); + expect(forward.message_snapshots?.[0]?.embeds?.length).toBe(1); + await deleteMessage(forward.id); + expect(deletedKeys()).not.toContain(sourceKey); + expect(stored(sourceKey)).toBe(true); + }); + + async function forward( + sourceId: string, + sourceChannelId: string, + destinationChannelId: string, + ): Promise { + return createBuilder(harness, account.token) + .post(`/channels/${destinationChannelId}/messages`) + .body({ + message_reference: { + message_id: sourceId, + channel_id: sourceChannelId, + guild_id: guildId, + type: MessageReferenceTypes.FORWARD, + }, + }) + .expect(HTTP_STATUS.OK) + .execute(); + } + + async function createAdmin(): Promise { + return setUserACLs(harness, await createTestAccount(harness), [ + AdminACLs.AUTHENTICATE, + AdminACLs.MESSAGE_DELETE, + AdminACLs.CSAM_SUBMIT_NCMEC, + AdminACLs.USER_DELETE, + AdminACLs.ARCHIVE_TRIGGER_USER, + ]); + } + + function embedKeys(message: MessageResponse): Array { + const embed = message.embeds![0]!; + return [storageKey(embed.image?.url), storageKey(embed.thumbnail?.url)]; + } + + it.each([ + ['another channel', true], + ['the same channel', false], + ])('keeps a forwarded embed file in %s after the source is deleted', async (_label, otherChannel) => { + const source = await sendEmbedOnly(); + const sourceKeys = embedKeys(source); + const destinationChannelId = otherChannel + ? (await createChannel(harness, account.token, guildId, 'embed-purge-forward')).id + : channelId; + const forwarded = await forward(source.id, channelId, destinationChannelId); + const snapshotEmbed = forwarded.message_snapshots![0]!.embeds![0]!; + expect(snapshotEmbed.image?.flags).toBe(0); + const copyKeys = [storageKey(snapshotEmbed.image?.url), storageKey(snapshotEmbed.thumbnail?.url)]; + for (const key of copyKeys) { + expect(key.startsWith(`attachments/${destinationChannelId}/`)).toBe(true); + expect(sourceKeys).not.toContain(key); + } + await deleteMessage(source.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(sourceKeys)); + for (const key of copyKeys) { + expect(stored(key)).toBe(true); + } + const reforwarded = await forward(forwarded.id, destinationChannelId, channelId); + const reforwardedEmbed = reforwarded.message_snapshots![0]!.embeds![0]!; + const reforwardedKeys = [storageKey(reforwardedEmbed.image?.url), storageKey(reforwardedEmbed.thumbnail?.url)]; + for (const key of reforwardedKeys) { + expect(key.startsWith(`attachments/${channelId}/`)).toBe(true); + expect(copyKeys).not.toContain(key); + } + await createBuilder(harness, account.token) + .delete(`/channels/${destinationChannelId}/messages/${forwarded.id}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + expect(deletedKeys()).toEqual(expect.arrayContaining(copyKeys)); + for (const key of reforwardedKeys) { + expect(stored(key)).toBe(true); + } + }); + + it('removes files shown only through an embed on an admin delete', async () => { + const message = await sendEmbedOnly(); + const keys = embedKeys(message); + const admin = await createAdmin(); + await createBuilder(harness, admin.token) + .delete(`/admin/channels/${channelId}/messages/${message.id}`) + .expect(HTTP_STATUS.OK) + .execute(); + expect(deletedKeys()).toEqual(expect.arrayContaining(keys)); + }); + + it('removes files shown only through an embed on a silent NCMEC delete', async () => { + const message = await sendEmbedOnly(); + const keys = embedKeys(message); + const service = getNcmecSubmissionService() as unknown as { + deleteMessageSilently(channelId: ChannelID, messageId: MessageID, fallbackUserId: UserID): Promise; + }; + await service.deleteMessageSilently( + createChannelID(BigInt(channelId)), + createMessageID(BigInt(message.id)), + createUserID(BigInt(account.userId)), + ); + expect(deletedKeys()).toEqual(expect.arrayContaining(keys)); + }); + + it('keeps reported embed files as report evidence after the author deletes the message', async () => { + const message = await sendEmbedOnly(); + const [imageKey] = embedKeys(message); + const [, , attachmentId, filename] = imageKey!.split('/'); + const reporter = await createTestAccount(harness); + const invite = await createChannelInvite(harness, account.token, channelId); + await acceptInvite(harness, reporter.token, invite.code); + const report = await createBuilder<{report_id: string}>(harness, reporter.token) + .post('/reports/message') + .body({channel_id: channelId, message_id: message.id, category: 'harassment'}) + .expect(HTTP_STATUS.OK) + .execute(); + await deleteMessage(message.id); + expect(stored(imageKey!)).toBe(false); + expect(harness.storageService.hasObject(Config.s3.buckets.reports, imageKey!)).toBe(true); + const row = await new ReportRepository().getReport(createReportID(BigInt(report.report_id))); + const context = row!.messageContext!.find((entry) => entry.messageId.toString() === message.id); + expect(context?.attachments.map((attachment) => attachment.filename).sort()).toEqual(['image.png', 'thumb.png']); + const admin = await createAdmin(); + await createBuilder(harness, admin.token) + .post('/admin/messages/ncmec-reports') + .body({ + channel_id: channelId, + message_id: message.id, + attachment_id: attachmentId, + filename, + reporter_full_name: 'Embed Reporter', + confirmed_viewed: true, + source_report_id: report.report_id, + }) + .expect(HTTP_STATUS.OK) + .execute(); + }); + + it('does not expose the ownership marker on embed media flags', async () => { + const message = await sendEmbedOnly(); + expect(message.embeds?.[0]?.image?.flags).toBe(0); + expect(message.embeds?.[0]?.thumbnail?.flags).toBe(0); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/MessageCrosspost.test.ts b/fluxer_api/src/api/channel/tests/MessageCrosspost.test.ts new file mode 100644 index 000000000..c1f541ac9 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/MessageCrosspost.test.ts @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createAttachmentID, createChannelID, createGuildID, createMessageID, createUserID} from '@app/api/BrandedTypes'; +import {enqueueCrosspostFamilyPurgeFromCopies} from '@app/api/channel/services/message/CrosspostPropagation'; +import {loadFixture, sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils'; +import { + createChannel, + createPermissionOverwrite as createChannelPermissionOverwrite, + pinMessage, +} from '@app/api/channel/tests/ChannelTestUtils'; +import { + type AnnouncementWorld, + addGuildFeature, + channelBudgetRemaining, + createAnnouncementWorld, + createTestChannelService, + crosspostRequest, + editMessage, + listCrosspostSources, + patchGuildRow, + postMessage, + publish, + RecordingWorkerService, + readMessageRow, + writeMessageRow, +} from '@app/api/channel/tests/CrosspostTestUtils'; +import {Logger} from '@app/api/Logger'; +import {getMessages} from '@app/api/message/tests/MessageTestUtils'; +import {fileShaCache} from '@app/api/middleware/FileShaCache'; +import {phraseBlocklistCache} from '@app/api/middleware/PhraseBlocklistCache'; +import {getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import type {MockSnowflakeService} from '@app/api/test/mocks/MockSnowflakeService'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {createWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError'; +import {CROSSPOST_CHANNEL_RATE_LIMIT} from '@fluxer/constants/src/AnnouncementConstants'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes, MessageFlags, MessageTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures, GuildOperations} from '@fluxer/constants/src/GuildConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +const SERVER_BITS = MessageFlags.CROSSPOSTED | MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED; + +describe('Publishing messages in announcement channels', () => { + let harness: ApiTestHarness; + let worker: RecordingWorkerService; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + worker = new RecordingWorkerService(); + setInjectedWorkerService(worker); + world = await createAnnouncementWorld(harness); + }); + + afterEach(() => { + setInjectedWorkerService(new NoopWorkerService()); + phraseBlocklistCache.remove('forbiddenword'); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('the author publishes their own message', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Release 1.0 is out'); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const infoSpy = vi.spyOn(Logger.child({}), 'info'); + const published = await publish(harness, world.author.token, world.announcement.id, message.id); + expect(published.id).toBe(message.id); + expect(published.flags & MessageFlags.CROSSPOSTED).toBe(MessageFlags.CROSSPOSTED); + expect(published.edited_timestamp ?? null).toBeNull(); + const row = await readMessageRow(world.announcement.id, message.id); + expect(row?.flags).toBe(MessageFlags.CROSSPOSTED); + expect(row?.editedTimestamp).toBeNull(); + const updates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'MESSAGE_UPDATE'); + expect(updates).toHaveLength(1); + expect(updates[0]?.data).toMatchObject({id: message.id, flags: MessageFlags.CROSSPOSTED}); + expect(infoSpy).toHaveBeenCalledWith( + { + actorId: world.author.userId, + guildId: world.guild.id, + channelId: world.announcement.id, + messageId: message.id, + }, + 'message published', + ); + }); + + test('publishing another member message needs Manage Messages and Send Messages', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Weekly update'); + await crosspostRequest(harness, world.member.token, world.announcement.id, message.id) + .expect(403, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await createChannelPermissionOverwrite(harness, world.owner.token, world.announcement.id, world.modRoleId, { + type: 0, + allow: '0', + deny: Permissions.SEND_MESSAGES.toString(), + }); + await crosspostRequest(harness, world.mod.token, world.announcement.id, message.id) + .expect(403, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await createChannelPermissionOverwrite(harness, world.owner.token, world.announcement.id, world.modRoleId, { + type: 0, + allow: '0', + deny: '0', + }); + const published = await publish(harness, world.mod.token, world.announcement.id, message.id); + expect(published.flags & MessageFlags.CROSSPOSTED).toBe(MessageFlags.CROSSPOSTED); + }); + + test('the author cannot publish without Send Messages', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Muted soon'); + await createChannelPermissionOverwrite(harness, world.owner.token, world.announcement.id, world.author.userId, { + type: 1, + allow: '0', + deny: Permissions.SEND_MESSAGES.toString(), + }); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(403, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + }); + + test('publishing an incoming webhook message needs Manage Messages', async () => { + const webhook = await createWebhook(harness, world.announcement.id, world.owner.token, 'Status Bot'); + const executed = await createBuilderWithoutAuth(harness) + .post(`/webhooks/${webhook.id}/${webhook.token}?wait=true`) + .body({content: 'Status: all systems normal'}) + .expect(200) + .execute(); + await crosspostRequest(harness, world.member.token, world.announcement.id, executed.id) + .expect(403, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + const published = await publish(harness, world.mod.token, world.announcement.id, executed.id); + expect(published.flags & MessageFlags.CROSSPOSTED).toBe(MessageFlags.CROSSPOSTED); + }); + + test('a message can only be published once', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Once only'); + await publish(harness, world.author.token, world.announcement.id, message.id); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(400, APIErrorCodes.MESSAGE_ALREADY_CROSSPOSTED) + .execute(); + expect(worker.byTask('crosspostMessage')).toHaveLength(1); + }); + + test('publishing in a text channel is rejected', async () => { + const message = await postMessage(harness, world.author.token, world.text.id, 'Not news'); + await crosspostRequest(harness, world.author.token, world.text.id, message.id) + .expect(400, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + const row = await readMessageRow(world.text.id, message.id); + expect(row?.flags).toBe(0); + }); + + test('replies, forwards, follow notices and pin notices cannot be published', async () => { + const original = await postMessage(harness, world.author.token, world.announcement.id, 'Original'); + const reply = await createBuilder(harness, world.author.token) + .post(`/channels/${world.announcement.id}/messages`) + .body({content: 'A reply', message_reference: {message_id: original.id}}) + .expect(200) + .execute(); + expect(reply.type).toBe(MessageTypes.REPLY); + const forward = await createBuilder(harness, world.author.token) + .post(`/channels/${world.announcement.id}/messages`) + .body({ + message_reference: { + type: 1, + message_id: original.id, + channel_id: world.announcement.id, + guild_id: world.guild.id, + }, + }) + .expect(200) + .execute(); + expect(forward.message_snapshots?.length ?? 0).toBeGreaterThan(0); + const followNotice = await postMessage(harness, world.author.token, world.announcement.id, 'Follow notice'); + const followRow = await readMessageRow(world.announcement.id, followNotice.id); + await writeMessageRow(followRow!, {type: MessageTypes.CHANNEL_FOLLOW_ADD}); + await pinMessage(harness, world.owner.token, world.announcement.id, original.id); + const messages = await getMessages(harness, world.owner.token, world.announcement.id); + const pinNotice = messages.find((candidate) => candidate.type === MessageTypes.CHANNEL_PINNED_MESSAGE); + expect(pinNotice).toBeDefined(); + for (const id of [reply.id, forward.id, followNotice.id, pinNotice!.id]) { + await crosspostRequest(harness, world.owner.token, world.announcement.id, id) + .expect(400, APIErrorCodes.MESSAGE_NOT_CROSSPOSTABLE) + .execute(); + } + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + }); + + test('a copy cannot be published again', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Copied text'); + const row = await readMessageRow(world.announcement.id, message.id); + await writeMessageRow(row!, {flags: MessageFlags.IS_CROSSPOST}); + await crosspostRequest(harness, world.owner.token, world.announcement.id, message.id) + .expect(400, APIErrorCodes.MESSAGE_NOT_CROSSPOSTABLE) + .execute(); + }); + + test('unknown messages and channels return 404', async () => { + await crosspostRequest(harness, world.author.token, world.announcement.id, '123456789012345678') + .expect(404, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + await crosspostRequest(harness, world.author.token, '123456789012345678', '123456789012345679') + .expect(404, APIErrorCodes.UNKNOWN_CHANNEL) + .execute(); + }); + + test('a community with sending disabled or the kill switch cannot publish', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Blocked update'); + await patchGuildRow(world.guild.id, {disabled_operations: GuildOperations.SEND_MESSAGE}); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(403, APIErrorCodes.FEATURE_TEMPORARILY_DISABLED) + .execute(); + await patchGuildRow(world.guild.id, {disabled_operations: 0}); + await addGuildFeature(world.guild.id, GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(403, APIErrorCodes.FEATURE_TEMPORARILY_DISABLED) + .execute(); + expect((await readMessageRow(world.announcement.id, message.id))?.flags).toBe(0); + }); + + test('clients cannot set server-owned flags, and a suppress-embeds toggle keeps the published flag', async () => { + const sent = await createBuilder(harness, world.author.token) + .post(`/channels/${world.announcement.id}/messages`) + .body({content: 'Flag probe', flags: SERVER_BITS}) + .expect(200) + .execute(); + expect(sent.flags & SERVER_BITS).toBe(0); + expect((await readMessageRow(world.announcement.id, sent.id))?.flags).toBe(0); + const edited = await editMessage(harness, world.author.token, world.announcement.id, sent.id, { + flags: SERVER_BITS | MessageFlags.SUPPRESS_EMBEDS, + }) + .expect(200) + .execute(); + expect(edited.flags).toBe(MessageFlags.SUPPRESS_EMBEDS); + await publish(harness, world.author.token, world.announcement.id, sent.id); + const toggled = await editMessage(harness, world.author.token, world.announcement.id, sent.id, {flags: 0}) + .expect(200) + .execute(); + expect(toggled.flags).toBe(MessageFlags.CROSSPOSTED); + const moderated = await editMessage(harness, world.mod.token, world.announcement.id, sent.id, { + flags: MessageFlags.SUPPRESS_EMBEDS | SERVER_BITS, + }) + .expect(200) + .execute(); + expect(moderated.flags).toBe(MessageFlags.CROSSPOSTED | MessageFlags.SUPPRESS_EMBEDS); + }); + + test('publishing enqueues the fan-out job and indexes the source', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Fan out'); + await publish(harness, world.author.token, world.announcement.id, message.id); + const jobs = worker.byTask('crosspostMessage'); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.payload).toEqual({channelId: world.announcement.id, messageId: message.id}); + expect(jobs[0]?.options?.jobKey).toBe(`crosspost:${message.id}`); + expect(jobs[0]?.options?.skipLedger).toBe(true); + expect(await listCrosspostSources(world.announcement.id)).toEqual([message.id]); + }); + + test('repeated queue failures never use up the channel allowance', async () => { + worker.failure = (taskType) => + taskType === 'crosspostMessage' ? new WorkerQueueOverflowError(taskType, 'stream full') : null; + const attempts = CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts + 2; + for (let index = 0; index < attempts; index++) { + const message = await postMessage(harness, world.author.token, world.announcement.id, `Overflow ${index}`); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(503, APIErrorCodes.SERVICE_UNAVAILABLE) + .execute(); + } + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + worker.failure = null; + const message = await postMessage(harness, world.author.token, world.announcement.id, 'After the overflow'); + await publish(harness, world.author.token, world.announcement.id, message.id); + }); + + test('a full queue returns 503 and leaves the message unpublished', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Overflow'); + worker.failure = (taskType) => + taskType === 'crosspostMessage' ? new WorkerQueueOverflowError(taskType, 'stream full') : null; + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await crosspostRequest(harness, world.author.token, world.announcement.id, message.id) + .expect(503, APIErrorCodes.SERVICE_UNAVAILABLE) + .execute(); + expect((await readMessageRow(world.announcement.id, message.id))?.flags).toBe(0); + expect(await listCrosspostSources(world.announcement.id)).toEqual([]); + const updates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'MESSAGE_UPDATE'); + expect(updates.map((update) => (update.data as {flags: number}).flags)).toEqual([MessageFlags.CROSSPOSTED, 0]); + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + worker.failure = null; + const published = await publish(harness, world.author.token, world.announcement.id, message.id); + expect(published.flags & MessageFlags.CROSSPOSTED).toBe(MessageFlags.CROSSPOSTED); + expect(worker.byTask('crosspostMessage')).toHaveLength(1); + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts - 1); + }); + + test('a message with large attachments can be published', async () => { + const {json} = await sendMessageWithAttachments( + harness, + world.author.token, + world.announcement.id, + {content: 'Big file', attachments: [{id: 0, filename: 'yeah.png'}]}, + [{index: 0, filename: 'yeah.png', data: loadFixture('yeah.png')}], + ); + const row = (await readMessageRow(world.announcement.id, json.id))!; + const large = 2n * 1024n * 1024n * 1024n; + await writeMessageRow(row, { + attachments: [ + {...row.attachments[0]!.toMessageAttachment(), size: large}, + { + ...row.attachments[0]!.toMessageAttachment(), + attachment_id: createAttachmentID(row.attachments[0]!.id + 1n), + filename: 'second.png', + size: large, + }, + ], + }); + const published = await publish(harness, world.author.token, world.announcement.id, json.id); + expect(published.flags & MessageFlags.CROSSPOSTED).toBe(MessageFlags.CROSSPOSTED); + expect(published.attachments).toHaveLength(2); + expect(worker.byTask('crosspostMessage')).toHaveLength(1); + }); + + test('blocklisted content cannot be published and does not consume budget', async () => { + const plain = await postMessage(harness, world.author.token, world.announcement.id, 'says forbiddenword here'); + const embedded = await createBuilder(harness, world.author.token) + .post(`/channels/${world.announcement.id}/messages`) + .body({embeds: [{title: 'Weekly', fields: [{name: 'Notes', value: 'contains forbiddenword'}]}]}) + .expect(200) + .execute(); + const {json: withFile} = await sendMessageWithAttachments( + harness, + world.author.token, + world.announcement.id, + {content: 'Picture', attachments: [{id: 0, filename: 'yeah.png'}]}, + [{index: 0, filename: 'yeah.png', data: loadFixture('yeah.png')}], + ); + const fileRow = await readMessageRow(world.announcement.id, withFile.id); + const blockedSha = 'ab'.repeat(32); + await writeMessageRow(fileRow!, { + attachments: fileRow!.attachments.map((attachment) => ({ + ...attachment.toMessageAttachment(), + content_hash: blockedSha, + })), + }); + phraseBlocklistCache.add('forbiddenword'); + fileShaCache.add(blockedSha); + try { + for (const id of [plain.id, embedded.id, withFile.id]) { + await crosspostRequest(harness, world.author.token, world.announcement.id, id) + .expect(403, APIErrorCodes.CONTENT_BLOCKED) + .execute(); + } + } finally { + fileShaCache.remove(blockedSha); + } + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + }); + + test('files shown only through embeds count toward the hash blocklist', async () => { + const {json} = await sendMessageWithAttachments( + harness, + world.owner.token, + world.announcement.id, + { + content: 'Card', + embeds: [{title: 'Card', image: {url: 'attachment://yeah.png'}}], + attachments: [{id: 0, filename: 'yeah.png'}], + }, + [{index: 0, filename: 'yeah.png', data: loadFixture('yeah.png')}], + ); + const row = (await readMessageRow(world.announcement.id, json.id))!; + expect(row.attachments).toHaveLength(0); + const embedHash = row.embeds[0]!.toMessageEmbed().image?.content_hash; + expect(embedHash).toBeTruthy(); + fileShaCache.add(embedHash!); + try { + await crosspostRequest(harness, world.owner.token, world.announcement.id, json.id) + .expect(403, APIErrorCodes.CONTENT_BLOCKED) + .execute(); + } finally { + fileShaCache.remove(embedHash!); + } + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts); + await publish(harness, world.owner.token, world.announcement.id, json.id); + }); + + test('the history cutoff applies to moderators without Read Message History', async () => { + const older = await postMessage(harness, world.author.token, world.announcement.id, 'Old news'); + const olderTimestamp = snowflakeToDate(BigInt(older.id)).getTime(); + const snowflakes = getSnowflakeService() as MockSnowflakeService; + snowflakes.configure({startTimestampMs: olderTimestamp + 20_000}); + const newer = await postMessage(harness, world.author.token, world.announcement.id, 'Fresh news'); + await patchGuildRow(world.guild.id, {message_history_cutoff: new Date(olderTimestamp + 10_000)}); + await createChannelPermissionOverwrite(harness, world.owner.token, world.announcement.id, world.mod.userId, { + type: 1, + allow: Permissions.VIEW_CHANNEL.toString(), + deny: Permissions.READ_MESSAGE_HISTORY.toString(), + }); + await crosspostRequest(harness, world.mod.token, world.announcement.id, older.id) + .expect(404, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + await publish(harness, world.mod.token, world.announcement.id, newer.id); + }); + + test('two concurrent publishes of one message succeed once and consume one slot', async () => { + const message = await postMessage(harness, world.author.token, world.announcement.id, 'Race'); + const attempt = () => crosspostRequest(harness, world.author.token, world.announcement.id, message.id).executeRaw(); + const results = await Promise.all([attempt(), attempt()]); + const statuses = results.map((result) => result.response.status).sort(); + expect(statuses).toEqual([200, 400]); + const rejected = results.find((result) => result.response.status === 400); + expect((rejected?.json as {code?: string} | undefined)?.code).toBe(APIErrorCodes.MESSAGE_ALREADY_CROSSPOSTED); + expect(await channelBudgetRemaining(world.announcement.id)).toBe(CROSSPOST_CHANNEL_RATE_LIMIT.maxAttempts - 1); + expect(worker.byTask('crosspostMessage')).toHaveLength(1); + }); + + test('a second announcement channel publishes independently', async () => { + const second = await createChannel( + harness, + world.owner.token, + world.guild.id, + 'news-two', + ChannelTypes.GUILD_ANNOUNCEMENT, + ); + const message = await postMessage(harness, world.author.token, second.id, 'Second channel'); + await publish(harness, world.author.token, second.id, message.id); + expect(await listCrosspostSources(second.id)).toEqual([message.id]); + expect(await listCrosspostSources(world.announcement.id)).toEqual([]); + }); + + test('every delete path of an announcement message enqueues a source-deleted sync', async () => { + const sources = []; + for (let index = 0; index < 5; index++) { + const message = await postMessage(harness, world.author.token, world.announcement.id, `Source ${index}`); + await publish(harness, world.author.token, world.announcement.id, message.id); + sources.push(message.id); + } + const unpublished = await postMessage(harness, world.author.token, world.announcement.id, 'Never published'); + const textMessage = await postMessage(harness, world.author.token, world.text.id, 'Plain text channel'); + const webhook = await createWebhook(harness, world.announcement.id, world.owner.token, 'Digest'); + const webhookMessage = await createBuilderWithoutAuth(harness) + .post(`/webhooks/${webhook.id}/${webhook.token}?wait=true`) + .body({content: 'Digest'}) + .expect(200) + .execute(); + await publish(harness, world.mod.token, world.announcement.id, webhookMessage.id); + await createBuilder(harness, world.mod.token) + .delete(`/channels/${world.announcement.id}/messages/${sources[0]}`) + .expect(204) + .execute(); + await createBuilder(harness, world.mod.token) + .post(`/channels/${world.announcement.id}/messages/bulk-delete`) + .body({message_ids: [sources[1], unpublished.id]}) + .expect(204) + .execute(); + await createBuilder(harness, world.mod.token) + .post(`/channels/${world.text.id}/messages/bulk-delete`) + .body({message_ids: [textMessage.id]}) + .expect(204) + .execute(); + await createBuilderWithoutAuth(harness) + .delete(`/webhooks/${webhook.id}/${webhook.token}/messages/${webhookMessage.id}`) + .expect(204) + .execute(); + const channelService = createTestChannelService(); + await channelService.messages.deletion.deleteUserMessagesInGuild({ + userId: createUserID(BigInt(world.author.userId)), + guildId: createGuildID(BigInt(world.guild.id)), + seconds: 3600, + }); + const removalsFor = (messageId: string) => + worker.syncJobsFor(messageId).filter((job) => job.payload.mode === 'source_deleted'); + for (const messageId of [...sources, webhookMessage.id, unpublished.id]) { + const removals = removalsFor(messageId); + expect(removals.length).toBeGreaterThanOrEqual(1); + expect(removals[0]?.payload).toEqual({ + channelId: world.announcement.id, + messageId, + mode: 'source_deleted', + }); + expect(removals[0]?.options?.jobKey).toBe(`crosspost-sync:${messageId}:source_deleted`); + expect(removals[0]?.options?.skipLedger).toBe(true); + } + expect(worker.syncJobsFor(textMessage.id)).toHaveLength(0); + }); + + test('bulk deleting your own messages enqueues a source-deleted sync for announcement ones', async () => { + const published = await postMessage(harness, world.author.token, world.announcement.id, 'Mine, published'); + await publish(harness, world.author.token, world.announcement.id, published.id); + const unpublished = await postMessage(harness, world.author.token, world.announcement.id, 'Mine, unpublished'); + const plain = await postMessage(harness, world.author.token, world.text.id, 'Mine, plain'); + const deleted = await createTestChannelService().userMessageDeletion.deleteUserMessagesBulk( + createUserID(BigInt(world.author.userId)), + ); + expect(deleted).toBeGreaterThanOrEqual(3); + expect(await readMessageRow(world.announcement.id, published.id)).toBeNull(); + expect(worker.syncJobsFor(published.id).map((job) => job.payload.mode)).toEqual(['source_deleted']); + expect(worker.syncJobsFor(unpublished.id).map((job) => job.payload.mode)).toEqual(['source_deleted']); + expect(worker.syncJobsFor(plain.id)).toHaveLength(0); + }); + + test('the reverse cascade helper purges each copied family once', async () => { + const source = await postMessage(harness, world.author.token, world.announcement.id, 'Source'); + const copy = await postMessage(harness, world.author.token, world.text.id, 'Copy'); + const plain = await postMessage(harness, world.author.token, world.text.id, 'Plain'); + const copyRow = await readMessageRow(world.text.id, copy.id); + await writeMessageRow(copyRow!, { + flags: MessageFlags.IS_CROSSPOST, + message_reference: { + channel_id: createChannelID(BigInt(world.announcement.id)), + message_id: createMessageID(BigInt(source.id)), + guild_id: createGuildID(BigInt(world.guild.id)), + type: 0, + }, + }); + const rows = await Promise.all([ + readMessageRow(world.text.id, copy.id), + readMessageRow(world.text.id, copy.id), + readMessageRow(world.text.id, plain.id), + ]); + await enqueueCrosspostFamilyPurgeFromCopies(worker, {messages: rows.filter((row) => row !== null)}); + const syncs = worker.byTask('syncCrosspostedMessage'); + expect(syncs).toHaveLength(1); + expect(syncs[0]?.payload).toEqual({ + channelId: world.announcement.id, + messageId: source.id, + mode: 'purge', + deleteSource: true, + }); + expect(syncs[0]?.options?.jobKey).toBe(`crosspost-sync:${source.id}:purge`); + }); +}); diff --git a/fluxer_api/src/api/channel/tests/MessageCrosspostFanout.test.ts b/fluxer_api/src/api/channel/tests/MessageCrosspostFanout.test.ts new file mode 100644 index 000000000..9604e9ac3 --- /dev/null +++ b/fluxer_api/src/api/channel/tests/MessageCrosspostFanout.test.ts @@ -0,0 +1,1060 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {AdminRepository} from '@app/api/admin/AdminRepository'; +import {setUserACLs} from '@app/api/auth/tests/AuthTestUtils'; +import { + createAttachmentID, + createChannelID, + createGuildID, + createMessageID, + createStickerID, + createWebhookID, +} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import {enqueueCrosspostSync} from '@app/api/channel/services/message/CrosspostPropagation'; +import {MessagePersistenceService} from '@app/api/channel/services/message/MessagePersistenceService'; +import {MessageSearchService} from '@app/api/channel/services/message/MessageSearchService'; +import {MessageWriteLock} from '@app/api/channel/services/message/MessageWriteLock'; +import { + addGuildMember, + createAnnouncementTargetGuild, + createGuildChannel, + disableCrosspostWorker, + findWebhookRow, + follow, + publish, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {loadFixture} from '@app/api/channel/tests/AttachmentTestUtils'; +import {createPermissionOverwrite, updateGuild} from '@app/api/channel/tests/ChannelTestUtils'; +import {addGuildFeature, createStallGate, patchGuildRow} from '@app/api/channel/tests/CrosspostTestUtils'; +import { + copiesOf, + deleteMessageRequest, + editRequest, + type FanoutWorld, + followInto, + listCopies, + mappingRow, + patchChannel, + patchChannelRow, + postAndPublish, + publishAndDrain, + readRow, + sendMessage, + sendWithImage, + setupFanoutWorld, + workerHelpers, + writeRow, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils'; +import {Logger} from '@app/api/Logger'; +import {getMessages} from '@app/api/message/tests/MessageTestUtils'; +import {startContentBlocklistCaches, stopContentBlocklistCaches} from '@app/api/middleware/ContentBlocklistCaches'; +import {fileShaCache} from '@app/api/middleware/FileShaCache'; +import {getKVClient} from '@app/api/middleware/ServiceRegistry'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk'; +import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import {CROSSPOST_SOURCE_DELETED_CONTENT} from '@fluxer/constants/src/AnnouncementConstants'; +import { + ChannelTypes, + MessageAttachmentFlags, + MessageFlags, + MessageTypes, + Permissions, +} from '@fluxer/constants/src/ChannelConstants'; +import { + ContentWarningLevel, + GuildExplicitContentFilterTypes, + GuildFeatures, + GuildOperations, +} from '@fluxer/constants/src/GuildConstants'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import sharp from 'sharp'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +function createBarrier(parties: number): () => Promise { + let arrived = 0; + let open: () => void = () => {}; + const opened = new Promise((resolve) => { + open = resolve; + }); + return async () => { + arrived++; + if (arrived >= parties) open(); + await opened; + }; +} + +const fanoutSizes = vi.hoisted(() => ({chunk: null as number | null, page: null as number | null})); + +vi.mock('@fluxer/constants/src/AnnouncementConstants', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + get CROSSPOST_FANOUT_CHUNK_SIZE() { + return fanoutSizes.chunk ?? actual.CROSSPOST_FANOUT_CHUNK_SIZE; + }, + get CROSSPOST_FANOUT_PAGE_SIZE() { + return fanoutSizes.page ?? actual.CROSSPOST_FANOUT_PAGE_SIZE; + }, + }; +}); + +describe('Crosspost fan-out', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + fanoutSizes.chunk = null; + fanoutSizes.page = null; + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function createTextChannel(name: string): Promise { + return createGuildChannel(harness, world.b.owner.token, world.b.guild.id, {name, type: ChannelTypes.GUILD_TEXT}); + } + + async function setSourceIcon(icon: string | null): Promise { + const updated = await updateGuild(harness, world.a.owner.token, world.a.guild.id, {icon}); + return updated.icon ?? null; + } + + async function solidPngDataUrl(background: string): Promise { + const png = await sharp({create: {width: 32, height: 32, channels: 3, background}}) + .png() + .toBuffer(); + return `data:image/png;base64,${png.toString('base64')}`; + } + + function webhookAvatarCopies(webhookId: string): Array<{sourceKey: string; destinationKey: string}> { + return harness.storageService + .getCopiedObjects() + .filter((copy) => copy.destinationKey.startsWith(`avatars/${webhookId}/`)); + } + + function attachmentObjectCopies(): Array { + return harness.storageService + .getCopiedObjects() + .filter((entry) => entry.sourceBucket === Config.s3.buckets.cdn && entry.sourceKey.startsWith('attachments/')) + .map((entry) => entry.destinationKey); + } + + function deletedKeys(): Array { + return harness.storageService.getDeletedObjects().map((entry) => entry.key); + } + + function failNextCopyCreate(webhookId?: string): void { + const original = MessagePersistenceService.prototype.createMessage; + let failed = false; + vi.spyOn(MessagePersistenceService.prototype, 'createMessage').mockImplementation(async function ( + this: MessagePersistenceService, + params, + ) { + const matches = webhookId ? params.webhookId?.toString() === webhookId : params.webhookId != null; + if (!failed && matches) { + failed = true; + throw new Error('database hiccup'); + } + return original.call(this, params); + }); + } + + function stallNextCopyCreate(): ReturnType { + const gate = createStallGate(); + const original = MessagePersistenceService.prototype.createMessage; + let stalled = false; + vi.spyOn(MessagePersistenceService.prototype, 'createMessage').mockImplementation(async function ( + this: MessagePersistenceService, + params, + ) { + if (!stalled && params.webhookId != null) { + stalled = true; + await gate.hit(); + } + return original.call(this, params); + }); + return gate; + } + + async function expectNoCopies(channelId: string): Promise { + expect(await listCopies(harness, world.b.owner.token, channelId)).toHaveLength(0); + expect(world.worker.deadLetters).toHaveLength(0); + } + + test('one follower receives a copy with the published wire shape', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const source = await postAndPublish(harness, world, {content: 'Version 2 is out'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy).toBeDefined(); + expect(copy!.type).toBe(MessageTypes.DEFAULT); + expect(copy!.webhook_id).toBe(webhookId); + expect(copy!.author.id).toBe(webhookId); + expect(copy!.author.username).toBe(`${world.a.guild.name} #${world.a.ann.name}`); + expect(copy!.author.bot).toBe(true); + expect(copy!.flags).toBe(MessageFlags.IS_CROSSPOST); + expect(copy!.content).toBe('Version 2 is out'); + expect(copy!.message_reference).toEqual({ + type: 0, + guild_id: world.a.guild.id, + channel_id: world.a.ann.id, + message_id: source.id, + }); + expect('referenced_message' in copy!).toBe(false); + const row = await mappingRow(source.id, webhookId); + expect(row?.state).toBe('delivered'); + expect(row?.target_message_id.toString()).toBe(copy!.id); + expect(row?.target_guild_id.toString()).toBe(world.b.guild.id); + }); + + test('copies store no mentions and reading them does not repair any', async () => { + await followInto(harness, world, world.b.t1.id); + const content = `@everyone <@&${world.a.guild.id}> <@${world.a.member.userId}> hello`; + const source = await postAndPublish(harness, world, {content}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy!.content).toBe(content); + expect(copy!.mention_everyone).toBe(false); + expect(copy!.mentions).toEqual([]); + expect(copy!.mention_roles).toEqual([]); + const before = await readRow(world.b.t1.id, copy!.id); + for (let index = 0; index < 3; index++) { + await createBuilder(harness, world.b.owner.token) + .get(`/channels/${world.b.t1.id}/messages/${copy!.id}`) + .execute(); + await getMessages(harness, world.b.owner.token, world.b.t1.id); + } + const after = await readRow(world.b.t1.id, copy!.id); + expect(after?.version).toBe(before?.version); + expect(after?.mentionEveryone).toBe(false); + expect(after?.mentionedUserIds.size).toBe(0); + expect(after?.mentionedRoleIds.size).toBe(0); + }); + + test('sendable flags carry over to the copy', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'quiet'}); + const sendable = MessageFlags.SUPPRESS_EMBEDS | MessageFlags.SUPPRESS_NOTIFICATIONS | MessageFlags.VOICE_MESSAGE; + await writeRow((await readRow(world.a.ann.id, message.id))!, {flags: sendable}); + await publishAndDrain(harness, world, message.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.flags).toBe(MessageFlags.IS_CROSSPOST | sendable); + }); + + test('copies carry the source attachments and resolve to the source channel', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'files'}, [ + 'first.png', + 'second.gif', + ]); + const row = (await readRow(world.a.ann.id, message.id))!; + await writeRow(row, { + attachments: row.attachments.map((attachment, index) => ({ + ...attachment.toMessageAttachment(), + flags: index === 0 ? MessageAttachmentFlags.IS_SPOILER : 0, + })), + }); + await publishAndDrain(harness, world, message.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + const sourceAttachments = (await readRow(world.a.ann.id, message.id))!.attachments; + const copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.attachments.map((attachment) => attachment.toMessageAttachment())).toEqual( + sourceAttachments.map((attachment) => attachment.toMessageAttachment()), + ); + expect(copy!.attachments).toHaveLength(2); + copy!.attachments!.forEach((attachment, index) => { + const original = sourceAttachments[index]!; + expect(attachment.id).toBe(original.id.toString()); + expect(attachment.url).toContain(`/attachments/${world.a.ann.id}/${original.id}/${original.filename}`); + expect(attachment.url).not.toContain(`/attachments/${world.b.t1.id}/`); + expect(attachment.proxy_url).toBe(attachment.url); + }); + expect(attachmentObjectCopies()).toEqual([]); + }); + + test('rich embeds are copied with their source attachment urls', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, { + content: 'card', + embeds: [{title: 'Release card', description: 'All the details', image: {url: 'attachment://yeah.png'}}], + }); + const sourceRow = (await readRow(world.a.ann.id, message.id))!; + expect(sourceRow.attachments).toHaveLength(0); + const sourceImageUrl = sourceRow.embeds[0]!.image!.url!; + expect(sourceImageUrl.startsWith(`${Config.endpoints.media}/attachments/${world.a.ann.id}/`)).toBe(true); + const sourceKey = sourceImageUrl.slice(`${Config.endpoints.media}/`.length); + await publishAndDrain(harness, world, message.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + const copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.embeds[0]!.title).toBe('Release card'); + expect(copyRow.embeds[0]!.description).toBe('All the details'); + expect(copyRow.embeds[0]!.image!.url).toBe(sourceImageUrl); + expect(copy!.embeds?.[0]?.image?.url).toContain(`/${sourceKey}`); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, sourceKey)).toBe(true); + expect(attachmentObjectCopies()).toEqual([]); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain(); + expect(deletedKeys().some((key) => key.startsWith(`attachments/${world.b.t1.id}/`))).toBe(false); + const marked = (await readRow(world.b.t1.id, copy!.id))!; + expect(marked.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED); + }); + + test('favoriting the embed image of a copy by embed index copies the source object', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, { + content: 'card', + embeds: [{title: 'Release card', image: {url: 'attachment://yeah.png'}}], + }); + const sourceRow = (await readRow(world.a.ann.id, message.id))!; + const sourceKey = sourceRow.embeds[0]!.image!.url!.slice(`${Config.endpoints.media}/`.length); + await publishAndDrain(harness, world, message.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + await createBuilder(harness, world.b.owner.token) + .post(`/channels/${world.b.t1.id}/messages/${copy!.id}/memes`) + .body({embed_index: 0, name: 'Release card'}) + .expect(201) + .execute(); + const fromSource = harness.storageService + .getCopiedObjects() + .filter((entry) => entry.sourceBucket === Config.s3.buckets.cdn && entry.sourceKey === sourceKey); + expect(fromSource).toHaveLength(1); + }); + + test('stickers are copied', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'sticker'}); + const stickerId = createStickerID(123456789012345678n); + await writeRow((await readRow(world.a.ann.id, message.id))!, { + sticker_items: [{sticker_id: stickerId, name: 'party', animated: false}], + }); + await publishAndDrain(harness, world, message.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + const copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.stickers.map((sticker) => [sticker.id, sticker.name])).toEqual([[stickerId, 'party']]); + }); + + test('several followers in several guilds each get one copy', async () => { + const c = await createAnnouncementTargetGuild(harness, 'Third Community'); + await addGuildMember(harness, world.a.owner, world.a.guild, c.owner); + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + await followInto(harness, world, c.t1.id, c.owner.token); + const source = await postAndPublish(harness, world, {content: 'everyone gets this'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(1); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, source.id)).toHaveLength(1); + expect(await copiesOf(harness, c.owner.token, c.t1.id, source.id)).toHaveLength(1); + }); + + test('small chunks and pages still give every follower exactly one copy', async () => { + const targets = [world.b.t1, world.b.t2]; + for (const name of ['t3', 't4', 't5']) { + targets.push(await createTextChannel(name)); + } + for (const target of targets) { + await followInto(harness, world, target.id); + } + fanoutSizes.chunk = 1; + fanoutSizes.page = 2; + const source = await postAndPublish(harness, world, {content: 'paged'}); + for (const target of targets) { + expect(await copiesOf(harness, world.b.owner.token, target.id, source.id)).toHaveLength(1); + } + expect(world.worker.byTask('crosspostMessage')).toHaveLength(3); + expect(world.worker.byTask('crosspostMessageChunk')).toHaveLength(5); + }); + + test('running a chunk twice for one follower creates one copy', async () => { + const source = await postAndPublish(harness, world, {content: 'once'}); + const webhookId = await followInto(harness, world, world.b.t1.id); + const payload = {channelId: world.a.ann.id, messageId: source.id, webhookIds: [webhookId], attempt: 0}; + await crosspostMessageChunk(payload, workerHelpers(world.worker)); + await crosspostMessageChunk(payload, workerHelpers(world.worker)); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(1); + }); + + test('a stale pending reservation is reclaimed with a newer id, a fresh one waits', async () => { + const staleSource = await postAndPublish(harness, world, {content: 'stale'}); + const freshSource = await postAndPublish(harness, world, {content: 'fresh'}); + const webhookId = await followInto(harness, world, world.b.t1.id); + const repository = new ChannelRepository(); + const staleTargetId = createMessageID(BigInt(staleSource.id)); + const base = { + webhook_id: createWebhookID(BigInt(webhookId)), + source_channel_id: createChannelID(BigInt(world.a.ann.id)), + target_guild_id: createGuildID(BigInt(world.b.guild.id)), + target_channel_id: createChannelID(BigInt(world.b.t1.id)), + state: 'pending' as const, + source_fingerprint: null, + created_at: new Date(), + }; + await repository.crossposts.insertPending({ + ...base, + source_message_id: createMessageID(BigInt(staleSource.id)), + target_message_id: staleTargetId, + reserved_at: new Date(Date.now() - 120_000), + }); + await repository.crossposts.insertPending({ + ...base, + source_message_id: createMessageID(BigInt(freshSource.id)), + target_message_id: createMessageID(BigInt(freshSource.id)), + reserved_at: new Date(), + }); + await crosspostMessageChunk( + {channelId: world.a.ann.id, messageId: staleSource.id, webhookIds: [webhookId], attempt: 0}, + workerHelpers(world.worker), + ); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, staleSource.id); + expect(BigInt(copy!.id)).toBeGreaterThan(staleTargetId); + expect((await mappingRow(staleSource.id, webhookId))?.state).toBe('delivered'); + const recordedBefore = world.worker.recorded.length; + await crosspostMessageChunk( + {channelId: world.a.ann.id, messageId: freshSource.id, webhookIds: [webhookId], attempt: 0}, + workerHelpers(world.worker), + ); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, freshSource.id)).toHaveLength(0); + const retries = world.worker.recorded.slice(recordedBefore); + expect(retries).toHaveLength(1); + expect(retries[0]!.taskType).toBe('crosspostMessageChunk'); + expect(retries[0]!.payload).toMatchObject({attempt: 1, webhookIds: [webhookId]}); + expect(retries[0]!.runAt!.getTime()).toBeGreaterThan(Date.now()); + expect(retries[0]!.jobKey).toBe(`crosspost-chunk:${freshSource.id}:${webhookId}:1`); + }); + + test('a copy delivered on retry still lands after newer target messages', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'late'}); + failNextCopyCreate(); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain({maxJobs: 2}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id)).toHaveLength(0); + const newer = await sendMessage(harness, world.b.owner.token, world.b.t1.id, {content: 'meanwhile'}); + await world.worker.drain(); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(BigInt(copy!.id)).toBeGreaterThan(BigInt(newer.id)); + const channel = await createBuilder(harness, world.b.owner.token) + .get(`/channels/${world.b.t1.id}`) + .execute(); + expect(channel.last_message_id).toBe(copy!.id); + }); + + describe('skips without errors', () => { + test('a deleted follower webhook', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + await createBuilder(harness, world.b.owner.token).delete(`/webhooks/${webhookId}`).expect(204).execute(); + await postAndPublish(harness, world, {content: 'nobody'}); + await expectNoCopies(world.b.t1.id); + }); + + test('a follower webhook removed while the chunk is queued', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'raced'}); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain({maxJobs: 1}); + await createBuilder(harness, world.b.owner.token).delete(`/webhooks/${webhookId}`).expect(204).execute(); + await world.worker.drain(); + await expectNoCopies(world.b.t1.id); + }); + + test('a deleted target channel', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'gone'}); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain({maxJobs: 1}); + await createBuilder(harness, world.b.owner.token).delete(`/channels/${world.b.t1.id}`).expect(204).execute(); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + }); + + test('a target converted to a non-text type', async () => { + await followInto(harness, world, world.b.t1.id); + await patchChannelRow(world.b.t1.id, {type: ChannelTypes.GUILD_ANNOUNCEMENT}); + await postAndPublish(harness, world, {content: 'wrong type'}); + await expectNoCopies(world.b.t1.id); + }); + + test('an age-restricted source into a target that lost its restriction', async () => { + await patchChannel(harness, world.a.owner.token, world.a.ann.id, {nsfw_override: true}).execute(); + await followInto(harness, world, world.b.ageRestricted.id); + await patchChannel(harness, world.b.owner.token, world.b.ageRestricted.id, {nsfw_override: false}).execute(); + await postAndPublish(harness, world, {content: 'restricted'}); + await expectNoCopies(world.b.ageRestricted.id); + }); + + test('a content-warning source into a target that lost its warning', async () => { + await patchChannel(harness, world.a.owner.token, world.a.ann.id, { + content_warning_level: ContentWarningLevel.CONTENT_WARNING, + }).execute(); + await followInto(harness, world, world.b.contentWarning.id); + await patchChannel(harness, world.b.owner.token, world.b.contentWarning.id, { + content_warning_level: ContentWarningLevel.INHERIT, + }).execute(); + await postAndPublish(harness, world, {content: 'warned'}); + await expectNoCopies(world.b.contentWarning.id); + }); + + test('a target guild with sending disabled', async () => { + await followInto(harness, world, world.b.t1.id); + await patchGuildRow(world.b.guild.id, {disabled_operations: GuildOperations.SEND_MESSAGE}); + await postAndPublish(harness, world, {content: 'muted target'}); + await patchGuildRow(world.b.guild.id, {disabled_operations: 0}); + await expectNoCopies(world.b.t1.id); + }); + + test('a source guild that disables announcement channels after publishing', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'switched off'}); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await addGuildFeature(world.a.guild.id, GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED); + await world.worker.drain(); + await expectNoCopies(world.b.t1.id); + }); + + test('a follower creator who can no longer view the source', async () => { + await followInto(harness, world, world.b.t1.id); + await createPermissionOverwrite(harness, world.a.owner.token, world.a.ann.id, world.b.owner.userId, { + type: 1, + allow: '0', + deny: Permissions.VIEW_CHANNEL.toString(), + }); + const infoSpy = vi.spyOn(Logger.child({}), 'info'); + await postAndPublish(harness, world, {content: 'private'}); + await expectNoCopies(world.b.t1.id); + expect(infoSpy.mock.calls.some(([, message]) => String(message).includes('cannot view'))).toBe(true); + }); + }); + + test('flagged media is stripped for targets that do not allow it', async () => { + const c = await createAnnouncementTargetGuild(harness, 'Filtered Community'); + await addGuildMember(harness, world.a.owner, world.a.guild, c.owner); + await patchGuildRow(world.a.guild.id, {explicit_content_filter: GuildExplicitContentFilterTypes.DISABLED}); + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.ageRestricted.id); + await followInto(harness, world, c.t1.id, c.owner.token); + await addGuildFeature(world.b.guild.id, GuildFeatures.DISCOVERABLE); + await patchGuildRow(c.guild.id, {explicit_content_filter: GuildExplicitContentFilterTypes.ALL_MEMBERS}); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, { + content: 'mixed', + embeds: [{title: 'Safe card'}, {title: 'Flagged card'}], + }); + const row = (await readRow(world.a.ann.id, message.id))!; + await writeRow(row, { + attachments: row.attachments.map((attachment) => ({...attachment.toMessageAttachment(), nsfw: true})), + embeds: row.embeds.map((embed, index) => ({...embed.toMessageEmbed(), nsfw: index === 1})), + }); + await publishAndDrain(harness, world, message.id); + for (const [token, channelId] of [ + [world.b.owner.token, world.b.t1.id], + [c.owner.token, c.t1.id], + ] as const) { + const [copy] = await copiesOf(harness, token, channelId, message.id); + const copyRow = (await readRow(channelId, copy!.id))!; + expect(copyRow.attachments).toHaveLength(0); + expect(copyRow.embeds.map((embed) => embed.title)).toEqual(['Safe card']); + } + const [restricted] = await copiesOf(harness, world.b.owner.token, world.b.ageRestricted.id, message.id); + const restrictedRow = (await readRow(world.b.ageRestricted.id, restricted!.id))!; + expect(restrictedRow.attachments).toHaveLength(1); + expect(restrictedRow.embeds.map((embed) => embed.title)).toEqual(['Safe card', 'Flagged card']); + }); + + test('a sha blocklisted after publishing stops the fan-out', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'image'}); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + const blockedSha = 'cd'.repeat(32); + const row = (await readRow(world.a.ann.id, message.id))!; + await writeRow(row, { + attachments: row.attachments.map((attachment) => ({ + ...attachment.toMessageAttachment(), + content_hash: blockedSha, + })), + }); + fileShaCache.add(blockedSha); + const warnSpy = vi.spyOn(Logger.child({}), 'warn'); + try { + await world.worker.drain(); + } finally { + fileShaCache.remove(blockedSha); + } + await expectNoCopies(world.b.t1.id); + expect(warnSpy.mock.calls.some(([, text]) => String(text).includes('blocked content'))).toBe(true); + }); + + test('an edit that removes and adds attachments reaches the copies', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'v1'}, [ + 'first.png', + 'second.png', + ]); + await publishAndDrain(harness, world, message.id); + const row = (await readRow(world.a.ann.id, message.id))!; + const [first, second] = row.attachments; + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.attachments?.map((attachment) => attachment.id)).toEqual([ + first!.id.toString(), + second!.id.toString(), + ]); + await createBuilder(harness, world.a.owner.token) + .delete(`/channels/${world.a.ann.id}/messages/${message.id}/attachments/${second!.id}`) + .expect(204) + .execute(); + await world.worker.drain(); + let copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.attachments.map((attachment) => attachment.id)).toEqual([first!.id]); + const secondKey = `attachments/${world.a.ann.id}/${second!.id}/${second!.filename}`; + expect(deletedKeys().filter((key) => key === secondKey)).toHaveLength(1); + const extraId = first!.id + 1000n; + const edited = (await readRow(world.a.ann.id, message.id))!; + await writeRow(edited, { + content: 'v2', + attachments: [ + first!.toMessageAttachment(), + {...first!.toMessageAttachment(), attachment_id: createAttachmentID(extraId), filename: 'extra.png'}, + ], + }); + await enqueueCrosspostSync(world.worker, { + channelId: createChannelID(BigInt(world.a.ann.id)), + messageId: createMessageID(BigInt(message.id)), + mode: 'update', + }); + await world.worker.drain(); + copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.content).toBe('v2'); + expect(copyRow.attachments.map((attachment) => [attachment.id, attachment.filename])).toEqual([ + [first!.id, first!.filename], + [extraId, 'extra.png'], + ]); + const [synced] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(synced!.attachments?.[1]?.url).toContain(`/attachments/${world.a.ann.id}/${extraId}/extra.png`); + expect(attachmentObjectCopies()).toEqual([]); + expect(deletedKeys().some((key) => key.startsWith(`attachments/${world.b.t1.id}/`))).toBe(false); + }); + + test('a published message with large attachments reaches every copy without copying bytes', async () => { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'big'}); + const row = (await readRow(world.a.ann.id, message.id))!; + const large = 500n * 1024n * 1024n; + await writeRow(row, { + attachments: row.attachments.map((attachment) => ({...attachment.toMessageAttachment(), size: large})), + }); + await publishAndDrain(harness, world, message.id); + for (const target of [world.b.t1, world.b.t2]) { + const [copy] = await copiesOf(harness, world.b.owner.token, target.id, message.id); + expect(copy!.attachments).toHaveLength(1); + expect(copy!.attachments![0]!.size).toBe(Number(large)); + expect(copy!.attachments![0]!.id).toBe(row.attachments[0]!.id.toString()); + } + expect(attachmentObjectCopies()).toEqual([]); + }); + + test('a failed target is retried with backoff and only the missing copy is created', async () => { + const webhookT1 = await followInto(harness, world, world.b.t1.id); + const webhookT2 = await followInto(harness, world, world.b.t2.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'retry'}); + failNextCopyCreate(); + await publishAndDrain(harness, world, message.id); + const retries = world.worker + .byTask('crosspostMessageChunk') + .filter((job) => (job.payload as {attempt: number}).attempt === 1); + expect(retries).toHaveLength(1); + expect(retries[0]!.runAt).toBeInstanceOf(Date); + expect((retries[0]!.payload as {webhookIds: Array}).webhookIds).toHaveLength(1); + expect([webhookT1, webhookT2]).toContain((retries[0]!.payload as {webhookIds: Array}).webhookIds[0]); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id)).toHaveLength(1); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, message.id)).toHaveLength(1); + }); + + test('an edit racing the delivery ends in the copy', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'draft'}); + const gate = stallNextCopyCreate(); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + const draining = world.worker.drain(); + await gate.reached; + await editRequest(harness, world.a.owner.token, world.a.ann.id, message.id, {content: 'final'}) + .expect(200) + .execute(); + gate.release(); + await draining; + await world.worker.drain(); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('final'); + }); + + test('a delete racing the delivery leaves the copy marked as source deleted', async () => { + await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'short lived'}); + const gate = stallNextCopyCreate(); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + const draining = world.worker.drain(); + await gate.reached; + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, message.id); + gate.release(); + await draining; + await world.worker.drain(); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED); + expect(copy!.content).toBe(CROSSPOST_SOURCE_DELETED_CONTENT); + expect(copy!.attachments ?? []).toHaveLength(0); + }); + + test('a sync between reserve and create keeps the pending row and the copy converges', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'one'}); + const gate = createStallGate(); + const original = MessagePersistenceService.prototype.createMessage; + vi.spyOn(MessagePersistenceService.prototype, 'createMessage').mockImplementation(async function ( + this: MessagePersistenceService, + params, + ) { + if (params.webhookId?.toString() === webhookId) { + await gate.hit(); + } + return original.call(this, params); + }); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + const draining = world.worker.drain(); + await gate.reached; + expect((await mappingRow(message.id, webhookId))?.state).toBe('pending'); + await editRequest(harness, world.a.owner.token, world.a.ann.id, message.id, {content: 'two'}).expect(200).execute(); + const syncCopies = (await import('@app/api/worker/tasks/SyncCrosspostCopies')).default; + await syncCopies( + {channelId: world.a.ann.id, messageId: message.id, mode: 'update', webhookIds: [webhookId]}, + workerHelpers(world.worker), + ); + expect((await mappingRow(message.id, webhookId))?.state).toBe('pending'); + gate.release(); + await draining; + await world.worker.drain(); + let [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('two'); + await editRequest(harness, world.a.owner.token, world.a.ann.id, message.id, {content: 'three'}) + .expect(200) + .execute(); + await world.worker.drain(); + [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('three'); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain(); + [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.flags & MessageFlags.SOURCE_MESSAGE_DELETED).toBe(MessageFlags.SOURCE_MESSAGE_DELETED); + }); + + test('a copy moves the target last message id, is indexed and is dispatched to the target guild', async () => { + await followInto(harness, world, world.b.t1.id); + await patchChannelRow(world.b.t1.id, {indexed_at: new Date()}); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const indexSpy = vi.spyOn(MessageSearchService.prototype, 'indexMessage'); + const source = await postAndPublish(harness, world, {content: 'indexed'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + const channel = await createBuilder(harness, world.b.owner.token) + .get(`/channels/${world.b.t1.id}`) + .execute(); + expect(channel.last_message_id).toBe(copy!.id); + expect(indexSpy.mock.calls.some(([message]) => message.id.toString() === copy!.id)).toBe(true); + const created = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'MESSAGE_CREATE' && params.guildId.toString() === world.b.guild.id); + expect(created.some((params) => (params.data as {id: string}).id === copy!.id)).toBe(true); + }); + + test('messages published before a follow are not backfilled', async () => { + const source = await postAndPublish(harness, world, {content: 'before'}); + await followInto(harness, world, world.b.t1.id); + await world.worker.drain(); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(0); + await follow(harness, world.b.owner.token, world.a.ann.id, world.b.t2.id); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, source.id)).toHaveLength(0); + }); + async function sendEmbeddedImage(content: string): Promise<{id: string; imageUrl: string}> { + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, { + content, + embeds: [{title: 'Card', image: {url: 'attachment://yeah.png'}}], + }); + const row = (await readRow(world.a.ann.id, message.id))!; + expect(row.attachments).toHaveLength(0); + return {id: message.id, imageUrl: row.embeds[0]!.toMessageEmbed().image!.url!}; + } + + test('a sha blocklisted on embed media after publishing stops the fan-out', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendEmbeddedImage('blocked card'); + await publish(harness, world.a.owner.token, world.a.ann.id, source.id); + const blockedSha = 'ef'.repeat(32); + const row = (await readRow(world.a.ann.id, source.id))!; + await writeRow(row, { + embeds: row.embeds.map((embed) => { + const value = embed.toMessageEmbed(); + return {...value, image: value.image ? {...value.image, content_hash: blockedSha} : value.image}; + }), + }); + fileShaCache.add(blockedSha); + try { + await world.worker.drain(); + } finally { + fileShaCache.remove(blockedSha); + } + await expectNoCopies(world.b.t1.id); + expect(attachmentObjectCopies()).toEqual([]); + }); + + test('the worker blocklist caches load entries added after publishing', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendMessage(harness, world.a.owner.token, world.a.ann.id, { + content: 'news with lateblockedphrase inside', + }); + await publish(harness, world.a.owner.token, world.a.ann.id, source.id); + await new AdminRepository().banPhrase('lateblockedphrase'); + await stopContentBlocklistCaches(); + await startContentBlocklistCaches({kvClient: getKVClient(), storageService: harness.storageService}); + try { + await world.worker.drain(); + } finally { + await stopContentBlocklistCaches(); + } + await expectNoCopies(world.b.t1.id); + }); + + test('removing a copy never deletes target objects that a source embed points at', async () => { + await followInto(harness, world, world.b.t1.id); + const victim = await sendWithImage(harness, world.b.owner.token, world.b.t1.id, {content: 'mine'}); + const victimAttachment = (await readRow(world.b.t1.id, victim.id))!.attachments[0]!; + const victimKey = `attachments/${world.b.t1.id}/${victimAttachment.id}/${victimAttachment.filename}`; + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, victimKey)).toBe(true); + const source = await sendMessage(harness, world.a.owner.token, world.a.ann.id, { + content: 'look', + embeds: [{title: 'Pointer', image: {url: 'https://example.com/pointer.png'}}], + }); + const sourceRow = (await readRow(world.a.ann.id, source.id))!; + const pointerUrl = `${Config.endpoints.media}/attachments/${world.b.t1.id}/${victimAttachment.id}/x.png`; + await writeRow(sourceRow, { + embeds: sourceRow.embeds.map((embed) => { + const value = embed.toMessageEmbed(); + return {...value, image: {...value.image!, url: pointerUrl}}; + }), + }); + await publishAndDrain(harness, world, source.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect((await readRow(world.b.t1.id, copy!.id))!.embeds[0]!.toMessageEmbed().image?.url).toBe(pointerUrl); + await deleteMessageRequest(harness, world.a.owner.token, world.a.ann.id, source.id); + await world.worker.drain(); + expect((await readRow(world.b.t1.id, copy!.id))!.flags).toBe( + MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED, + ); + expect(deletedKeys().some((key) => key.startsWith(`attachments/${world.b.t1.id}/${victimAttachment.id}/`))).toBe( + false, + ); + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, victimKey)).toBe(true); + }); + + test('a delivered copy whose race check failed catches up on the retry', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'one'}); + const original = MessagePersistenceService.prototype.createMessage; + vi.spyOn(MessagePersistenceService.prototype, 'createMessage').mockImplementation(async function ( + this: MessagePersistenceService, + params, + ) { + if (params.webhookId?.toString() !== webhookId) { + return original.call(this, params); + } + await writeRow((await readRow(world.a.ann.id, message.id))!, {content: 'two'}); + const created = await original.call(this, params); + vi.spyOn(MessageWriteLock.prototype, 'withFreshMessage').mockRejectedValueOnce(new Error('lock hiccup')); + return created; + }); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await world.worker.drain(); + expect(world.worker.byTask('crosspostMessageChunk').some((job) => job.payload.attempt === 1)).toBe(true); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('two'); + }); + + test('an admin takedown racing the delivery deletes the copy instead of marking it', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendMessage(harness, world.a.owner.token, world.a.ann.id, {content: 'illegal'}); + const gate = createStallGate(); + const original = MessagePersistenceService.prototype.createMessage; + vi.spyOn(MessagePersistenceService.prototype, 'createMessage').mockImplementation(async function ( + this: MessagePersistenceService, + params, + ) { + if (params.webhookId?.toString() === webhookId) { + await gate.hit(); + } + return original.call(this, params); + }); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + const draining = world.worker.drain(); + await gate.reached; + const admin = await setUserACLs(harness, world.a.member, [AdminACLs.AUTHENTICATE, AdminACLs.MESSAGE_DELETE]); + await createBuilder(harness, admin.token) + .delete(`/admin/channels/${world.a.ann.id}/messages/${message.id}`) + .expect(200) + .execute(); + gate.release(); + await draining; + await world.worker.drain(); + await expectNoCopies(world.b.t1.id); + expect(await mappingRow(message.id, webhookId)).toBeNull(); + }); + + test('a failed copy create leaves no copy behind and the retry delivers once', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'files'}); + failNextCopyCreate(webhookId); + await publish(harness, world.a.owner.token, world.a.ann.id, message.id); + await crosspostMessageChunk( + {channelId: world.a.ann.id, messageId: message.id, webhookIds: [webhookId], attempt: 0}, + workerHelpers(world.worker), + ); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id)).toHaveLength(0); + await world.worker.drain(); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id)).toHaveLength(1); + expect(attachmentObjectCopies()).toEqual([]); + expect(deletedKeys().filter((key) => key.startsWith('attachments/'))).toEqual([]); + }); + + test('two syncs of one edit converge on the source attachments', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const message = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'v1'}); + await publishAndDrain(harness, world, message.id); + const row = (await readRow(world.a.ann.id, message.id))!; + const first = row.attachments[0]!; + const extraId = first.id + 1n; + await harness.storageService.uploadObject({ + bucket: Config.s3.buckets.cdn, + key: `attachments/${world.a.ann.id}/${extraId}/extra.png`, + body: loadFixture('yeah.png'), + contentType: 'image/png', + }); + await writeRow(row, { + content: 'v2', + attachments: [ + first.toMessageAttachment(), + {...first.toMessageAttachment(), attachment_id: createAttachmentID(extraId), filename: 'extra.png'}, + ], + }); + const arrive = createBarrier(2); + const originalLock = MessageWriteLock.prototype.withFreshMessage; + vi.spyOn(MessageWriteLock.prototype, 'withFreshMessage').mockImplementation(async function ( + this: MessageWriteLock, + ...args: Parameters + ) { + await arrive(); + return originalLock.apply(this, args); + } as MessageWriteLock['withFreshMessage']); + const payload = { + channelId: world.a.ann.id, + messageId: message.id, + mode: 'update' as const, + webhookIds: [webhookId], + }; + const results = await Promise.allSettled([ + syncCrosspostCopies(payload, workerHelpers(world.worker)), + syncCrosspostCopies(payload, workerHelpers(world.worker)), + ]); + vi.restoreAllMocks(); + expect(results.some((result) => result.status === 'fulfilled')).toBe(true); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, message.id); + expect(copy!.content).toBe('v2'); + const copyRow = (await readRow(world.b.t1.id, copy!.id))!; + expect(copyRow.attachments.map((attachment) => attachment.id)).toEqual([first.id, extraId]); + expect(attachmentObjectCopies()).toEqual([]); + expect(deletedKeys().filter((key) => key.startsWith('attachments/'))).toEqual([]); + }); + + describe('copy author avatar', () => { + test('a follow captures the source icon and the copy uses it', async () => { + const icon = await setSourceIcon(getPngDataUrl()); + expect(icon).toBeTruthy(); + const webhookId = await followInto(harness, world, world.b.t1.id); + expect((await findWebhookRow(webhookId))?.avatarHash).toBe(icon); + expect(webhookAvatarCopies(webhookId)).toEqual([ + { + sourceBucket: Config.s3.buckets.cdn, + sourceKey: `icons/${world.a.guild.id}/${icon!.replace(/^a_/, '')}`, + destinationBucket: Config.s3.buckets.cdn, + destinationKey: `avatars/${webhookId}/${icon!.replace(/^a_/, '')}`, + }, + ]); + const source = await postAndPublish(harness, world, {content: 'with icon'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy!.author.avatar).toBe(icon); + expect(webhookAvatarCopies(webhookId)).toHaveLength(1); + }); + + test('a changed source icon reaches new copies while the webhook keeps its snapshot', async () => { + const oldIcon = await setSourceIcon(getPngDataUrl()); + const webhookId = await followInto(harness, world, world.b.t1.id); + const before = await postAndPublish(harness, world, {content: 'before'}); + const newIcon = await setSourceIcon(await solidPngDataUrl('#ff0000')); + expect(newIcon).toBeTruthy(); + expect(newIcon).not.toBe(oldIcon); + const first = await postAndPublish(harness, world, {content: 'after one'}); + const second = await postAndPublish(harness, world, {content: 'after two'}); + expect((await findWebhookRow(webhookId))?.avatarHash).toBe(oldIcon); + const newKey = newIcon!.replace(/^a_/, ''); + const [firstCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, first.id); + const [secondCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, second.id); + expect(firstCopy!.author.avatar).toBe(newIcon); + expect(secondCopy!.author.avatar).toBe(newIcon); + expect((await readRow(world.b.t1.id, firstCopy!.id))?.webhookAvatarHash).toBe(newIcon); + const newIconCopies = webhookAvatarCopies(webhookId).filter( + (copy) => copy.destinationKey === `avatars/${webhookId}/${newKey}`, + ); + expect(newIconCopies).toEqual([expect.objectContaining({sourceKey: `icons/${world.a.guild.id}/${newKey}`})]); + expect( + await harness.storageService.getObjectMetadata(Config.s3.buckets.cdn, `avatars/${webhookId}/${newKey}`), + ).not.toBeNull(); + const [beforeCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, before.id); + expect(beforeCopy!.author.avatar).toBe(oldIcon); + await editRequest(harness, world.a.owner.token, world.a.ann.id, before.id, {content: 'before edited'}) + .expect(200) + .execute(); + await world.worker.drain(); + const [editedCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, before.id); + expect(editedCopy!.content).toBe('before edited'); + expect(editedCopy!.author.avatar).toBe(oldIcon); + }); + + test('a source guild without an icon gives a null avatar on the webhook and the copy', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + expect((await findWebhookRow(webhookId))?.avatarHash ?? null).toBeNull(); + const source = await postAndPublish(harness, world, {content: 'no icon'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy!.author.avatar ?? null).toBeNull(); + expect(webhookAvatarCopies(webhookId)).toHaveLength(0); + }); + + test('removing the source icon after following gives new copies a null avatar', async () => { + const icon = await setSourceIcon(getPngDataUrl()); + const webhookId = await followInto(harness, world, world.b.t1.id); + expect(await setSourceIcon(null)).toBeNull(); + const source = await postAndPublish(harness, world, {content: 'icon removed'}); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy!.author.avatar ?? null).toBeNull(); + expect((await findWebhookRow(webhookId))?.avatarHash).toBe(icon); + }); + }); +}); diff --git a/fluxer_api/src/api/channel/utils/CrosspostContentRules.ts b/fluxer_api/src/api/channel/utils/CrosspostContentRules.ts new file mode 100644 index 000000000..84f596ebf --- /dev/null +++ b/fluxer_api/src/api/channel/utils/CrosspostContentRules.ts @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + type ContentWarningChannelLike, + computeEffectiveChannelNsfw, + computeEffectiveContentWarning, +} from '@app/api/channel/utils/EffectiveContentWarning'; +import {ContentWarningLevel} from '@fluxer/constants/src/GuildConstants'; +import {FollowTargetContentWarningRequiredError} from '@fluxer/errors/src/domains/channel/FollowTargetContentWarningRequiredError'; +import {FollowTargetNotAgeRestrictedError} from '@fluxer/errors/src/domains/channel/FollowTargetNotAgeRestrictedError'; + +type ContentWarningGuildLike = Parameters[2]; + +export type CrosspostContentRuleResult = 'ok' | 'nsfw' | 'content_warning'; + +export interface CrosspostContentRuleInput { + source: ContentWarningChannelLike; + sourceParent: ContentWarningChannelLike | null; + sourceGuild: ContentWarningGuildLike; + target: ContentWarningChannelLike; + targetParent: ContentWarningChannelLike | null; + targetGuild: ContentWarningGuildLike; +} + +export function checkCrosspostContentRules(input: CrosspostContentRuleInput): CrosspostContentRuleResult { + const sourceNsfw = computeEffectiveChannelNsfw(input.source, input.sourceParent, input.sourceGuild); + const targetNsfw = computeEffectiveChannelNsfw(input.target, input.targetParent, input.targetGuild); + if (sourceNsfw && !targetNsfw) { + return 'nsfw'; + } + const sourceWarning = computeEffectiveContentWarning(input.source, input.sourceParent, input.sourceGuild); + if (sourceWarning.level !== ContentWarningLevel.CONTENT_WARNING) { + return 'ok'; + } + if (targetNsfw) { + return 'ok'; + } + const targetWarning = computeEffectiveContentWarning(input.target, input.targetParent, input.targetGuild); + return targetWarning.level === ContentWarningLevel.CONTENT_WARNING ? 'ok' : 'content_warning'; +} + +export function assertCrosspostContentRules(input: CrosspostContentRuleInput): void { + const result = checkCrosspostContentRules(input); + if (result === 'nsfw') { + throw new FollowTargetNotAgeRestrictedError(); + } + if (result === 'content_warning') { + throw new FollowTargetContentWarningRequiredError(); + } +} diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 352df953d..64eddf8f5 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -4,7 +4,7 @@ import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig'; import type {CachePurgeAdapterName, StoreProductSlotName} from '@fluxer/config/src/MasterConfig'; export type APIWorkerMode = 'all_lanes' | 'single_lane' | 'single_task'; -export type APIWorkerLaneName = 'realtime' | 'unfurl' | 'lifecycle' | 'batch'; +export type APIWorkerLaneName = 'realtime' | 'unfurl' | 'lifecycle' | 'batch' | 'crosspost'; export type PushProviderEnvironment = 'production' | 'development'; export interface PushProviderAppConfig { @@ -342,6 +342,7 @@ export interface APIConfig { unfurl?: number; lifecycle?: number; batch?: number; + crosspost?: number; }; }; ncmec: { diff --git a/fluxer_api/src/api/csam/NcmecSubmissionService.ts b/fluxer_api/src/api/csam/NcmecSubmissionService.ts index 0647960f8..c243bcf4e 100644 --- a/fluxer_api/src/api/csam/NcmecSubmissionService.ts +++ b/fluxer_api/src/api/csam/NcmecSubmissionService.ts @@ -18,6 +18,11 @@ import {Config} from '@app/api/Config'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository'; import { + enqueueCrosspostFamilyPurgeFromCopies, + enqueueCrosspostSourceRemoval, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import { + attachmentStorageChannelId, collectMessageAttachments, makeAttachmentCdnKey, makeAttachmentCdnUrl, @@ -45,6 +50,7 @@ import {deleteMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCl import type {IUserRepository} from '@app/api/user/IUserRepository'; import {reschedulePendingDeletion} from '@app/api/user/services/PendingDeletionCoordinator'; import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig'; +import {MessageFlags} from '@fluxer/constants/src/ChannelConstants'; import {DeletionReasons} from '@fluxer/constants/src/Core'; import {CATEGORY_CHILD_SAFETY} from '@fluxer/constants/src/ReportCategories'; import {UserFlags} from '@fluxer/constants/src/UserConstants'; @@ -474,33 +480,37 @@ export class NcmecSubmissionService { private async resolveAttachment(input: SubmitAttachmentToNcmecInput): Promise { const liveMessage = await this.deps.channelRepository.getMessage(input.channelId, input.messageId); - if (liveMessage?.authorId) { + const liveAuthorId = liveMessage ? await this.resolveMessageAuthorId(liveMessage) : null; + if (liveMessage && liveAuthorId) { const liveAttachment = collectMessageAttachments(liveMessage).find( (attachment) => attachment.id === input.attachmentId && attachment.filename === input.filename, ); + const storageChannelId = attachmentStorageChannelId(liveMessage); if (liveAttachment) { return this.buildAttachmentContext(input, { contentType: liveAttachment.contentType, reportedAt: snowflakeToDate(input.messageId), - userId: liveMessage.authorId, + userId: liveAuthorId, sourceReportId: input.sourceReportId ?? null, bucket: Config.s3.buckets.cdn, - storageKey: makeAttachmentCdnKey(input.channelId, input.attachmentId, input.filename), + storageKey: makeAttachmentCdnKey(storageChannelId, input.attachmentId, input.filename), }); } - const embedMedia = findEmbedReferencedAttachmentMedia( - liveMessage, - makeAttachmentCdnUrl(input.channelId, input.attachmentId, input.filename), - ); - if (embedMedia) { - return this.buildAttachmentContext(input, { - contentType: embedMedia.contentType, - reportedAt: snowflakeToDate(input.messageId), - userId: liveMessage.authorId, - sourceReportId: input.sourceReportId ?? null, - bucket: Config.s3.buckets.cdn, - storageKey: makeAttachmentCdnKey(input.channelId, input.attachmentId, input.filename), - }); + for (const embedChannelId of new Set([input.channelId, storageChannelId])) { + const embedMedia = findEmbedReferencedAttachmentMedia( + liveMessage, + makeAttachmentCdnUrl(embedChannelId, input.attachmentId, input.filename), + ); + if (embedMedia) { + return this.buildAttachmentContext(input, { + contentType: embedMedia.contentType, + reportedAt: snowflakeToDate(input.messageId), + userId: liveAuthorId, + sourceReportId: input.sourceReportId ?? null, + bucket: Config.s3.buckets.cdn, + storageKey: makeAttachmentCdnKey(embedChannelId, input.attachmentId, input.filename), + }); + } } } if (!input.sourceReportId) { @@ -530,6 +540,17 @@ export class NcmecSubmissionService { throw new UnknownMessageError(); } + private async resolveMessageAuthorId(message: Message): Promise { + if ((message.flags & MessageFlags.IS_CROSSPOST) === 0 || !message.reference?.messageId) { + return message.authorId; + } + const source = await this.deps.channelRepository.getMessage( + message.reference.channelId, + message.reference.messageId, + ); + return source?.authorId ?? message.authorId; + } + private buildAttachmentContext( input: SubmitAttachmentToNcmecInput, context: { @@ -553,7 +574,10 @@ export class NcmecSubmissionService { sourceReportId: context.sourceReportId, cdnBucket: context.bucket, storageKey: context.storageKey, - cdnUrl: makeAttachmentCdnUrl(input.channelId, input.attachmentId, input.filename), + cdnUrl: + context.bucket === Config.s3.buckets.cdn + ? `${Config.endpoints.media}/${context.storageKey}` + : makeAttachmentCdnUrl(input.channelId, input.attachmentId, input.filename), }; } @@ -754,9 +778,7 @@ export class NcmecSubmissionService { const channel = await this.deps.channelRepository.findUnique(channelId); const message = await this.deps.channelRepository.getMessage(channelId, messageId); if (!message) return; - if (message.attachments.length > 0) { - await purgeMessageAttachments(message, this.deps.storageService, this.deps.purgeQueue); - } + await purgeMessageAttachments(message, this.deps.storageService, this.deps.purgeQueue); await this.deps.channelRepository.deleteMessage( channelId, messageId, @@ -764,6 +786,8 @@ export class NcmecSubmissionService { message.pinnedTimestamp || undefined, ); await deleteMessageSearchDocuments([messageId], {context: {source: 'ncmec_submission_delete'}}); + await enqueueCrosspostSourceRemoval(this.deps.workerService, {messages: [message], mode: 'purge'}); + await enqueueCrosspostFamilyPurgeFromCopies(this.deps.workerService, {messages: [message]}); if (!channel) return; if (channel.guildId) { await this.deps.gatewayService.dispatchGuild({ diff --git a/fluxer_api/src/api/database/types/ChannelTypes.ts b/fluxer_api/src/api/database/types/ChannelTypes.ts index f668e72a9..8676e6c87 100644 --- a/fluxer_api/src/api/database/types/ChannelTypes.ts +++ b/fluxer_api/src/api/database/types/ChannelTypes.ts @@ -71,9 +71,37 @@ export interface WebhookRow { creator_id: Nullish; name: string; avatar_hash: Nullish; + source_guild_id: Nullish; + source_channel_id: Nullish; version: number; } +export interface WebhooksBySourceChannelRow { + source_channel_id: ChannelID; + webhook_id: WebhookID; + guild_id: GuildID; +} + +export type CrosspostedMessageState = 'pending' | 'delivered'; + +export interface CrosspostedMessageRow { + source_message_id: MessageID; + webhook_id: WebhookID; + source_channel_id: ChannelID; + target_guild_id: GuildID; + target_channel_id: ChannelID; + target_message_id: MessageID; + state: CrosspostedMessageState; + reserved_at: Date; + source_fingerprint: Nullish; + created_at: Date; +} + +export interface CrosspostSourceByChannelRow { + source_channel_id: ChannelID; + source_message_id: MessageID; +} + export interface PrivateChannelRow { user_id: UserID; channel_id: ChannelID; @@ -165,8 +193,31 @@ export const WEBHOOK_COLUMNS = [ 'creator_id', 'name', 'avatar_hash', + 'source_guild_id', + 'source_channel_id', 'version', ] as const satisfies ReadonlyArray; +export const WEBHOOKS_BY_SOURCE_CHANNEL_COLUMNS = [ + 'source_channel_id', + 'webhook_id', + 'guild_id', +] as const satisfies ReadonlyArray; +export const CROSSPOSTED_MESSAGE_COLUMNS = [ + 'source_message_id', + 'webhook_id', + 'source_channel_id', + 'target_guild_id', + 'target_channel_id', + 'target_message_id', + 'state', + 'reserved_at', + 'source_fingerprint', + 'created_at', +] as const satisfies ReadonlyArray; +export const CROSSPOST_SOURCE_BY_CHANNEL_COLUMNS = [ + 'source_channel_id', + 'source_message_id', +] as const satisfies ReadonlyArray; export const READ_STATE_COLUMNS = [ 'user_id', 'channel_id', diff --git a/fluxer_api/src/api/database/types/MessageTypes.ts b/fluxer_api/src/api/database/types/MessageTypes.ts index f244d1abf..42b790eb3 100644 --- a/fluxer_api/src/api/database/types/MessageTypes.ts +++ b/fluxer_api/src/api/database/types/MessageTypes.ts @@ -100,7 +100,7 @@ export interface MessageStickerItem { export interface MessageReference { channel_id: ChannelID; - message_id: MessageID; + message_id: Nullish; guild_id: Nullish; type: number; } diff --git a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts index 0c989f454..cf6207d25 100644 --- a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts +++ b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts @@ -6,7 +6,7 @@ import type {ChannelID, MemeID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createAttachmentID, createMemeID, userIdToChannelId} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; -import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers'; +import {attachmentStorageChannelId, makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers'; import {mapFavoriteMemeToResponse} from '@app/api/favorite_meme/FavoriteMemeModel'; import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository'; import { @@ -551,7 +551,11 @@ export class FavoriteMemeService { embedCount: embeds.length, }); } - return this.mediaFromEmbed(embeds[preferredEmbedIndex], `embed_${preferredEmbedIndex}`, message.channelId); + return this.mediaFromEmbed( + embeds[preferredEmbedIndex], + `embed_${preferredEmbedIndex}`, + attachmentStorageChannelId(message), + ); } if (attachments.length > 0) { let attachment: MessageAttachmentCandidate | undefined; @@ -573,7 +577,7 @@ export class FavoriteMemeService { } } for (const embed of embeds) { - const media = await this.mediaFromEmbed(embed, 'media', message.channelId); + const media = await this.mediaFromEmbed(embed, 'media', attachmentStorageChannelId(message)); if (media) return media; } return null; @@ -594,8 +598,8 @@ export class FavoriteMemeService { const isGifv = isAnimatedAttachment(attachment.contentType, attachment.flags); return { isExternal: false, - url: makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename), - sourceKey: makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename), + url: makeSignedAttachmentCdnUrl(attachmentStorageChannelId(message), attachment.id, attachment.filename), + sourceKey: makeAttachmentCdnKey(attachmentStorageChannelId(message), attachment.id, attachment.filename), filename: attachment.filename, contentType: attachment.contentType, size: attachment.size, diff --git a/fluxer_api/src/api/guild/services/channel/ChannelHelpers.ts b/fluxer_api/src/api/guild/services/channel/ChannelHelpers.ts index 8222cd497..f217139d2 100644 --- a/fluxer_api/src/api/guild/services/channel/ChannelHelpers.ts +++ b/fluxer_api/src/api/guild/services/channel/ChannelHelpers.ts @@ -14,6 +14,12 @@ export interface ChannelReorderOperation { precedingSiblingId: ChannelID | null; } +function isTextLikeChannelType(type: number): boolean { + return ( + type === ChannelTypes.GUILD_TEXT || type === ChannelTypes.GUILD_ANNOUNCEMENT || type === ChannelTypes.GUILD_LINK + ); +} + // biome-ignore lint/complexity/noStaticOnlyClass: Existing callers use this as a namespaced helper API. export class ChannelHelpers { static getNextGlobalChannelPosition( @@ -32,9 +38,7 @@ export class ChannelHelpers { return existingChannels.reduce((max, c) => Math.max(max, c.position || 0), 0) + 1; } const channelsInCategory = existingChannels.filter((c) => c.parentId === parentId); - const textChannels = channelsInCategory.filter( - (c) => c.type === ChannelTypes.GUILD_TEXT || c.type === ChannelTypes.GUILD_LINK, - ); + const textChannels = channelsInCategory.filter((c) => isTextLikeChannelType(c.type)); const voiceChannels = channelsInCategory.filter((c) => c.type === ChannelTypes.GUILD_VOICE); if (channelType === ChannelTypes.GUILD_VOICE) { if (voiceChannels.length > 0) { @@ -82,11 +86,10 @@ export class ChannelHelpers { if (movedIndex < 0) continue; const movedChannel = siblings[movedIndex]; const isMovedVoice = movedChannel.type === ChannelTypes.GUILD_VOICE; - const isMovedText = - movedChannel.type === ChannelTypes.GUILD_TEXT || movedChannel.type === ChannelTypes.GUILD_LINK; + const isMovedText = isTextLikeChannelType(movedChannel.type); if (isMovedVoice) { const next = siblings[movedIndex + 1]; - if (next && (next.type === ChannelTypes.GUILD_TEXT || next.type === ChannelTypes.GUILD_LINK)) { + if (next && isTextLikeChannelType(next.type)) { throw InputValidationError.fromCode( 'preceding_sibling_id', ValidationErrorCodes.VOICE_CHANNELS_CANNOT_BE_ABOVE_TEXT_CHANNELS, @@ -108,7 +111,7 @@ export class ChannelHelpers { encounteredVoice = true; continue; } - const isText = sibling.type === ChannelTypes.GUILD_TEXT || sibling.type === ChannelTypes.GUILD_LINK; + const isText = isTextLikeChannelType(sibling.type); if (encounteredVoice && isText) { throw InputValidationError.fromCode( 'preceding_sibling_id', diff --git a/fluxer_api/src/api/guild/services/channel/ChannelOperationsService.ts b/fluxer_api/src/api/guild/services/channel/ChannelOperationsService.ts index 54fe8eaa4..5db66288a 100644 --- a/fluxer_api/src/api/guild/services/channel/ChannelOperationsService.ts +++ b/fluxer_api/src/api/guild/services/channel/ChannelOperationsService.ts @@ -120,14 +120,18 @@ export class ChannelOperationsService { } let channelName = params.data.name; let guildFeatures: Array | null = null; - if (params.data.type === ChannelTypes.GUILD_TEXT || params.data.type === ChannelTypes.GUILD_VOICE) { + if ( + params.data.type === ChannelTypes.GUILD_TEXT || + params.data.type === ChannelTypes.GUILD_ANNOUNCEMENT || + params.data.type === ChannelTypes.GUILD_VOICE + ) { const guildData = await this.gatewayService.getGuildData({ guildId: params.guildId, userId: params.userId, }); guildFeatures = guildData.features; } - if (params.data.type === ChannelTypes.GUILD_TEXT) { + if (params.data.type === ChannelTypes.GUILD_TEXT || params.data.type === ChannelTypes.GUILD_ANNOUNCEMENT) { const hasFlexibleNamesEnabled = (guildFeatures ?? []).includes(GuildFeatures.TEXT_CHANNEL_FLEXIBLE_NAMES); if (!hasFlexibleNamesEnabled) { channelName = ChannelNameType.parse(channelName); @@ -230,7 +234,10 @@ export class ChannelOperationsService { let hasChanges = false; const updatedChannels: Array = []; for (const channel of channels) { - if (channel.type !== ChannelTypes.GUILD_TEXT || channel.name == null) { + if ( + (channel.type !== ChannelTypes.GUILD_TEXT && channel.type !== ChannelTypes.GUILD_ANNOUNCEMENT) || + channel.name == null + ) { updatedChannels.push(channel); continue; } diff --git a/fluxer_api/src/api/guild/services/data/GuildOperationsService.ts b/fluxer_api/src/api/guild/services/data/GuildOperationsService.ts index 293a6f2d4..84a6029b7 100644 --- a/fluxer_api/src/api/guild/services/data/GuildOperationsService.ts +++ b/fluxer_api/src/api/guild/services/data/GuildOperationsService.ts @@ -3,6 +3,10 @@ import type {ChannelID, GuildID, RoleID, UserID} from '@app/api/BrandedTypes'; import {createChannelID, createGuildID, createRoleID, guildIdToRoleId} from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import { + type ChannelFollowerRemovalCopyMode, + scheduleDeletedChannelFollowerRemoval, +} from '@app/api/channel/services/ChannelFollowers'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; import {getContentMessage} from '@app/api/content_i18n/ContentI18n'; import {BatchBuilder} from '@app/api/database/CassandraQueryExecution'; @@ -162,9 +166,12 @@ function setsEqual(a: ReadonlySet, b: ReadonlySet): boolean { const TEMPLATE_AFK_TIMEOUT_MIN_SECONDS = 60; const TEMPLATE_AFK_TIMEOUT_MAX_SECONDS = 3600; -const THE_OTHER_PLATFORM_GUILD_ANNOUNCEMENT_CHANNEL_TYPE = 5; const THE_OTHER_PLATFORM_GUILD_STAGE_VOICE_CHANNEL_TYPE = 13; +function isSystemChannelType(type: number): boolean { + return type === ChannelTypes.GUILD_TEXT || type === ChannelTypes.GUILD_ANNOUNCEMENT; +} + export class GuildOperationsService { constructor( private readonly guildRepository: IGuildRepositoryAggregate, @@ -525,7 +532,7 @@ export class GuildOperationsService { ValidationErrorCodes.SYSTEM_CHANNEL_MUST_BE_IN_GUILD, ); } - if (systemChannel.type !== ChannelTypes.GUILD_TEXT) { + if (!isSystemChannelType(systemChannel.type)) { throw InputValidationError.fromCode('system_channel_id', ValidationErrorCodes.SYSTEM_CHANNEL_MUST_BE_TEXT); } patch.system_channel_id = systemChannelId; @@ -735,18 +742,22 @@ export class GuildOperationsService { if (!guildData || guildData.owner_id !== user.id.toString()) { throw new MissingPermissionsError(); } - await this.performGuildDeletion(guildId); + await this.performGuildDeletion(guildId, 'source_deleted'); } async deleteGuildById(guildId: GuildID): Promise { - await this.performGuildDeletion(guildId); + await this.performGuildDeletion(guildId, 'purge'); } - private async performGuildDeletion(guildId: GuildID): Promise { + private async performGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise { const guild = await this.guildRepository.findUnique(guildId); if (!guild) { throw new UnknownGuildError(); } + const channels = await this.channelRepository.listGuildChannels(guildId); + for (const channel of channels) { + await scheduleDeletedChannelFollowerRemoval({channel, crossposts: this.channelRepository.crossposts, copyMode}); + } const members = await this.guildRepository.listMembers(guildId); await this.gatewayService.dispatchGuild({ guildId, @@ -776,12 +787,13 @@ export class GuildOperationsService { await Promise.all(invites.map((invite) => this.inviteRepository.delete(invite.code))); const webhooks = await this.webhookRepository.listByGuild(guildId); await Promise.all(webhooks.map((webhook) => this.webhookRepository.delete(webhook.id))); - const channels = await this.channelRepository.listGuildChannels(guildId); + for (const channel of channels) { + await this.channelService.attachments.purgeChannelAttachments(channel); + } await Promise.all(channels.map((channel) => this.channelRepository.deleteAllChannelMessages(channel.id))); await Promise.all( channels.map((channel) => deleteChannelMessageSearchDocuments(channel.id, {context: {source: 'guild_delete'}})), ); - await Promise.all(channels.map((channel) => this.channelService.attachments.purgeChannelAttachments(channel))); const discoveryRow = await this.discoveryRepository.findByGuildId(guildId); if (discoveryRow) { await this.discoveryRepository.deleteByGuildId(guildId, discoveryRow.status, discoveryRow.applied_at); @@ -1032,7 +1044,7 @@ export class GuildOperationsService { permissionOverwrites = null; } } - if (fluxerType === ChannelTypes.GUILD_TEXT && !firstTextChannelId) { + if (isSystemChannelType(fluxerType) && !firstTextChannelId) { firstTextChannelId = channelId; } batch.addPrepared( @@ -1078,7 +1090,7 @@ export class GuildOperationsService { if (template.system_channel_id != null) { const templateSystemChannelKey = this.getTemplateEntityKey(template.system_channel_id); const templateSystemChannelType = channelTypeMap.get(templateSystemChannelKey); - if (templateSystemChannelType === ChannelTypes.GUILD_TEXT) { + if (templateSystemChannelType !== undefined && isSystemChannelType(templateSystemChannelType)) { systemChannelId = channelIdMap.get(templateSystemChannelKey) ?? null; } } @@ -1151,14 +1163,12 @@ export class GuildOperationsService { private mapOtherPlatformTemplateChannelTypeToFluxer(channelType: number): number | null { if ( channelType === ChannelTypes.GUILD_TEXT || + channelType === ChannelTypes.GUILD_ANNOUNCEMENT || channelType === ChannelTypes.GUILD_VOICE || channelType === ChannelTypes.GUILD_CATEGORY ) { return channelType; } - if (channelType === THE_OTHER_PLATFORM_GUILD_ANNOUNCEMENT_CHANNEL_TYPE) { - return ChannelTypes.GUILD_TEXT; - } if (channelType === THE_OTHER_PLATFORM_GUILD_STAGE_VOICE_CHANNEL_TYPE) { return ChannelTypes.GUILD_VOICE; } diff --git a/fluxer_api/src/api/guild/tests/GuildDeletionAttachmentPurge.test.ts b/fluxer_api/src/api/guild/tests/GuildDeletionAttachmentPurge.test.ts new file mode 100644 index 000000000..3ff51f12e --- /dev/null +++ b/fluxer_api/src/api/guild/tests/GuildDeletionAttachmentPurge.test.ts @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {Config} from '@app/api/Config'; +import {disableCrosspostWorker} from '@app/api/channel/tests/AnnouncementTestUtils'; +import { + copiesOf, + type FanoutWorld, + followInto, + publishAndDrain, + readRow, + sendWithImage, + setupFanoutWorld, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {MessageFlags} from '@fluxer/constants/src/ChannelConstants'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test} from 'vitest'; + +function attachmentKeys(channelId: string, message: MessageResponse): Array { + return (message.attachments ?? []).map( + (attachment) => `attachments/${channelId}/${attachment.id}/${attachment.filename}`, + ); +} + +describe('Guild deletion attachment purge', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + disableCrosspostWorker(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + function deletedKeys(): Array { + return harness.storageService.getDeletedObjects().map((entry) => entry.key); + } + + function deletedKeysForAttachments(message: MessageResponse): Array { + const ids = (message.attachments ?? []).map((attachment) => `/${attachment.id}/`); + return deletedKeys().filter((key) => ids.some((id) => key.includes(id))); + } + + async function deleteGuild(owner: TestAccount, guildId: string): Promise { + await createBuilder(harness, owner.token) + .post(`/guilds/${guildId}/delete`) + .body({password: owner.password}) + .expect(HTTP_STATUS.NO_CONTENT) + .executeWithResponse(); + await world.worker.drain(); + } + + test('purges the attachments of every message in every channel', async () => { + const first = await sendWithImage(harness, world.b.owner.token, world.b.t1.id, {content: 'one'}, [ + 'a.png', + 'b.gif', + ]); + const second = await sendWithImage(harness, world.b.owner.token, world.b.t2.id, {content: 'two'}); + const expected = [...attachmentKeys(world.b.t1.id, first), ...attachmentKeys(world.b.t2.id, second)]; + expect(expected).toHaveLength(3); + for (const key of expected) { + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(true); + } + await deleteGuild(world.b.owner, world.b.guild.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(expected)); + for (const key of expected) { + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(false); + } + }); + + test('a follower guild deletion leaves the source objects of its copies in place', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'release'}); + await publishAndDrain(harness, world, source.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy?.attachments?.map((attachment) => attachment.id)).toEqual( + source.attachments?.map((attachment) => attachment.id), + ); + const own = await sendWithImage(harness, world.b.owner.token, world.b.t1.id, {content: 'local'}); + const sourceKeys = attachmentKeys(world.a.ann.id, source); + await deleteGuild(world.b.owner, world.b.guild.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(attachmentKeys(world.b.t1.id, own))); + expect(deletedKeys().some((key) => key.startsWith(`attachments/${world.a.ann.id}/`))).toBe(false); + expect(deletedKeysForAttachments(copy!)).toEqual([]); + for (const key of sourceKeys) { + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(true); + } + expect(await readRow(world.a.ann.id, source.id)).not.toBeNull(); + }); + + test('an admin guild deletion purges every channel and leaves follower copy sources in place', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'release'}); + await publishAndDrain(harness, world, source.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy?.attachments?.map((attachment) => attachment.id)).toEqual( + source.attachments?.map((attachment) => attachment.id), + ); + const first = await sendWithImage(harness, world.b.owner.token, world.b.t1.id, {content: 'one'}); + const second = await sendWithImage(harness, world.b.owner.token, world.b.t2.id, {content: 'two'}); + const admin = await setUserACLs(harness, await createTestAccount(harness), [ + 'admin:authenticate', + 'guild:lookup', + 'guild:delete', + ]); + await createBuilder(harness, admin.token) + .delete(`/admin/guilds/${world.b.guild.id}`) + .body(null) + .expect(HTTP_STATUS.OK) + .execute(); + await world.worker.drain(); + expect(deletedKeys()).toEqual( + expect.arrayContaining([...attachmentKeys(world.b.t1.id, first), ...attachmentKeys(world.b.t2.id, second)]), + ); + expect(deletedKeys().some((key) => key.startsWith(`attachments/${world.a.ann.id}/`))).toBe(false); + expect(deletedKeysForAttachments(copy!)).toEqual([]); + for (const key of attachmentKeys(world.a.ann.id, source)) { + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(true); + } + }); + + test('a source guild deletion purges its objects and marks follower copies source deleted', async () => { + await followInto(harness, world, world.b.t1.id); + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'release'}); + await publishAndDrain(harness, world, source.id); + const [copy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id); + expect(copy).toBeDefined(); + await deleteGuild(world.a.owner, world.a.guild.id); + expect(deletedKeys()).toEqual(expect.arrayContaining(attachmentKeys(world.a.ann.id, source))); + const marked = await readRow(world.b.t1.id, copy!.id); + expect(marked?.flags).toBe(MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED); + }); +}); diff --git a/fluxer_api/src/api/guild/tests/GuildTemplateImport.test.ts b/fluxer_api/src/api/guild/tests/GuildTemplateImport.test.ts index 6ffb38f74..c17350f11 100644 --- a/fluxer_api/src/api/guild/tests/GuildTemplateImport.test.ts +++ b/fluxer_api/src/api/guild/tests/GuildTemplateImport.test.ts @@ -140,7 +140,7 @@ describe('Guild Template Import', () => { }, { id: '4102', - type: 5, + type: ChannelTypes.GUILD_ANNOUNCEMENT, name: 'announcements', parent_id: '4101', position: 1, @@ -170,7 +170,7 @@ describe('Guild Template Import', () => { const mediaChannel = channels.find((channel) => channel.name === 'media-feed'); expect(categoryChannel).toBeDefined(); expect(announcementsChannel).toBeDefined(); - expect(announcementsChannel?.type).toBe(ChannelTypes.GUILD_TEXT); + expect(announcementsChannel?.type).toBe(ChannelTypes.GUILD_ANNOUNCEMENT); expect(announcementsChannel?.parent_id).toBe(categoryChannel?.id ?? null); expect(stageChannel).toBeDefined(); expect(stageChannel?.type).toBe(ChannelTypes.GUILD_VOICE); diff --git a/fluxer_api/src/api/infrastructure/AvatarService.ts b/fluxer_api/src/api/infrastructure/AvatarService.ts index cd22b504e..acfb9db9f 100644 --- a/fluxer_api/src/api/infrastructure/AvatarService.ts +++ b/fluxer_api/src/api/infrastructure/AvatarService.ts @@ -253,6 +253,45 @@ export class AvatarService { }); } + async copyGuildIconToWebhookAvatar(params: { + guildId: bigint; + iconHash: string | null; + webhookId: bigint; + }): Promise { + const {guildId, iconHash, webhookId} = params; + if (!iconHash) return null; + const key = this.stripAnimationPrefix(iconHash); + try { + await this.storageService.copyObject({ + sourceBucket: Config.s3.buckets.cdn, + sourceKey: `icons/${guildId}/${key}`, + destinationBucket: Config.s3.buckets.cdn, + destinationKey: `avatars/${webhookId}/${key}`, + }); + } catch (error) { + if (error instanceof Error && (error.name === 'NoSuchKey' || error.name === 'NotFound')) { + return null; + } + throw error; + } + return iconHash; + } + + async ensureWebhookAvatarFromGuildIcon(params: { + guildId: bigint; + iconHash: string | null; + webhookId: bigint; + }): Promise { + const {iconHash, webhookId} = params; + if (!iconHash) return null; + const existing = await this.storageService.getObjectMetadata( + Config.s3.buckets.cdn, + `avatars/${webhookId}/${this.stripAnimationPrefix(iconHash)}`, + ); + if (existing) return iconHash; + return this.copyGuildIconToWebhookAvatar(params); + } + async processSticker(params: {errorPath: string; base64Image: string; guildFeatures: Iterable}): Promise<{ imageBuffer: Uint8Array; animated: boolean; diff --git a/fluxer_api/src/api/middleware/ContentBlocklistCaches.ts b/fluxer_api/src/api/middleware/ContentBlocklistCaches.ts new file mode 100644 index 000000000..30548a122 --- /dev/null +++ b/fluxer_api/src/api/middleware/ContentBlocklistCaches.ts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {IStorageService} from '@app/api/infrastructure/IStorageService'; +import {fileShaCache} from '@app/api/middleware/FileShaCache'; +import {phraseBlocklistCache} from '@app/api/middleware/PhraseBlocklistCache'; +import {urlBlocklistCache} from '@app/api/middleware/UrlBlocklistCache'; +import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider'; + +export async function startContentBlocklistCaches({ + kvClient, + storageService, +}: { + kvClient: IKVProvider; + storageService: IStorageService; +}): Promise { + urlBlocklistCache.setRefreshSubscriber(kvClient); + urlBlocklistCache.setStorageService(storageService); + fileShaCache.setRefreshSubscriber(kvClient); + phraseBlocklistCache.setRefreshSubscriber(kvClient); + await Promise.all([urlBlocklistCache.initialize(), fileShaCache.initialize(), phraseBlocklistCache.initialize()]); +} + +export async function stopContentBlocklistCaches(): Promise { + await Promise.all([urlBlocklistCache.shutdown(), fileShaCache.shutdown(), phraseBlocklistCache.shutdown()]); +} diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.ts index 6b1e70fbf..d9ad6ff01 100644 --- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts +++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts @@ -11,6 +11,7 @@ import {Config} from '@app/api/Config'; import {createApiContext} from '@app/api/CreateApiContext'; import {ChannelRepository} from '@app/api/channel/ChannelRepository'; import {ChannelRequestService} from '@app/api/channel/services/ChannelRequestService'; +import {CrosspostSourceService} from '@app/api/channel/services/message/CrosspostSourceService'; import {MessageRequestService} from '@app/api/channel/services/message/MessageRequestService'; import {createMessageResponseDataService} from '@app/api/channel/services/message/MessageResponseDataService'; import {StreamService} from '@app/api/channel/services/StreamService'; @@ -79,6 +80,7 @@ import { getGatewayRequestService, getGifService, getGuildAuditLogService, + getGuildDiscoveryRepository, getGuildDiscoveryService, getGuildRepository, getInstanceConfigRepository, @@ -135,6 +137,7 @@ import {UserRelationshipRequestService} from '@app/api/user/services/UserRelatio import {UserService} from '@app/api/user/services/UserService'; import {getRequestClientIp} from '@app/api/utils/RequestClientIp'; import {VoiceService} from '@app/api/voice/VoiceService'; +import {ChannelFollowService} from '@app/api/webhook/ChannelFollowService'; import {WebhookRequestService} from '@app/api/webhook/WebhookRequestService'; import {WebhookService} from '@app/api/webhook/WebhookService'; import {createMiddleware} from 'hono/factory'; @@ -238,6 +241,7 @@ class RequestServices implements RequestScopedServices { private cachedUserContentRequestService: UserContentRequestService | undefined; private cachedUserRelationshipRequestService: UserRelationshipRequestService | undefined; private cachedWebhookService: WebhookService | undefined; + private cachedChannelFollowService: ChannelFollowService | undefined; private cachedWebhookRequestService: WebhookRequestService | undefined; constructor( @@ -586,6 +590,12 @@ class RequestServices implements RequestScopedServices { this.cachedMessageRequestService ??= new MessageRequestService( this.channelService, createMessageResponseDataService(), + new CrosspostSourceService( + this.requestGuildRepository, + getGuildDiscoveryRepository(), + this.gatewayService, + this.cacheService, + ), ); return this.cachedMessageRequestService; } @@ -871,6 +881,20 @@ class RequestServices implements RequestScopedServices { return this.cachedWebhookService; } + get channelFollowService(): ChannelFollowService { + this.cachedChannelFollowService ??= new ChannelFollowService( + this.webhookService, + getWebhookRepository(), + this.channelService, + getChannelRepository(), + this.guildService, + getAvatarService(), + getCacheService(), + getSnowflakeService(), + ); + return this.cachedChannelFollowService; + } + get webhookRequestService(): WebhookRequestService { this.cachedWebhookRequestService ??= new WebhookRequestService( this.webhookService, @@ -878,6 +902,7 @@ class RequestServices implements RequestScopedServices { getUserCacheService(), this.liveKitWebhookService ?? null, getSweegoWebhookService(), + this.gatewayService, ); return this.cachedWebhookRequestService; } diff --git a/fluxer_api/src/api/models/MessageRef.ts b/fluxer_api/src/api/models/MessageRef.ts index bd3db8d67..bbc762063 100644 --- a/fluxer_api/src/api/models/MessageRef.ts +++ b/fluxer_api/src/api/models/MessageRef.ts @@ -6,13 +6,13 @@ import type {MessageReferenceTypeValue} from '@fluxer/constants/src/ChannelConst export class MessageRef { readonly channelId: ChannelID; - readonly messageId: MessageID; + readonly messageId: MessageID | null; readonly guildId: GuildID | null; readonly type: MessageReferenceTypeValue; constructor(ref: MessageReference) { this.channelId = ref.channel_id; - this.messageId = ref.message_id; + this.messageId = ref.message_id ?? null; this.guildId = ref.guild_id ?? null; this.type = ref.type as MessageReferenceTypeValue; } diff --git a/fluxer_api/src/api/models/Webhook.ts b/fluxer_api/src/api/models/Webhook.ts index f8bf3b4b8..2f46441f1 100644 --- a/fluxer_api/src/api/models/Webhook.ts +++ b/fluxer_api/src/api/models/Webhook.ts @@ -12,6 +12,8 @@ export class Webhook { readonly creatorId: UserID | null; readonly name: string; readonly avatarHash: string | null; + readonly sourceGuildId: GuildID | null; + readonly sourceChannelId: ChannelID | null; readonly version: number; constructor(row: WebhookRow) { @@ -23,6 +25,8 @@ export class Webhook { this.creatorId = row.creator_id ?? null; this.name = row.name; this.avatarHash = row.avatar_hash ?? null; + this.sourceGuildId = row.source_guild_id ?? null; + this.sourceChannelId = row.source_channel_id ?? null; this.version = row.version; } @@ -36,6 +40,8 @@ export class Webhook { creator_id: this.creatorId, name: this.name, avatar_hash: this.avatarHash, + source_guild_id: this.sourceGuildId, + source_channel_id: this.sourceChannelId, version: this.version, }; } diff --git a/fluxer_api/src/api/openapi/openapi.json b/fluxer_api/src/api/openapi/openapi.json index fdaa3fa5e..97c0ec646 100644 --- a/fluxer_api/src/api/openapi/openapi.json +++ b/fluxer_api/src/api/openapi/openapi.json @@ -2688,6 +2688,134 @@ } } }, + "/channels/{channel_id}/follower-stats": { + "get": { + "operationId": "get_channel_follower_stats", + "summary": "Get announcement channel follower stats", + "tags": ["Channels"], + "responses": { + "200": { + "description": "Success", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ChannelFollowerStatsResponse"}}} + }, + "400": { + "description": "Bad Request - The request was malformed or contained invalid data", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "401": { + "description": "Unauthorized - Authentication is required or the token is invalid", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "403": { + "description": "Forbidden - You do not have permission to perform this action", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "429": { + "description": "Too Many Requests - You are being rate limited", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}}, + "headers": { + "Retry-After": { + "description": "Number of seconds to wait before retrying (only on 429)", + "schema": {"type": "integer"} + }, + "X-RateLimit-Limit": { + "description": "The number of requests that can be made in the current window", + "schema": {"type": "integer"} + }, + "X-RateLimit-Remaining": { + "description": "The number of remaining requests that can be made", + "schema": {"type": "integer"} + }, + "X-RateLimit-Reset": { + "description": "Unix timestamp when the rate limit resets", + "schema": {"type": "integer"} + } + } + }, + "500": { + "description": "Internal Server Error - An unexpected error occurred", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + }, + "description": "Returns how many channels and distinct guilds follow an announcement channel. Requires View Channel on the announcement channel.", + "security": [{"botToken": []}, {"sessionToken": []}], + "parameters": [ + { + "name": "channel_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the channel", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the channel" + } + ] + } + }, + "/channels/{channel_id}/followers": { + "post": { + "operationId": "follow_channel", + "summary": "Follow an announcement channel", + "tags": ["Channels"], + "responses": { + "200": { + "description": "Success", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/FollowedChannelResponse"}}} + }, + "400": { + "description": "Bad Request - The request was malformed or contained invalid data", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "401": { + "description": "Unauthorized - Authentication is required or the token is invalid", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "403": { + "description": "Forbidden - You do not have permission to perform this action", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "429": { + "description": "Too Many Requests - You are being rate limited", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}}, + "headers": { + "Retry-After": { + "description": "Number of seconds to wait before retrying (only on 429)", + "schema": {"type": "integer"} + }, + "X-RateLimit-Limit": { + "description": "The number of requests that can be made in the current window", + "schema": {"type": "integer"} + }, + "X-RateLimit-Remaining": { + "description": "The number of remaining requests that can be made", + "schema": {"type": "integer"} + }, + "X-RateLimit-Reset": { + "description": "Unix timestamp when the rate limit resets", + "schema": {"type": "integer"} + } + } + }, + "500": { + "description": "Internal Server Error - An unexpected error occurred", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + }, + "description": "Follows an announcement channel into a text channel. Creates a channel follower webhook in the target channel that receives every message published in the announcement channel. Requires Manage Webhooks in the target channel and View Channel on the announcement channel.", + "security": [{"botToken": []}, {"sessionToken": []}], + "parameters": [ + { + "name": "channel_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the channel", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the channel" + } + ], + "requestBody": { + "required": true, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ChannelFollowRequest"}}} + } + } + }, "/channels/{channel_id}/invites": { "post": { "operationId": "create_channel_invite", @@ -3663,6 +3791,144 @@ ] } }, + "/channels/{channel_id}/messages/{message_id}/crosspost": { + "post": { + "operationId": "crosspost_message", + "summary": "Publish a message to following channels", + "tags": ["Channels", "Messages"], + "responses": { + "200": { + "description": "Success", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MessageResponseSchema"}}} + }, + "400": { + "description": "Bad Request - The request was malformed or contained invalid data", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "401": { + "description": "Unauthorized - Authentication is required or the token is invalid", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "403": { + "description": "Forbidden - You do not have permission to perform this action", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "429": { + "description": "Too Many Requests - You are being rate limited", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}}, + "headers": { + "Retry-After": { + "description": "Number of seconds to wait before retrying (only on 429)", + "schema": {"type": "integer"} + }, + "X-RateLimit-Limit": { + "description": "The number of requests that can be made in the current window", + "schema": {"type": "integer"} + }, + "X-RateLimit-Remaining": { + "description": "The number of remaining requests that can be made", + "schema": {"type": "integer"} + }, + "X-RateLimit-Reset": { + "description": "Unix timestamp when the rate limit resets", + "schema": {"type": "integer"} + } + } + }, + "500": { + "description": "Internal Server Error - An unexpected error occurred", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + }, + "description": "Publishes a message in an announcement channel to every channel that follows it. The author needs Send Messages. Anyone else needs Send Messages and Manage Messages. Only default messages that are not replies, forwards or copies can be published, and each message can be published once. Copies are delivered asynchronously. Publishing is limited per channel (10 in a row, then one every 6 minutes) and per community (30 in a row, then one every 2 minutes). Returns the updated message with the CROSSPOSTED flag set.", + "security": [{"botToken": []}, {"sessionToken": []}], + "parameters": [ + { + "name": "channel_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the channel", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the channel" + }, + { + "name": "message_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the message", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the message" + } + ] + } + }, + "/channels/{channel_id}/messages/{message_id}/crosspost-source": { + "get": { + "operationId": "get_message_crosspost_source", + "summary": "Get the source community of a published message copy", + "tags": ["Channels", "Messages"], + "responses": { + "200": { + "description": "Success", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/CrosspostSourceResponse"}}} + }, + "400": { + "description": "Bad Request - The request was malformed or contained invalid data", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "401": { + "description": "Unauthorized - Authentication is required or the token is invalid", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "403": { + "description": "Forbidden - You do not have permission to perform this action", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "429": { + "description": "Too Many Requests - You are being rate limited", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}}, + "headers": { + "Retry-After": { + "description": "Number of seconds to wait before retrying (only on 429)", + "schema": {"type": "integer"} + }, + "X-RateLimit-Limit": { + "description": "The number of requests that can be made in the current window", + "schema": {"type": "integer"} + }, + "X-RateLimit-Remaining": { + "description": "The number of remaining requests that can be made", + "schema": {"type": "integer"} + }, + "X-RateLimit-Reset": { + "description": "Unix timestamp when the rate limit resets", + "schema": {"type": "integer"} + } + } + }, + "500": { + "description": "Internal Server Error - An unexpected error occurred", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + }, + "description": "Returns the public profile of the community a message copy was published from. Works on copies delivered to a following channel and on the system message posted when a channel starts following. Needs the same access as fetching the message. The response holds the community name, icon, banner, badge features, approximate counts and whether it can be joined through discovery.", + "security": [{"botToken": []}, {"sessionToken": []}], + "parameters": [ + { + "name": "channel_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the channel", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the channel" + }, + { + "name": "message_id", + "in": "path", + "required": true, + "schema": {"description": "The ID of the message", "$ref": "#/components/schemas/SnowflakeType"}, + "description": "The ID of the message" + } + ] + } + }, "/channels/{channel_id}/messages/{message_id}/memes": { "post": { "operationId": "create_meme_from_message", @@ -5044,7 +5310,7 @@ "responses": { "200": { "description": "Success", - "content": {"application/json": {"schema": {"$ref": "#/components/schemas/WebhookResponse"}}} + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/WebhookCreateResponse"}}} }, "400": { "description": "Bad Request - The request was malformed or contained invalid data", @@ -21501,8 +21767,8 @@ "$ref": "#/components/schemas/SnowflakeStringType" }, "message_id": { - "description": "The ID of the referenced message", - "$ref": "#/components/schemas/SnowflakeStringType" + "description": "The ID of the referenced message, absent on a channel follow system message", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}, {"type": "null"}] }, "guild_id": { "description": "The ID of the guild containing the referenced message", @@ -21510,7 +21776,7 @@ }, "type": {"$ref": "#/components/schemas/MessageReferenceType"} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, {"type": "null"} @@ -21664,8 +21930,8 @@ "$ref": "#/components/schemas/SnowflakeStringType" }, "message_id": { - "description": "The ID of the referenced message", - "$ref": "#/components/schemas/SnowflakeStringType" + "description": "The ID of the referenced message, absent on a channel follow system message", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}, {"type": "null"}] }, "guild_id": { "description": "The ID of the guild containing the referenced message", @@ -21673,7 +21939,7 @@ }, "type": {"$ref": "#/components/schemas/MessageReferenceType"} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, {"type": "null"} @@ -22442,9 +22708,10 @@ }, "name": {"type": "string", "description": "The display name of the webhook"}, "avatar": {"description": "The hash of the webhook avatar image", "type": ["string", "null"]}, + "type": {"$ref": "#/components/schemas/WebhookType"}, "token": {"type": "string", "description": "The secure token used to execute the webhook"} }, - "required": ["id", "guild_id", "channel_id", "name", "token"], + "required": ["id", "guild_id", "channel_id", "name", "type", "token"], "additionalProperties": false }, "WebhookUpdateRequest": { @@ -22478,13 +22745,50 @@ }, "name": {"type": "string", "description": "The display name of the webhook"}, "avatar": {"description": "The hash of the webhook avatar image", "type": ["string", "null"]}, - "token": {"type": "string", "description": "The secure token used to execute the webhook"}, + "type": {"$ref": "#/components/schemas/WebhookType"}, + "token": { + "description": "The secure token used to execute the webhook, omitted for channel follower webhooks", + "type": "string" + }, "user": { "description": "The user who created the webhook", "$ref": "#/components/schemas/UserPartialResponse" + }, + "source_guild": { + "description": "The guild of the followed announcement channel, present on channel follower webhooks while the creator can view it", + "type": "object", + "properties": { + "id": { + "description": "The ID of the guild that owns the followed announcement channel", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "name": { + "type": "string", + "description": "The name of the guild that owns the followed announcement channel" + }, + "icon": { + "description": "The icon hash of the guild that owns the followed announcement channel", + "type": ["string", "null"] + } + }, + "required": ["id", "name"], + "additionalProperties": false + }, + "source_channel": { + "description": "The followed announcement channel, present on channel follower webhooks while the creator can view it", + "type": "object", + "properties": { + "id": { + "description": "The ID of the followed announcement channel", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "name": {"type": "string", "description": "The name of the followed announcement channel"} + }, + "required": ["id", "name"], + "additionalProperties": false } }, - "required": ["id", "guild_id", "channel_id", "name", "token", "user"], + "required": ["id", "guild_id", "channel_id", "name", "type", "user"], "additionalProperties": false }, "EntranceSoundPlayRequest": { @@ -28047,6 +28351,7 @@ "ChannelCreateRequest": { "oneOf": [ {"$ref": "#/components/schemas/GuildTextChannelCreateRequest"}, + {"$ref": "#/components/schemas/GuildAnnouncementChannelCreateRequest"}, {"$ref": "#/components/schemas/GuildVoiceChannelCreateRequest"}, {"$ref": "#/components/schemas/GuildCategoryChannelCreateRequest"}, {"$ref": "#/components/schemas/GuildLinkChannelCreateRequest"} @@ -28531,6 +28836,33 @@ }, "required": ["name"] }, + "WebhookCreateResponse": { + "type": "object", + "properties": { + "id": { + "description": "The unique identifier (snowflake) for the webhook", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "guild_id": { + "description": "The ID of the guild this webhook belongs to", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "channel_id": { + "description": "The ID of the channel this webhook posts to", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "name": {"type": "string", "description": "The display name of the webhook"}, + "avatar": {"description": "The hash of the webhook avatar image", "type": ["string", "null"]}, + "type": {"$ref": "#/components/schemas/WebhookType"}, + "token": {"type": "string", "description": "The secure token used to execute the webhook"}, + "user": { + "description": "The user who created the webhook", + "$ref": "#/components/schemas/UserPartialResponse" + } + }, + "required": ["id", "guild_id", "channel_id", "name", "type", "token", "user"], + "additionalProperties": false + }, "ChannelSlowmodeStateResponse": { "type": "object", "properties": { @@ -28616,6 +28948,47 @@ }, "required": ["name"] }, + "CrosspostSourceResponse": { + "type": "object", + "properties": { + "guild": { + "properties": { + "id": { + "description": "The ID of the source community", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "name": {"type": "string", "description": "The name of the source community"}, + "icon": {"description": "The hash of the source community icon", "type": ["string", "null"]}, + "banner": {"description": "The hash of the source community banner", "type": ["string", "null"]}, + "description": { + "description": "The discovery description of the source community, or null when it is not listed in discovery", + "type": ["string", "null"] + }, + "features": { + "type": "array", + "items": {"$ref": "#/components/schemas/GuildFeatureSchema"}, + "description": "The public badge features of the source community: VERIFIED, PARTNERED and DISCOVERABLE only" + }, + "approximate_member_count": { + "anyOf": [{"$ref": "#/components/schemas/Int32Type"}, {"type": "null"}], + "description": "Approximate number of members in the source community, or null when unavailable" + }, + "approximate_presence_count": { + "anyOf": [{"$ref": "#/components/schemas/Int32Type"}, {"type": "null"}], + "description": "Approximate number of online members in the source community, or null when unavailable" + }, + "discoverable": { + "type": "boolean", + "description": "Whether the source community can be joined through discovery" + } + }, + "description": "The community the message was published from", + "$ref": "#/components/schemas/CrosspostSourceGuildResponse" + } + }, + "required": ["guild"], + "additionalProperties": false + }, "MessageAckRequest": { "type": "object", "properties": { @@ -28797,6 +29170,46 @@ {"$ref": "#/components/schemas/GroupDmInviteMetadataResponse"} ] }, + "ChannelFollowRequest": { + "type": "object", + "properties": { + "webhook_channel_id": { + "description": "The ID of the text channel that receives published messages", + "$ref": "#/components/schemas/SnowflakeType" + } + }, + "required": ["webhook_channel_id"] + }, + "FollowedChannelResponse": { + "type": "object", + "properties": { + "channel_id": { + "description": "The ID of the followed announcement channel", + "$ref": "#/components/schemas/SnowflakeStringType" + }, + "webhook_id": { + "description": "The ID of the channel follower webhook created in the target channel", + "$ref": "#/components/schemas/SnowflakeStringType" + } + }, + "required": ["channel_id", "webhook_id"], + "additionalProperties": false + }, + "ChannelFollowerStatsResponse": { + "type": "object", + "properties": { + "channel_count": { + "description": "The number of channels following this announcement channel", + "$ref": "#/components/schemas/Int32Type" + }, + "guild_count": { + "description": "The number of distinct guilds following this announcement channel", + "$ref": "#/components/schemas/Int32Type" + } + }, + "required": ["channel_count", "guild_count"], + "additionalProperties": false + }, "CallRingBodySchema": { "type": "object", "properties": { @@ -28966,7 +29379,103 @@ "description": "Voice region ID for the voice channel (1-64 characters)", "type": ["string", "null"] }, - "name": {"description": "The name of the channel", "type": ["string", "null"]} + "name": {"description": "The name of the channel", "type": ["string", "null"]}, + "type": { + "description": "Convert between text (0) and announcement (5). Other conversions are rejected.", + "anyOf": [{"type": "number", "const": 0}, {"type": "number", "const": 5}] + } + } + }, + { + "type": "object", + "properties": { + "topic": {"description": "The channel topic (1-1024 characters)", "type": ["string", "null"]}, + "url": {"description": "External URL for link channels", "type": ["string", "null"]}, + "parent_id": { + "description": "ID of the parent category for this channel", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"type": "null"}] + }, + "bitrate": { + "description": "Voice channel bitrate in bits per second (8000-384000), clamped to 96000 unless the guild holds an AUDIO_BITRATE feature", + "anyOf": [{"type": "integer", "minimum": 8000, "maximum": 384000}, {"type": "null"}] + }, + "user_limit": { + "description": "Maximum users allowed in voice channel (0-99, 0 means unlimited)", + "anyOf": [{"type": "integer", "minimum": 0, "maximum": 99}, {"type": "null"}] + }, + "voice_connection_limit": { + "description": "Maximum active voice connections allowed per user in a voice channel (1-100)", + "anyOf": [{"type": "integer", "minimum": 1, "maximum": 100}, {"type": "null"}] + }, + "permission_overwrites": { + "description": "Permission overwrites for roles and members", + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "description": "The ID of the role or user to overwrite permissions for", + "$ref": "#/components/schemas/SnowflakeType" + }, + "type": { + "description": "The type of overwrite (0 = role, 1 = member)", + "x-enumNames": ["ROLE", "MEMBER"], + "enum": [0, 1], + "type": "integer" + }, + "allow": { + "description": "Bitwise value of allowed permissions", + "$ref": "#/components/schemas/UnsignedInt64Type" + }, + "deny": { + "description": "Bitwise value of denied permissions", + "$ref": "#/components/schemas/UnsignedInt64Type" + } + }, + "required": ["id", "type"] + } + }, + "rate_limit_per_user": { + "description": "Slowmode delay in seconds (0-21600)", + "anyOf": [{"type": "integer", "minimum": 0, "maximum": 21600}, {"type": "null"}] + }, + "nsfw": { + "description": "Legacy: setting true maps to nsfw_override=true; setting false maps to nsfw_override=null (inherit). Prefer nsfw_override.", + "type": ["boolean", "null"] + }, + "nsfw_override": { + "description": "Per-channel adult-content override (true=on, false=off, null=inherit from category then guild). Takes precedence over the legacy `nsfw` field if both are present.", + "type": ["boolean", "null"] + }, + "content_warning_level": { + "description": "Channel-level content warning override (0=inherit, 1=force-warn)", + "$ref": "#/components/schemas/ContentWarningLevelInput" + }, + "content_warning_text": { + "description": "Custom channel content warning text (max 200 characters); null inherits from parent or guild", + "anyOf": [{"type": "string", "maxLength": 200}, {"type": "null"}] + }, + "icon": { + "description": "Base64-encoded icon image for group DM channels", + "anyOf": [{"$ref": "#/components/schemas/Base64ImageType"}, {"type": "null"}] + }, + "owner_id": { + "description": "ID of the new owner for group DM channels", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"type": "null"}] + }, + "nicks": { + "description": "Custom nicknames for users in this channel", + "$ref": "#/components/schemas/ChannelNicknameOverrides" + }, + "rtc_region": { + "description": "Voice region ID for the voice channel (1-64 characters)", + "type": ["string", "null"] + }, + "name": {"description": "The name of the channel", "type": ["string", "null"]}, + "type": { + "description": "Convert between text (0) and announcement (5). Other conversions are rejected.", + "anyOf": [{"type": "number", "const": 0}, {"type": "number", "const": 5}] + } } }, { @@ -31044,7 +31553,7 @@ "additionalProperties": false }, "GuildFeatureSchema": { - "description": "A guild feature flag Known values: ANIMATED_ICON, ANIMATED_BANNER, AUDIO_BITRATE_128_KBPS, AUDIO_BITRATE_256_KBPS, AUDIO_BITRATE_384_KBPS, BANNER, CLONE_EMOJI_DISABLED, CLONE_EMOJI_ENABLED, CLONE_STICKER_DISABLED, CLONE_STICKER_ENABLED, DETACHED_BANNER, INVITE_SPLASH, INVITES_DISABLED, RAID_DETECTED, TEXT_CHANNEL_FLEXIBLE_NAMES, HIDE_OWNER_CROWN, MORE_EMOJI, MORE_STICKERS, UNLIMITED_EMOJI, UNLIMITED_STICKERS, EXPRESSION_PURGE_ALLOWED, VANITY_URL, DISCOVERABLE, PARTNERED, VERIFIED, VIP_VOICE, VOICE_E2EE, UNAVAILABLE_FOR_EVERYONE, UNAVAILABLE_FOR_EVERYONE_BUT_STAFF, UNAVAILABLE_HIDDEN, VISIONARY, LARGE_GUILD_OVERRIDE, VERY_LARGE_GUILD (other values allowed)", + "description": "A guild feature flag Known values: ANIMATED_ICON, ANIMATED_BANNER, AUDIO_BITRATE_128_KBPS, AUDIO_BITRATE_256_KBPS, AUDIO_BITRATE_384_KBPS, BANNER, CLONE_EMOJI_DISABLED, CLONE_EMOJI_ENABLED, CLONE_STICKER_DISABLED, CLONE_STICKER_ENABLED, DETACHED_BANNER, INVITE_SPLASH, INVITES_DISABLED, RAID_DETECTED, TEXT_CHANNEL_FLEXIBLE_NAMES, HIDE_OWNER_CROWN, MORE_EMOJI, MORE_STICKERS, UNLIMITED_EMOJI, UNLIMITED_STICKERS, EXPRESSION_PURGE_ALLOWED, VANITY_URL, DISCOVERABLE, PARTNERED, VERIFIED, VIP_VOICE, VOICE_E2EE, UNAVAILABLE_FOR_EVERYONE, UNAVAILABLE_FOR_EVERYONE_BUT_STAFF, UNAVAILABLE_HIDDEN, VISIONARY, LARGE_GUILD_OVERRIDE, VERY_LARGE_GUILD, ANNOUNCEMENT_CHANNELS_DISABLED (other values allowed)", "x-enumNames": [ "ANIMATED_ICON", "ANIMATED_BANNER", @@ -31078,7 +31587,8 @@ "UNAVAILABLE_HIDDEN", "VISIONARY", "LARGE_GUILD_OVERRIDE", - "VERY_LARGE_GUILD" + "VERY_LARGE_GUILD", + "ANNOUNCEMENT_CHANNELS_DISABLED" ], "x-enumDescriptions": [ "Guild can have an animated icon", @@ -31113,14 +31623,15 @@ "Guild is hidden when it is force unavailable", "Guild is a visionary guild", "Guild has large guild overrides enabled", - "Guild has increased member capacity enabled" + "Guild has increased member capacity enabled", + "Guild cannot publish announcement messages or gain new followers" ], "type": "string" }, "ChannelType": { "description": "The type of the channel", "type": "integer", - "enum": [0, 1, 2, 3, 4, 998, 999], + "enum": [0, 1, 2, 3, 4, 5, 998, 999], "format": "int32", "x-enumNames": [ "GUILD_TEXT", @@ -31128,6 +31639,7 @@ "GUILD_VOICE", "GROUP_DM", "GUILD_CATEGORY", + "GUILD_ANNOUNCEMENT", "GUILD_LINK", "DM_PERSONAL_NOTES" ], @@ -31137,6 +31649,7 @@ "A voice channel within a guild", "A group direct message between users", "A category that contains channels", + "A guild channel whose messages can be published to channels that follow it", "A link channel for external resources", "Personal notes DM channel" ] @@ -31152,11 +31665,26 @@ "description": "The bitwise flags of the original message", "format": "int32", "x-bitflagValues": [ + { + "name": "CROSSPOSTED", + "value": "1", + "description": "This message has been published to channels that follow this announcement channel" + }, + { + "name": "IS_CROSSPOST", + "value": "2", + "description": "This message was delivered from an announcement channel this channel follows" + }, { "name": "SUPPRESS_EMBEDS", "value": "4", "description": "Do not include embeds when serialising this message" }, + { + "name": "SOURCE_MESSAGE_DELETED", + "value": "8", + "description": "The published message this copy came from has been deleted" + }, { "name": "SUPPRESS_NOTIFICATIONS", "value": "4096", @@ -31537,8 +32065,8 @@ "$ref": "#/components/schemas/SnowflakeStringType" }, "message_id": { - "description": "The ID of the referenced message", - "$ref": "#/components/schemas/SnowflakeStringType" + "description": "The ID of the referenced message, absent on a channel follow system message", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}, {"type": "null"}] }, "guild_id": { "description": "The ID of the guild containing the referenced message", @@ -31546,7 +32074,7 @@ }, "type": {"$ref": "#/components/schemas/MessageReferenceType"} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, {"type": "null"} @@ -31844,7 +32372,7 @@ "MessageType": { "description": "The type of message", "type": "integer", - "enum": [0, 1, 2, 3, 4, 5, 6, 7, 19], + "enum": [0, 1, 2, 3, 4, 5, 6, 7, 12, 19], "format": "int32", "x-enumNames": [ "DEFAULT", @@ -31855,6 +32383,7 @@ "CHANNEL_ICON_CHANGE", "CHANNEL_PINNED_MESSAGE", "USER_JOIN", + "CHANNEL_FOLLOW_ADD", "REPLY" ], "x-enumDescriptions": [ @@ -31866,6 +32395,7 @@ "A system message indicating the channel icon changed", "A system message indicating a message was pinned", "A system message indicating a user joined", + "System message posted when a channel starts following an announcement channel", "A reply message" ] }, @@ -32061,6 +32591,46 @@ } } }, + "CrosspostSourceGuildResponse": { + "type": "object", + "properties": { + "id": {"description": "The ID of the source community", "$ref": "#/components/schemas/SnowflakeStringType"}, + "name": {"type": "string", "description": "The name of the source community"}, + "icon": {"description": "The hash of the source community icon", "type": ["string", "null"]}, + "banner": {"description": "The hash of the source community banner", "type": ["string", "null"]}, + "description": { + "description": "The discovery description of the source community, or null when it is not listed in discovery", + "type": ["string", "null"] + }, + "features": { + "type": "array", + "items": {"$ref": "#/components/schemas/GuildFeatureSchema"}, + "description": "The public badge features of the source community: VERIFIED, PARTNERED and DISCOVERABLE only" + }, + "approximate_member_count": { + "anyOf": [{"$ref": "#/components/schemas/Int32Type"}, {"type": "null"}], + "description": "Approximate number of members in the source community, or null when unavailable" + }, + "approximate_presence_count": { + "anyOf": [{"$ref": "#/components/schemas/Int32Type"}, {"type": "null"}], + "description": "Approximate number of online members in the source community, or null when unavailable" + }, + "discoverable": { + "type": "boolean", + "description": "Whether the source community can be joined through discovery" + } + }, + "required": [ + "id", + "name", + "description", + "features", + "approximate_member_count", + "approximate_presence_count", + "discoverable" + ], + "additionalProperties": false + }, "ChannelOverwriteType": { "description": "The type of entity the overwrite applies to", "x-enumNames": ["ROLE", "MEMBER"], @@ -32078,6 +32648,14 @@ "required": ["id", "name", "emoji"], "additionalProperties": false }, + "WebhookType": { + "description": "The type of webhook", + "type": "integer", + "enum": [1, 2], + "format": "int32", + "x-enumNames": ["INCOMING", "CHANNEL_FOLLOWER"], + "x-enumDescriptions": ["Incoming webhook", "Channel follower webhook"] + }, "DiscoveryCategoryResponse": { "type": "object", "properties": { @@ -32507,14 +33085,6 @@ "Message was unpinned" ] }, - "WebhookType": { - "description": "The type of webhook", - "type": "integer", - "enum": [1, 2], - "format": "int32", - "x-enumNames": ["INCOMING", "CHANNEL_FOLLOWER"], - "x-enumDescriptions": ["Incoming webhook", "Channel follower webhook"] - }, "GuildBanResponse": { "type": "object", "properties": { @@ -32773,6 +33343,82 @@ }, "required": ["type", "name"] }, + "GuildAnnouncementChannelCreateRequest": { + "type": "object", + "properties": { + "topic": {"description": "The channel topic (1-1024 characters)", "type": ["string", "null"]}, + "url": {"description": "External URL for link channels", "type": ["string", "null"]}, + "parent_id": { + "description": "ID of the parent category for this channel", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"type": "null"}] + }, + "bitrate": { + "description": "Voice channel bitrate in bits per second (8000-384000), clamped to 96000 unless the guild holds an AUDIO_BITRATE feature", + "anyOf": [{"type": "integer", "minimum": 8000, "maximum": 384000}, {"type": "null"}] + }, + "user_limit": { + "description": "Maximum users allowed in voice channel (0-99, 0 means unlimited)", + "anyOf": [{"type": "integer", "minimum": 0, "maximum": 99}, {"type": "null"}] + }, + "voice_connection_limit": { + "description": "Maximum active voice connections allowed per user in a voice channel (1-100)", + "anyOf": [{"type": "integer", "minimum": 1, "maximum": 100}, {"type": "null"}] + }, + "permission_overwrites": { + "description": "Permission overwrites for roles and members", + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "description": "The ID of the role or user to overwrite permissions for", + "$ref": "#/components/schemas/SnowflakeType" + }, + "type": { + "description": "The type of overwrite (0 = role, 1 = member)", + "x-enumNames": ["ROLE", "MEMBER"], + "enum": [0, 1], + "type": "integer" + }, + "allow": { + "description": "Bitwise value of allowed permissions", + "$ref": "#/components/schemas/UnsignedInt64Type" + }, + "deny": { + "description": "Bitwise value of denied permissions", + "$ref": "#/components/schemas/UnsignedInt64Type" + } + }, + "required": ["id", "type"] + } + }, + "rate_limit_per_user": { + "description": "Slowmode delay in seconds (0-21600)", + "anyOf": [{"type": "integer", "minimum": 0, "maximum": 21600}, {"type": "null"}] + }, + "nsfw": {"default": false, "description": "Whether the channel is marked as NSFW", "type": "boolean"}, + "nsfw_override": { + "description": "Per-channel adult-content override (true=on, false=off, null=inherit from category then guild). Takes precedence over the legacy `nsfw` field if both are present.", + "type": ["boolean", "null"] + }, + "content_warning_level": { + "description": "Channel-level content warning override (0=inherit, 1=force-warn)", + "$ref": "#/components/schemas/ContentWarningLevelInput" + }, + "content_warning_text": { + "description": "Custom channel content warning text (max 200 characters); null inherits from parent or guild", + "anyOf": [{"type": "string", "maxLength": 200}, {"type": "null"}] + }, + "type": { + "type": "integer", + "x-enumNames": ["GUILD_ANNOUNCEMENT"], + "x-enumDescriptions": ["Channel type (announcement channel)"], + "enum": [5] + }, + "name": {"description": "The name of the channel", "type": "string"} + }, + "required": ["type", "name"] + }, "GuildTextChannelCreateRequest": { "type": "object", "properties": { @@ -34474,8 +35120,8 @@ "$ref": "#/components/schemas/SnowflakeStringType" }, "message_id": { - "description": "The ID of the referenced message", - "$ref": "#/components/schemas/SnowflakeStringType" + "description": "The ID of the referenced message, absent on a channel follow system message", + "anyOf": [{"$ref": "#/components/schemas/SnowflakeStringType"}, {"type": "null"}] }, "guild_id": { "description": "The ID of the guild containing the referenced message", @@ -34483,7 +35129,7 @@ }, "type": {"$ref": "#/components/schemas/MessageReferenceType"} }, - "required": ["channel_id", "message_id", "type"], + "required": ["channel_id", "type"], "additionalProperties": false }, {"type": "null"} diff --git a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts index b9824196b..0e87cb09e 100644 --- a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts +++ b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts @@ -56,6 +56,14 @@ export const ChannelRateLimitConfigs = { bucket: 'channel:message:ack::channel_id', config: {limit: 100, windowMs: ms('10 seconds')}, } as RouteRateLimitConfig, + CHANNEL_MESSAGE_CROSSPOST: { + bucket: 'channel:message:crosspost::channel_id', + config: {limit: 5, windowMs: ms('5 seconds')}, + } as RouteRateLimitConfig, + CHANNEL_MESSAGE_CROSSPOST_SOURCE: { + bucket: 'channel:message:crosspost_source::channel_id', + config: {limit: 20, windowMs: ms('10 seconds')}, + } as RouteRateLimitConfig, CHANNEL_SEARCH: { bucket: 'channel:search::channel_id', config: {limit: 20, windowMs: ms('10 seconds')}, @@ -120,6 +128,14 @@ export const ChannelRateLimitConfigs = { bucket: 'channel:stream:preview:delete::stream_key', config: {limit: 20, windowMs: ms('10 seconds')}, } as RouteRateLimitConfig, + CHANNEL_FOLLOW: { + bucket: 'channel:follow::channel_id', + config: {limit: 5, windowMs: ms('10 seconds')}, + } as RouteRateLimitConfig, + CHANNEL_FOLLOWER_STATS: { + bucket: 'channel:follower_stats::channel_id', + config: {limit: 10, windowMs: ms('10 seconds')}, + } as RouteRateLimitConfig, VOICE_ENTRANCE_SOUND_PLAY: { bucket: 'voice:entrance_sound:play::user_id::channel_id', config: {limit: 3, windowMs: ms('30 seconds')}, diff --git a/fluxer_api/src/api/report/ReportService.ts b/fluxer_api/src/api/report/ReportService.ts index eb28c003c..3d320e03d 100644 --- a/fluxer_api/src/api/report/ReportService.ts +++ b/fluxer_api/src/api/report/ReportService.ts @@ -3,6 +3,7 @@ import {createHash, randomBytes, randomInt} from 'node:crypto'; import type {ChannelID, GuildID, InviteCode, MessageID, ReportID, UserID} from '@app/api/BrandedTypes'; import { + createAttachmentID, createChannelID, createGuildID, createInviteCode, @@ -48,7 +49,7 @@ import {ReportStatus, ReportType} from '@app/api/report/IReportRepository'; import type {IReportSearchService} from '@app/api/search/IReportSearchService'; import type {IUserRepository} from '@app/api/user/IUserRepository'; import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; -import {InviteTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import {InviteTypes, MessageFlags, Permissions} from '@fluxer/constants/src/ChannelConstants'; import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; import {UserFlags} from '@fluxer/constants/src/UserConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; @@ -155,7 +156,7 @@ export class ReportService { channelId, messageId, }); - const reportedUserId = message.authorId; + const reportedUserId = await this.resolveReportedAuthorId(message); if (reportedUserId == null) { throw new UnknownMessageError(); } @@ -164,7 +165,7 @@ export class ReportService { } const [reportedUser, messageContext] = await Promise.all([ this.userRepository.findUnique(reportedUserId), - this.gatherMessageContext(channelId, messageId, authChannel), + this.gatherMessageContext(channelId, messageId, authChannel, reportedUserId), ]); if (!reportedUser) { throw new UnknownUserError(); @@ -464,20 +465,21 @@ export class ReportService { if (!channel) throw new UnknownChannelError(); const message = await this.channelRepository.getMessage(channelId, messageId); if (!message) throw new UnknownMessageError(); - if (message.authorId == null) { + const reportedAuthorId = await this.resolveReportedAuthorId(message); + if (reportedAuthorId == null) { throw new UnknownUserError(); } if (report.reported_user_tag) { const tagged = await this.findUserByTag(report.reported_user_tag); - if (tagged.id !== message.authorId) { + if (tagged.id !== reportedAuthorId) { throw new InvalidDsaReportTargetError(); } } - const reportedUser = await this.userRepository.findUnique(message.authorId); + const reportedUser = await this.userRepository.findUnique(reportedAuthorId); if (!reportedUser) { throw new UnknownUserError(); } - const messageContext = await this.gatherMessageContext(channelId, messageId); + const messageContext = await this.gatherMessageContext(channelId, messageId, undefined, reportedAuthorId); const guild = channel.guildId ? await this.guildRepository.findUnique(channel.guildId) : null; const contentWarningSnapshot = await this.buildContentWarningSnapshot(guild, channel); return { @@ -491,7 +493,7 @@ export class ReportService { report_type: ReportType.MESSAGE, category: report.category, additional_info: report.additional_info ?? null, - reported_user_id: message.authorId, + reported_user_id: reportedAuthorId, reported_user_avatar_hash: reportedUser.avatarHash || null, reported_guild_id: channel.guildId || null, reported_guild_name: guild?.name ?? null, @@ -637,6 +639,24 @@ export class ReportService { }; } + private async resolveReportedAuthorId(message: Message): Promise { + if ((message.flags & MessageFlags.IS_CROSSPOST) === 0) { + return message.authorId; + } + if ((message.flags & MessageFlags.SOURCE_MESSAGE_DELETED) !== 0) { + return null; + } + const reference = message.reference; + if (!reference?.messageId) { + return null; + } + const source = await this.channelRepository.getMessage(reference.channelId, reference.messageId); + if (!source || (source.flags & MessageFlags.IS_CROSSPOST) !== 0) { + return null; + } + return source.authorId; + } + private async canAccessMessage(authChannel: AuthenticatedChannel, messageId: MessageID): Promise { if (!authChannel.guild) { return true; @@ -879,6 +899,7 @@ export class ReportService { channelId: ChannelID, targetMessageId: MessageID, authChannel?: AuthenticatedChannel, + targetAuthorId?: UserID, ): Promise> { const messagesBefore = await this.channelRepository.listMessages( channelId, @@ -900,10 +921,13 @@ export class ReportService { [...messagesBefore, targetMessage, ...messagesAfter], authChannel, ); + const contextAuthorId = (msg: Message): UserID | null => + msg.authorId ?? (msg.id === targetMessageId ? (targetAuthorId ?? null) : null); const userIds = new Set(); for (const msg of allMessages) { - if (msg.authorId) { - userIds.add(msg.authorId); + const authorId = contextAuthorId(msg); + if (authorId) { + userIds.add(authorId); } } const users = new Map(); @@ -915,15 +939,23 @@ export class ReportService { } const context: Array = []; for (const message of allMessages) { - const author = message.authorId != null ? users.get(message.authorId) : null; - if (!author) continue; - const clonedAttachments = message.attachments - ? await this.cloneAttachmentsForReport(message.attachments, channelId) - : []; + const authorId = contextAuthorId(message); + const author = authorId != null ? users.get(authorId) : null; + if (!author || authorId == null) continue; + const clonedAttachments = [ + ...(message.attachments + ? await this.cloneAttachmentsForReport( + message.attachments, + MessageHelpers.attachmentStorageChannelId(message), + channelId, + ) + : []), + ...(await this.cloneOwnedEmbedAttachmentsForReport(message, channelId)), + ]; context.push({ message_id: message.id, channel_id: channelId, - author_id: message.authorId!, + author_id: authorId, author_username: author.username, author_discriminator: author.discriminator, author_avatar_hash: author.avatarHash || null, @@ -966,6 +998,7 @@ export class ReportService { private async cloneAttachmentsForReport( attachments: Array, sourceChannelId: ChannelID, + reportedChannelId: ChannelID, ): Promise> { const clonedAttachments: Array = []; for (const attachment of attachments) { @@ -975,7 +1008,7 @@ export class ReportService { sourceBucket: Config.s3.buckets.cdn, sourceKey, destinationBucket: Config.s3.buckets.reports, - destinationKey: sourceKey, + destinationKey: MessageHelpers.makeAttachmentCdnKey(reportedChannelId, attachment.id, attachment.filename), newContentType: attachment.contentType, }); const clonedAttachment: MessageAttachment = { @@ -1005,6 +1038,49 @@ export class ReportService { return clonedAttachments; } + private async cloneOwnedEmbedAttachmentsForReport( + message: Message, + reportedChannelId: ChannelID, + ): Promise> { + const clonedAttachments: Array = []; + for (const {key, media} of MessageHelpers.collectOwnedEmbedAttachments(message)) { + const [, , attachmentId, ...filenameParts] = key.split('/'); + const filename = filenameParts.join('/'); + try { + const metadata = await this.storageService.getObjectMetadata(Config.s3.buckets.cdn, key); + if (!metadata) continue; + const id = createAttachmentID(BigInt(attachmentId!)); + const contentType = media.content_type ?? metadata.contentType; + await this.storageService.copyObject({ + sourceBucket: Config.s3.buckets.cdn, + sourceKey: key, + destinationBucket: Config.s3.buckets.reports, + destinationKey: MessageHelpers.makeAttachmentCdnKey(reportedChannelId, id, filename), + newContentType: contentType, + }); + clonedAttachments.push({ + attachment_id: id, + filename, + size: BigInt(metadata.contentLength), + title: null, + description: media.description, + width: media.width, + height: media.height, + content_type: contentType, + content_hash: media.content_hash, + placeholder: media.placeholder, + flags: media.flags & ~MessageHelpers.EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, + duration: media.duration, + nsfw: null, + waveform: null, + }); + } catch (error) { + Logger.error({error, key, reportedChannelId}, 'Failed to clone embed attachment for report'); + } + } + return clonedAttachments; + } + private async checkReportBan(userId: UserID | null): Promise { if (!userId) { return; diff --git a/fluxer_api/src/api/report/tests/CrosspostReport.test.ts b/fluxer_api/src/api/report/tests/CrosspostReport.test.ts new file mode 100644 index 000000000..9873fed2b --- /dev/null +++ b/fluxer_api/src/api/report/tests/CrosspostReport.test.ts @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + clearTestEmails, + createUniqueEmail, + findLastTestEmail, + listTestEmails, + type TestAccount, +} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createGuildID, createMessageID, createReportID} from '@app/api/BrandedTypes'; +import {type AnnouncementWorld, announcementWorld} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {sendChannelMessage} from '@app/api/channel/tests/ChannelTestUtils'; +import {readMessageRow, writeMessageRow} from '@app/api/channel/tests/CrosspostTestUtils'; +import {ReportRepository} from '@app/api/report/ReportRepository'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, executeWebhook} from '@app/api/webhook/tests/WebhookTestUtils'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {MessageFlags, MessageReferenceTypes} from '@fluxer/constants/src/ChannelConstants'; +import {afterEach, beforeEach, describe, expect, test} from 'vitest'; + +interface ReportResponse { + report_id: string; + status: string; +} + +interface CopyFixture { + world: AnnouncementWorld; + sourceId: string; + copyId: string; +} + +async function writeCopy(params: { + harness: ApiTestHarness; + owner: TestAccount; + targetChannelId: string; + sourceGuildId: string; + sourceChannelId: string; + sourceMessageId: string; + content: string; + extraFlags?: number; +}): Promise { + const webhook = await createWebhook(params.harness, params.targetChannelId, params.owner.token, 'Follower'); + const {json} = await executeWebhook( + params.harness, + webhook.id, + webhook.token, + {content: params.content, wait: true}, + 200, + ); + const copyId = json!.id; + const row = await readMessageRow(params.targetChannelId, copyId); + await writeMessageRow(row!, { + flags: MessageFlags.IS_CROSSPOST | (params.extraFlags ?? 0), + message_reference: { + channel_id: createChannelID(BigInt(params.sourceChannelId)), + guild_id: createGuildID(BigInt(params.sourceGuildId)), + message_id: createMessageID(BigInt(params.sourceMessageId)), + type: MessageReferenceTypes.DEFAULT, + }, + }); + return copyId; +} + +async function createCopyOfMemberMessage(harness: ApiTestHarness, extraFlags?: number): Promise { + const world = await announcementWorld(harness); + const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'published update'); + const copyId = await writeCopy({ + harness, + owner: world.b.owner, + targetChannelId: world.b.t1.id, + sourceGuildId: world.a.guild.id, + sourceChannelId: world.a.ann.id, + sourceMessageId: source.id, + content: 'published update', + extraFlags, + }); + return {world, sourceId: source.id, copyId}; +} + +function reportMessage(harness: ApiTestHarness, token: string, channelId: string, messageId: string) { + return createBuilder(harness, token) + .post('/reports/message') + .body({channel_id: channelId, message_id: messageId, category: 'harassment'}); +} + +async function readReport(reportId: string) { + const report = await new ReportRepository().getReport(createReportID(BigInt(reportId))); + expect(report).not.toBeNull(); + return report!; +} + +async function createDsaTicket(harness: ApiTestHarness): Promise { + await clearTestEmails(harness); + const email = createUniqueEmail('dsa-crosspost'); + await createBuilderWithoutAuth(harness).post('/reports/dsa/email/send').body({email}).execute(); + const dsaEmail = findLastTestEmail(await listTestEmails(harness), 'dsa_report_verification'); + const {ticket} = await createBuilder<{ticket: string}>(harness, '') + .post('/reports/dsa/email/verify') + .body({email, code: dsaEmail!.metadata.code}) + .expect(HTTP_STATUS.OK) + .execute(); + return ticket; +} + +function dsaMessageReport(harness: ApiTestHarness, ticket: string, channelId: string, messageId: string) { + return createBuilderWithoutAuth(harness) + .post('/reports/dsa') + .body({ + ticket, + report_type: 'message', + category: 'harassment', + message_link: `https://fluxer.test/channels/0/${channelId}/${messageId}`, + reporter_full_legal_name: 'Jane Doe', + reporter_country_of_residence: 'DE', + }); +} + +describe('Reporting published copies', () => { + let harness: ApiTestHarness; + + beforeEach(async () => { + harness = await createApiTestHarness(); + }); + + afterEach(async () => { + await harness?.shutdown(); + }); + + test('a member of the target reports a copy and the source author is reported', async () => { + const {world, copyId} = await createCopyOfMemberMessage(harness); + const result = await reportMessage(harness, world.b.webhookManager.token, world.b.t1.id, copyId) + .expect(HTTP_STATUS.OK) + .execute(); + const report = await readReport(result.report_id); + expect(report.reportedUserId?.toString()).toBe(world.a.member.userId); + expect(report.reportedMessageId?.toString()).toBe(copyId); + expect(report.reportedChannelId?.toString()).toBe(world.b.t1.id); + expect(report.reportedGuildId?.toString()).toBe(world.b.guild.id); + const reportedContext = report.messageContext?.find((entry) => entry.messageId.toString() === copyId); + expect(reportedContext?.authorId.toString()).toBe(world.a.member.userId); + expect(reportedContext?.content).toBe('published update'); + }); + + test('the source author cannot report a copy of their own message', async () => { + const world = await announcementWorld(harness); + const source = await sendChannelMessage(harness, world.b.owner.token, world.a.ann.id, 'my own update'); + const copyId = await writeCopy({ + harness, + owner: world.b.owner, + targetChannelId: world.b.t1.id, + sourceGuildId: world.a.guild.id, + sourceChannelId: world.a.ann.id, + sourceMessageId: source.id, + content: 'my own update', + }); + await reportMessage(harness, world.b.owner.token, world.b.t1.id, copyId) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CANNOT_REPORT_OWN_MESSAGE) + .execute(); + }); + + test('the message-link report on a copy reports the source author', async () => { + const {world, copyId} = await createCopyOfMemberMessage(harness); + const ticket = await createDsaTicket(harness); + const result = await dsaMessageReport(harness, ticket, world.b.t1.id, copyId).expect(HTTP_STATUS.OK).execute(); + const report = await readReport(result.report_id); + expect(report.reportedUserId?.toString()).toBe(world.a.member.userId); + expect(report.reportedMessageId?.toString()).toBe(copyId); + expect(report.reportedChannelId?.toString()).toBe(world.b.t1.id); + }); + + test('a copy of a webhook-authored source keeps the existing errors', async () => { + const world = await announcementWorld(harness); + const sourceWebhook = await createWebhook(harness, world.a.ann.id, world.a.owner.token, 'Source hook'); + const {json: source} = await executeWebhook( + harness, + sourceWebhook.id, + sourceWebhook.token, + {content: 'from a webhook', wait: true}, + 200, + ); + const copyId = await writeCopy({ + harness, + owner: world.b.owner, + targetChannelId: world.b.t1.id, + sourceGuildId: world.a.guild.id, + sourceChannelId: world.a.ann.id, + sourceMessageId: source!.id, + content: 'from a webhook', + }); + await reportMessage(harness, world.b.webhookManager.token, world.b.t1.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + const ticket = await createDsaTicket(harness); + await dsaMessageReport(harness, ticket, world.b.t1.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_USER) + .execute(); + }); + + test('a source-deleted copy keeps the existing errors', async () => { + const {world, copyId} = await createCopyOfMemberMessage(harness, MessageFlags.SOURCE_MESSAGE_DELETED); + await reportMessage(harness, world.b.webhookManager.token, world.b.t1.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + const ticket = await createDsaTicket(harness); + await dsaMessageReport(harness, ticket, world.b.t1.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_USER) + .execute(); + }); + + test('a copy whose source is gone keeps the existing error', async () => { + const {world, sourceId, copyId} = await createCopyOfMemberMessage(harness); + await createBuilder(harness, world.a.member.token) + .delete(`/channels/${world.a.ann.id}/messages/${sourceId}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + await reportMessage(harness, world.b.webhookManager.token, world.b.t1.id, copyId) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + }); + + test('reports on ordinary webhook messages are unchanged', async () => { + const world = await announcementWorld(harness); + const webhook = await createWebhook(harness, world.b.t1.id, world.b.owner.token, 'Plain hook'); + const {json} = await executeWebhook(harness, webhook.id, webhook.token, {content: 'plain', wait: true}, 200); + await reportMessage(harness, world.b.webhookManager.token, world.b.t1.id, json!.id) + .expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_MESSAGE) + .execute(); + }); +}); diff --git a/fluxer_api/src/api/test/SyncTaskWorkerService.ts b/fluxer_api/src/api/test/SyncTaskWorkerService.ts index fa8756a57..5d99621b5 100644 --- a/fluxer_api/src/api/test/SyncTaskWorkerService.ts +++ b/fluxer_api/src/api/test/SyncTaskWorkerService.ts @@ -2,26 +2,79 @@ import {NoopLogger} from '@app/api/test/mocks/NoopLogger'; import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; -import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; +import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask'; import type {WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; type TaskHandlerMap = Record; let nextSyntheticJobId = 1n; +export interface SyncTaskWorkerServiceOptions { + deferred?: boolean; +} + +export interface QueuedWorkerJob { + jobId: bigint; + taskType: string; + payload: WorkerJobPayload; + jobKey: string | undefined; + runAt: Date | undefined; + options: WorkerJobOptions | undefined; +} + +export interface DeadLetterWorkerJob extends QueuedWorkerJob { + attempts: number; + error: unknown; +} + +export interface DrainOptions { + maxJobs?: number; + maxAttempts?: number; +} + +export interface DrainResult { + ran: number; + failed: number; + remaining: number; +} + +interface PendingJob { + job: QueuedWorkerJob; + attempts: number; +} + export class SyncTaskWorkerService implements IWorkerService { private handlers: TaskHandlerMap; + private readonly deferred: boolean; + private queue: Array = []; + private readonly activeJobKeys = new Set(); + readonly recorded: Array = []; + readonly deduped: Array = []; + readonly deadLetters: Array = []; + failure: ((taskType: string, payload: WorkerJobPayload) => Error | null) | null = null; - constructor(handlers: TaskHandlerMap) { + constructor(handlers: TaskHandlerMap, options: SyncTaskWorkerServiceOptions = {}) { this.handlers = handlers; + this.deferred = options.deferred ?? false; } async addJob( taskType: string, payload: TPayload, - _options?: WorkerJobOptions, + options?: WorkerJobOptions, ): Promise { const jobId = nextSyntheticJobId++; + if (this.deferred) { + this.enqueue({ + jobId, + taskType, + payload, + jobKey: options?.jobKey, + runAt: options?.runAt, + options, + }); + return jobId; + } const handler = this.handlers[taskType]; if (handler) { await handler(payload, { @@ -43,4 +96,81 @@ export class SyncTaskWorkerService implements IWorkerService { async retryDeadLetterJob(_jobId: bigint): Promise { return false; } + + get pending(): ReadonlyArray { + return this.queue.map((entry) => entry.job); + } + + byTask(taskType: string): Array { + return this.recorded.filter((job) => job.taskType === taskType); + } + + async drain({maxJobs = 10_000, maxAttempts = 3}: DrainOptions = {}): Promise { + let ran = 0; + let failed = 0; + while (this.queue.length > 0 && ran < maxJobs) { + const entry = this.queue.shift()!; + ran++; + entry.attempts++; + try { + await this.runJob(entry, maxAttempts); + } catch (error) { + failed++; + if (entry.attempts < maxAttempts) { + this.queue.push(entry); + } else { + this.deadLetters.push({...entry.job, attempts: entry.attempts, error}); + } + } + } + if (this.queue.length === 0) { + this.activeJobKeys.clear(); + } + return {ran, failed, remaining: this.queue.length}; + } + + clear(): void { + this.queue = []; + this.activeJobKeys.clear(); + this.recorded.length = 0; + this.deduped.length = 0; + this.deadLetters.length = 0; + } + + private enqueue(job: QueuedWorkerJob): void { + const error = this.failure?.(job.taskType, job.payload) ?? null; + if (error) { + throw error; + } + if (job.jobKey !== undefined) { + if (this.activeJobKeys.has(job.jobKey)) { + this.deduped.push(job); + return; + } + this.activeJobKeys.add(job.jobKey); + } + this.recorded.push(job); + this.queue.push({job, attempts: 0}); + } + + private async runJob(entry: PendingJob, maxAttempts: number): Promise { + const handler = this.handlers[entry.job.taskType]; + if (!handler) { + return; + } + const helpers: WorkerTaskHelpers = { + logger: new NoopLogger(), + jobId: entry.job.jobId, + attempt: {isLastAttempt: entry.attempts >= maxAttempts}, + addJob: async (taskType, payload, options) => { + const jobId = nextSyntheticJobId++; + this.enqueue({jobId, taskType, payload, jobKey: options?.jobKey, runAt: options?.runAt, options}); + return jobId; + }, + reportProgress: async () => {}, + shouldCancel: async () => false, + setContextLink: async () => {}, + }; + await handler(entry.job.payload, helpers); + } } diff --git a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts index 690386173..03ae9f246 100644 --- a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts +++ b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts @@ -5,6 +5,11 @@ import {AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService import {makeSignedAttachmentCdnUrl, signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; +import { + attachmentStorageChannelId, + EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, + isCrosspostCopy, +} from '@app/api/channel/services/message/MessageHelpers'; import { type MessageResponseAccessContext, MessageResponseDataService, @@ -267,7 +272,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD message_reference: message.reference ? { channel_id: message.reference.channelId.toString(), - message_id: message.reference.messageId.toString(), + ...(message.reference.messageId ? {message_id: message.reference.messageId.toString()} : {}), guild_id: message.reference.guildId?.toString() ?? null, type: message.reference.type, } @@ -290,6 +295,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD global_name: message.webhookName ?? DELETED_USER_GLOBAL_NAME, avatar: message.webhookAvatarHash, avatar_color: null, + ...(message.webhookId ? {bot: true} : {}), flags: 0, }; } @@ -333,7 +339,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD message: Message, options: {currentUserId?: UserID; includeReactions: boolean; depth: number}, ): Promise { - if (!message.reference || options.depth > 0) { + if (!message.reference?.messageId || (message.flags & MessageFlags.IS_CROSSPOST) !== 0 || options.depth > 0) { return undefined; } const referenced = await getChannelRepository().messages.getMessage( @@ -351,7 +357,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD } private mapAttachmentUrl(message: Message, attachment: Attachment): string { - return makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename); + return makeSignedAttachmentCdnUrl(attachmentStorageChannelId(message), attachment.id, attachment.filename); } private async mapAttachments( @@ -359,14 +365,15 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD message: Message, ): Promise | null> { if (attachments.length === 0) return null; + const ownerMessageId = isCrosspostCopy(message) ? (message.reference?.messageId ?? message.id) : message.id; await this.attachmentDecayService.extendForAttachments( attachments.map((attachment) => ({ attachmentId: attachment.id, - channelId: message.channelId, - messageId: message.id, + channelId: attachmentStorageChannelId(message), + messageId: ownerMessageId, filename: attachment.filename, sizeBytes: attachment.size, - uploadedAt: snowflakeToDate(message.id), + uploadedAt: snowflakeToDate(ownerMessageId), })), ); return Promise.all( @@ -468,7 +475,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD description: media.description, placeholder: media.placeholder, duration: media.duration, - flags: media.flags, + flags: media.flags & ~EMBED_MEDIA_OWNED_ATTACHMENT_FLAG, }; } diff --git a/fluxer_api/src/api/types/HonoEnv.ts b/fluxer_api/src/api/types/HonoEnv.ts index b246c8d9b..3ae3a5d94 100644 --- a/fluxer_api/src/api/types/HonoEnv.ts +++ b/fluxer_api/src/api/types/HonoEnv.ts @@ -76,6 +76,7 @@ import type {UserContentRequestService} from '@app/api/user/services/UserContent import type {UserRelationshipRequestService} from '@app/api/user/services/UserRelationshipRequestService'; import type {UserService} from '@app/api/user/services/UserService'; import type {ClientIpResolution} from '@app/api/utils/RequestClientIp'; +import type {ChannelFollowService} from '@app/api/webhook/ChannelFollowService'; import type {SweegoWebhookService} from '@app/api/webhook/SweegoWebhookService'; import type {WebhookRequestService} from '@app/api/webhook/WebhookRequestService'; import type {WebhookService} from '@app/api/webhook/WebhookService'; @@ -174,6 +175,7 @@ export interface HonoEnv { userRelationshipRequestService: UserRelationshipRequestService; sweegoWebhookService: SweegoWebhookService; webhookService: WebhookService; + channelFollowService: ChannelFollowService; webhookRequestService: WebhookRequestService; workerService: IWorkerService; stripeService: StripeService; diff --git a/fluxer_api/src/api/webhook/ChannelFollowService.ts b/fluxer_api/src/api/webhook/ChannelFollowService.ts new file mode 100644 index 000000000..23246a040 --- /dev/null +++ b/fluxer_api/src/api/webhook/ChannelFollowService.ts @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {ChannelID, GuildID, UserID, WebhookID} from '@app/api/BrandedTypes'; +import {createMessageID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes'; +import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import {withChannelFollowLock} from '@app/api/channel/services/ChannelFollowers'; +import type {ChannelService} from '@app/api/channel/services/ChannelService'; +import type {GuildService} from '@app/api/guild/services/GuildService'; +import type {AvatarService} from '@app/api/infrastructure/AvatarService'; +import {contentModerationService} from '@app/api/infrastructure/ContentModerationService'; +import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService'; +import {Logger} from '@app/api/Logger'; +import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; +import type {Channel} from '@app/api/models/Channel'; +import type {Webhook} from '@app/api/models/Webhook'; +import * as RandomUtils from '@app/api/utils/RandomUtils'; +import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; +import type {WebhookService} from '@app/api/webhook/WebhookService'; +import { + CHANNEL_FOLLOWER_STATS_CACHE_SECONDS, + FOLLOWER_WEBHOOK_NAME_MAX_LENGTH, +} from '@fluxer/constants/src/AnnouncementConstants'; +import { + CHANNEL_FOLLOW_TARGET_TYPES, + ChannelTypes, + MessageTypes, + Permissions, + WebhookTypes, +} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import {AnnouncementChannelRequiredError} from '@fluxer/errors/src/domains/channel/AnnouncementChannelRequiredError'; +import {InvalidFollowTargetChannelError} from '@fluxer/errors/src/domains/channel/InvalidFollowTargetChannelError'; +import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError'; +import type {ChannelFollowerStatsResponse} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import {WebhookNameType} from '@fluxer/schema/src/primitives/UserValidators'; +import type {ICacheService} from '@pkgs/cache/src/ICacheService'; + +interface FollowChannelParams { + userId: UserID; + channelId: ChannelID; + webhookChannelId: ChannelID; + requestCache: RequestCache; + auditLogReason?: string | null; +} + +interface FollowedChannel { + channelId: ChannelID; + webhookId: WebhookID; +} + +type GuildChannel = Channel & {guildId: GuildID}; + +function truncateToLength(value: string, maxLength: number): string { + let result = ''; + for (const codePoint of value) { + if (result.length + codePoint.length > maxLength) break; + result += codePoint; + } + return result; +} + +export class ChannelFollowService { + constructor( + private readonly webhookService: WebhookService, + private readonly webhookRepository: IWebhookRepository, + private readonly channelService: ChannelService, + private readonly channelRepository: IChannelRepository, + private readonly guildService: GuildService, + private readonly avatarService: AvatarService, + private readonly cacheService: ICacheService, + private readonly snowflakeService: ISnowflakeService, + ) {} + + async followChannel(params: FollowChannelParams): Promise { + const {userId, channelId, webhookChannelId, requestCache, auditLogReason} = params; + const sourceAuth = await this.channelService.channelData.auth.getChannelAuthenticated({userId, channelId}); + const source = sourceAuth.channel; + const sourceGuild = sourceAuth.guild; + if (source.type !== ChannelTypes.GUILD_ANNOUNCEMENT || !source.guildId || !sourceGuild) { + throw new AnnouncementChannelRequiredError(); + } + if (sourceGuild.features.includes(GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED)) { + throw new FeatureTemporarilyDisabledError(); + } + const targetAuth = await this.channelService.channelData.auth.getChannelAuthenticated({ + userId, + channelId: webhookChannelId, + }); + const target = targetAuth.channel; + if (!target.guildId || !CHANNEL_FOLLOW_TARGET_TYPES.has(target.type)) { + throw new InvalidFollowTargetChannelError(); + } + const targetChannel = target as GuildChannel; + const {checkPermission, guildData: targetGuild} = await this.guildService.getGuildAuthenticated({ + userId, + guildId: targetChannel.guildId, + }); + await checkPermission(Permissions.MANAGE_WEBHOOKS); + await this.webhookService.assertChannelWebhookPermission({ + userId, + guildId: targetChannel.guildId, + channelId: targetChannel.id, + }); + await this.webhookService.assertFollowContentRules({ + sourceChannel: source, + sourceGuild, + targetChannel, + targetGuild, + }); + const webhook = await withChannelFollowLock(this.cacheService, targetChannel.id, async () => { + const current = await this.channelRepository.findUnique(targetChannel.id); + if (!current?.guildId || !CHANNEL_FOLLOW_TARGET_TYPES.has(current.type)) { + throw new InvalidFollowTargetChannelError(); + } + await this.webhookService.assertChannelNotFollowing({ + channelId: targetChannel.id, + sourceChannelId: source.id, + }); + await this.webhookService.assertWebhookCapacity({ + guildId: targetChannel.guildId, + channelId: targetChannel.id, + guildFeatures: targetGuild.features, + }); + const name = this.resolveFollowerWebhookName({ + sourceGuild, + source, + userId, + targetChannel, + }); + const webhookId = createWebhookID(await this.snowflakeService.generate()); + const avatarHash = await this.avatarService.copyGuildIconToWebhookAvatar({ + guildId: source.guildId!, + iconHash: sourceGuild.icon ?? null, + webhookId, + }); + return this.webhookRepository.create({ + webhookId, + token: createWebhookToken(RandomUtils.randomString(64)), + type: WebhookTypes.CHANNEL_FOLLOWER, + guildId: targetChannel.guildId, + channelId: targetChannel.id, + creatorId: userId, + name, + avatarHash, + sourceGuildId: source.guildId, + sourceChannelId: source.id, + }); + }); + const currentSource = await this.channelRepository.findUnique(source.id); + if (!currentSource || currentSource.type !== ChannelTypes.GUILD_ANNOUNCEMENT) { + await this.webhookRepository.delete(webhook.id); + throw new AnnouncementChannelRequiredError(); + } + await this.cacheService.delete(this.followerStatsCacheKey(source.id)); + await this.webhookService.dispatchWebhooksUpdate({guildId: targetChannel.guildId, channelId: targetChannel.id}); + await this.webhookService.recordWebhookAuditLog({ + guildId: targetChannel.guildId, + userId, + action: 'create', + webhook, + auditLogReason, + }); + await this.sendFollowSystemMessage({ + webhook, + userId, + source: currentSource, + targetChannel, + requestCache, + }); + return {channelId: source.id, webhookId: webhook.id}; + } + + async getFollowerStats({ + userId, + channelId, + }: { + userId: UserID; + channelId: ChannelID; + }): Promise { + const channel = await this.channelService.channelData.operations.getChannel({userId, channelId}); + if (channel.type !== ChannelTypes.GUILD_ANNOUNCEMENT) { + throw new AnnouncementChannelRequiredError(); + } + const cacheKey = this.followerStatsCacheKey(channelId); + const cached = await this.cacheService.get(cacheKey); + if (cached) return cached; + const {channelCount, guildCount} = await this.webhookRepository.countBySourceChannel(channelId); + const stats: ChannelFollowerStatsResponse = {channel_count: channelCount, guild_count: guildCount}; + await this.cacheService.set(cacheKey, stats, CHANNEL_FOLLOWER_STATS_CACHE_SECONDS); + return stats; + } + + private followerStatsCacheKey(channelId: ChannelID): string { + return `channel-follower-stats:${channelId}`; + } + + private resolveFollowerWebhookName({ + sourceGuild, + source, + userId, + targetChannel, + }: { + sourceGuild: GuildResponse; + source: Channel; + userId: UserID; + targetChannel: GuildChannel; + }): string { + const channelName = source.name ?? ''; + const candidates = [`${sourceGuild.name} #${channelName}`, `#${channelName}`]; + let lastError: unknown = null; + for (const candidate of candidates) { + const truncated = truncateToLength(candidate, FOLLOWER_WEBHOOK_NAME_MAX_LENGTH); + try { + const name = WebhookNameType.parse(truncated); + contentModerationService.scanText(name, { + userId, + guildId: targetChannel.guildId, + channelId: targetChannel.id, + messageId: null, + surface: 'webhook', + }); + return name; + } catch (error) { + lastError = error; + } + } + throw lastError; + } + + private async sendFollowSystemMessage({ + webhook, + userId, + source, + targetChannel, + requestCache, + }: { + webhook: Webhook; + userId: UserID; + source: Channel; + targetChannel: GuildChannel; + requestCache: RequestCache; + }): Promise { + try { + const messageId = createMessageID(await this.snowflakeService.generateForChannel(targetChannel.id)); + const {message} = await this.channelService.messages.persistence.createMessage({ + messageId, + channelId: targetChannel.id, + userId, + type: MessageTypes.CHANNEL_FOLLOW_ADD, + content: webhook.name, + flags: 0, + messageReference: { + channel_id: source.id, + guild_id: source.guildId, + message_id: null, + type: 0, + }, + guildId: targetChannel.guildId, + channel: targetChannel, + allowEmbeds: false, + }); + await this.channelService.messages.dispatch.dispatchMessageCreate({ + channel: targetChannel, + message, + requestCache, + }); + } catch (error) { + Logger.error( + { + error, + webhookId: webhook.id.toString(), + channelId: targetChannel.id.toString(), + }, + 'Failed to send channel follow system message', + ); + } + } +} diff --git a/fluxer_api/src/api/webhook/IWebhookRepository.ts b/fluxer_api/src/api/webhook/IWebhookRepository.ts index c7b441bf9..73d889450 100644 --- a/fluxer_api/src/api/webhook/IWebhookRepository.ts +++ b/fluxer_api/src/api/webhook/IWebhookRepository.ts @@ -16,6 +16,8 @@ export abstract class IWebhookRepository { creatorId: bigint | null; name: string; avatarHash: string | null; + sourceGuildId?: bigint | null; + sourceChannelId?: bigint | null; }): Promise; abstract update( @@ -40,4 +42,13 @@ export abstract class IWebhookRepository { abstract countByGuild(guildId: bigint): Promise; abstract countByChannel(channelId: bigint): Promise; + + abstract findManyByIds(webhookIds: Array): Promise>; + + abstract listIdsBySourceChannel( + sourceChannelId: bigint, + options: {afterWebhookId?: bigint; limit: number}, + ): Promise>; + + abstract countBySourceChannel(sourceChannelId: bigint): Promise<{channelCount: number; guildCount: number}>; } diff --git a/fluxer_api/src/api/webhook/WebhookController.ts b/fluxer_api/src/api/webhook/WebhookController.ts index 0bd9e94fe..c6fb75fa8 100644 --- a/fluxer_api/src/api/webhook/WebhookController.ts +++ b/fluxer_api/src/api/webhook/WebhookController.ts @@ -47,6 +47,7 @@ import { } from '@fluxer/schema/src/domains/webhook/WebhookRequestSchemas'; import { SlackWebhookResponse, + WebhookCreateResponse, WebhookListResponse, WebhookResponse, WebhookTokenResponse, @@ -180,7 +181,7 @@ export function WebhookController(app: HonoApp) { summary: 'Create webhook', description: 'Creates a new webhook in the specified channel with the provided name and optional avatar. Returns the newly created webhook object including its ID and token.', - responseSchema: WebhookResponse, + responseSchema: WebhookCreateResponse, statusCode: 200, security: ['botToken', 'bearerToken', 'sessionToken'], tags: ['Webhooks'], diff --git a/fluxer_api/src/api/webhook/WebhookModel.ts b/fluxer_api/src/api/webhook/WebhookModel.ts index 184e91308..82ff42439 100644 --- a/fluxer_api/src/api/webhook/WebhookModel.ts +++ b/fluxer_api/src/api/webhook/WebhookModel.ts @@ -1,10 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import type {ChannelID, GuildID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; +import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; +import type {IGatewayService} from '@app/api/infrastructure/IGatewayService'; import type {UserCacheService} from '@app/api/infrastructure/UserCacheService'; +import {Logger} from '@app/api/Logger'; import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; import type {Webhook} from '@app/api/models/Webhook'; import {getCachedUserPartialResponse} from '@app/api/user/UserCacheHelpers'; +import {Permissions, WebhookTypes, type WebhookTypeValue} from '@fluxer/constants/src/ChannelConstants'; import {DELETED_USER_ID} from '@fluxer/constants/src/UserConstants'; import type {WebhookResponse, WebhookTokenResponse} from '@fluxer/schema/src/domains/webhook/WebhookSchemas'; @@ -15,27 +20,119 @@ export function mapWebhookToTokenResponse(webhook: Webhook): WebhookTokenRespons channel_id: webhook.channelId?.toString() || '', name: webhook.name || '', avatar: webhook.avatarHash, + type: webhook.type as WebhookTypeValue, token: webhook.token, }; } +type WebhookSourceFields = Pick; + +export type WebhookSourceResolver = (webhook: Webhook) => Promise; + +const sourceResolutionsByRequest = new WeakMap>>(); + +export function createWebhookSourceResolver({ + channelRepository, + gatewayService, + requestCache, +}: { + channelRepository: IChannelRepository; + gatewayService: IGatewayService; + requestCache: RequestCache; +}): WebhookSourceResolver { + let resolutions = sourceResolutionsByRequest.get(requestCache); + if (!resolutions) { + resolutions = new Map(); + sourceResolutionsByRequest.set(requestCache, resolutions); + } + const memo = resolutions; + return async (webhook) => { + const {sourceGuildId, sourceChannelId, creatorId} = webhook; + if (webhook.type !== WebhookTypes.CHANNEL_FOLLOWER || !sourceGuildId || !sourceChannelId || !creatorId) { + return null; + } + const key = `${sourceGuildId}:${sourceChannelId}:${creatorId}`; + let pending = memo.get(key); + if (!pending) { + pending = resolveWebhookSource({ + channelRepository, + gatewayService, + sourceGuildId, + sourceChannelId, + creatorId, + }); + memo.set(key, pending); + } + return pending; + }; +} + +async function resolveWebhookSource({ + channelRepository, + gatewayService, + sourceGuildId, + sourceChannelId, + creatorId, +}: { + channelRepository: IChannelRepository; + gatewayService: IGatewayService; + sourceGuildId: GuildID; + sourceChannelId: ChannelID; + creatorId: UserID; +}): Promise { + try { + const channel = await channelRepository.findUnique(sourceChannelId); + if (!channel || channel.guildId !== sourceGuildId) return null; + const canView = await gatewayService.checkPermission({ + guildId: sourceGuildId, + userId: creatorId, + permission: Permissions.VIEW_CHANNEL, + channelId: sourceChannelId, + }); + if (!canView) return null; + const guild = await gatewayService.getGuildData({ + guildId: sourceGuildId, + userId: creatorId, + skipMembershipCheck: true, + }); + return { + source_guild: {id: sourceGuildId.toString(), name: guild.name, icon: guild.icon ?? null}, + source_channel: {id: sourceChannelId.toString(), name: channel.name ?? ''}, + }; + } catch (error) { + Logger.warn( + {error, sourceGuildId: sourceGuildId.toString(), sourceChannelId: sourceChannelId.toString()}, + 'Failed to resolve follower webhook source', + ); + return null; + } +} + export async function mapWebhookToResponseWithCache({ webhook, userCacheService, requestCache, + resolveSource, }: { webhook: Webhook; userCacheService: UserCacheService; requestCache: RequestCache; + resolveSource?: WebhookSourceResolver; }): Promise { - const creatorPartial = await getCachedUserPartialResponse({ - userId: webhook.creatorId ?? createUserID(DELETED_USER_ID), - userCacheService, - requestCache, - }); + const [creatorPartial, source] = await Promise.all([ + getCachedUserPartialResponse({ + userId: webhook.creatorId ?? createUserID(DELETED_USER_ID), + userCacheService, + requestCache, + }), + resolveSource ? resolveSource(webhook) : Promise.resolve(null), + ]); + const {token, ...common} = mapWebhookToTokenResponse(webhook); return { - ...mapWebhookToTokenResponse(webhook), + ...common, + ...(webhook.type === WebhookTypes.CHANNEL_FOLLOWER ? {} : {token}), user: creatorPartial, + ...(source ?? {}), }; } @@ -43,12 +140,14 @@ export async function mapWebhooksToResponse({ webhooks, userCacheService, requestCache, + resolveSource, }: { webhooks: Array; userCacheService: UserCacheService; requestCache: RequestCache; + resolveSource?: WebhookSourceResolver; }): Promise> { return await Promise.all( - webhooks.map((webhook) => mapWebhookToResponseWithCache({webhook, userCacheService, requestCache})), + webhooks.map((webhook) => mapWebhookToResponseWithCache({webhook, userCacheService, requestCache, resolveSource})), ); } diff --git a/fluxer_api/src/api/webhook/WebhookRepository.test.ts b/fluxer_api/src/api/webhook/WebhookRepository.test.ts new file mode 100644 index 000000000..59048f43b --- /dev/null +++ b/fluxer_api/src/api/webhook/WebhookRepository.test.ts @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + type ChannelID, + createChannelID, + createGuildID, + createUserID, + createWebhookID, + createWebhookToken, + type GuildID, + type WebhookID, +} from '@app/api/BrandedTypes'; +import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution'; +import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import {WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import {afterEach, beforeEach, describe, expect, it} from 'vitest'; + +const SOURCE_GUILD = createGuildID(1n); +const SOURCE_CHANNEL = createChannelID(10n); + +let executor: InMemoryCassandraQueryExecutor; +let repository: WebhookRepository; + +async function createFollower( + webhookId: WebhookID, + guildId: GuildID, + channelId: ChannelID, + sourceChannelId: ChannelID = SOURCE_CHANNEL, +) { + return repository.create({ + webhookId, + token: createWebhookToken(`token-${webhookId}`), + type: WebhookTypes.CHANNEL_FOLLOWER, + guildId, + channelId, + creatorId: createUserID(7n), + name: 'Source #news', + avatarHash: null, + sourceGuildId: SOURCE_GUILD, + sourceChannelId, + }); +} + +describe('WebhookRepository source channel index', () => { + beforeEach(() => { + executor = new InMemoryCassandraQueryExecutor(); + setCassandraQueryExecutorForTesting(executor); + repository = new WebhookRepository(); + }); + afterEach(() => { + executor.reset(); + setCassandraQueryExecutorForTesting(null); + }); + + it('stores and returns the source columns of a follower webhook', async () => { + const created = await createFollower(createWebhookID(100n), createGuildID(2n), createChannelID(20n)); + expect(created.sourceGuildId).toBe(1n); + expect(created.sourceChannelId).toBe(10n); + expect(created.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + const found = await repository.findUnique(createWebhookID(100n)); + expect(found?.sourceGuildId).toBe(1n); + expect(found?.sourceChannelId).toBe(10n); + expect(found?.toRow().source_channel_id).toBe(10n); + }); + + it('leaves incoming webhooks out of the index with null source columns', async () => { + const created = await repository.create({ + webhookId: createWebhookID(101n), + token: createWebhookToken('incoming'), + type: WebhookTypes.INCOMING, + guildId: createGuildID(2n), + channelId: SOURCE_CHANNEL, + creatorId: createUserID(7n), + name: 'Incoming', + avatarHash: null, + }); + expect(created.sourceGuildId).toBeNull(); + expect(created.sourceChannelId).toBeNull(); + expect(await repository.listIdsBySourceChannel(SOURCE_CHANNEL, {limit: 10})).toEqual([]); + expect(await repository.countBySourceChannel(SOURCE_CHANNEL)).toEqual({channelCount: 0, guildCount: 0}); + }); + + it('keeps the source columns across an update and a move', async () => { + await createFollower(createWebhookID(100n), createGuildID(2n), createChannelID(20n)); + const updated = await repository.update(createWebhookID(100n), { + name: 'Renamed', + channelId: createChannelID(21n), + }); + expect(updated?.name).toBe('Renamed'); + expect(updated?.sourceGuildId).toBe(1n); + expect(updated?.sourceChannelId).toBe(10n); + const found = await repository.findUnique(createWebhookID(100n)); + expect(found?.channelId).toBe(21n); + expect(found?.sourceChannelId).toBe(10n); + expect(await repository.listIdsBySourceChannel(SOURCE_CHANNEL, {limit: 10})).toEqual([ + {webhookId: 100n, guildId: 2n}, + ]); + }); + + it('pages follower ids by source channel in webhook id order', async () => { + await createFollower(createWebhookID(104n), createGuildID(3n), createChannelID(30n)); + await createFollower(createWebhookID(101n), createGuildID(2n), createChannelID(20n)); + await createFollower(createWebhookID(103n), createGuildID(2n), createChannelID(21n)); + await createFollower(createWebhookID(102n), createGuildID(4n), createChannelID(40n)); + await createFollower(createWebhookID(105n), createGuildID(2n), createChannelID(22n), createChannelID(11n)); + const first = await repository.listIdsBySourceChannel(SOURCE_CHANNEL, {limit: 3}); + expect(first).toEqual([ + {webhookId: 101n, guildId: 2n}, + {webhookId: 102n, guildId: 4n}, + {webhookId: 103n, guildId: 2n}, + ]); + const second = await repository.listIdsBySourceChannel(SOURCE_CHANNEL, { + afterWebhookId: first[first.length - 1]!.webhookId, + limit: 3, + }); + expect(second).toEqual([{webhookId: 104n, guildId: 3n}]); + }); + + it('counts follower channels and distinct target guilds', async () => { + await createFollower(createWebhookID(101n), createGuildID(2n), createChannelID(20n)); + await createFollower(createWebhookID(102n), createGuildID(2n), createChannelID(21n)); + await createFollower(createWebhookID(103n), createGuildID(3n), createChannelID(30n)); + expect(await repository.countBySourceChannel(SOURCE_CHANNEL)).toEqual({channelCount: 3, guildCount: 2}); + }); + + it('removes the index row when a follower webhook is deleted', async () => { + await createFollower(createWebhookID(101n), createGuildID(2n), createChannelID(20n)); + await createFollower(createWebhookID(102n), createGuildID(3n), createChannelID(30n)); + await repository.delete(createWebhookID(101n)); + expect(await repository.findUnique(createWebhookID(101n))).toBeNull(); + expect(await repository.listIdsBySourceChannel(SOURCE_CHANNEL, {limit: 10})).toEqual([ + {webhookId: 102n, guildId: 3n}, + ]); + expect(await repository.countBySourceChannel(SOURCE_CHANNEL)).toEqual({channelCount: 1, guildCount: 1}); + }); + + it('finds many webhooks by id', async () => { + await createFollower(createWebhookID(101n), createGuildID(2n), createChannelID(20n)); + await createFollower(createWebhookID(102n), createGuildID(3n), createChannelID(30n)); + const found = await repository.findManyByIds([createWebhookID(101n), createWebhookID(102n), createWebhookID(999n)]); + expect(found.map((webhook) => webhook.id).sort()).toEqual([101n, 102n]); + expect(await repository.findManyByIds([])).toEqual([]); + }); +}); diff --git a/fluxer_api/src/api/webhook/WebhookRepository.ts b/fluxer_api/src/api/webhook/WebhookRepository.ts index 524d0f263..08b35ef5b 100644 --- a/fluxer_api/src/api/webhook/WebhookRepository.ts +++ b/fluxer_api/src/api/webhook/WebhookRepository.ts @@ -9,10 +9,10 @@ import { fetchOne, } from '@app/api/database/CassandraQueryExecution'; import {buildPatchFromData, executeVersionedUpdate} from '@app/api/database/CassandraVersionedUpdate'; -import type {WebhookRow} from '@app/api/database/types/ChannelTypes'; +import type {WebhookRow, WebhooksBySourceChannelRow} from '@app/api/database/types/ChannelTypes'; import {WEBHOOK_COLUMNS} from '@app/api/database/types/ChannelTypes'; import {Webhook} from '@app/api/models/Webhook'; -import {Webhooks, WebhooksByChannel, WebhooksByGuild} from '@app/api/Tables'; +import {Webhooks, WebhooksByChannel, WebhooksByGuild, WebhooksBySourceChannel} from '@app/api/Tables'; import {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; const FETCH_WEBHOOK_BY_ID_CQL = Webhooks.selectCql({ @@ -34,6 +34,28 @@ const FETCH_WEBHOOK_IDS_BY_CHANNEL_CQL = WebhooksByChannel.selectCql({ const FETCH_WEBHOOKS_BY_IDS_CQL = Webhooks.selectCql({ where: Webhooks.where.in('webhook_id', 'webhook_ids'), }); +const FETCH_WEBHOOK_GUILDS_BY_SOURCE_CHANNEL_CQL = WebhooksBySourceChannel.selectCql({ + columns: ['webhook_id', 'guild_id'], + where: WebhooksBySourceChannel.where.eq('source_channel_id'), +}); + +function createSourceChannelFirstPageQuery(limit: number) { + return WebhooksBySourceChannel.select({ + columns: ['webhook_id', 'guild_id'], + where: WebhooksBySourceChannel.where.eq('source_channel_id'), + orderBy: {col: 'webhook_id', direction: 'ASC'}, + limit, + }); +} + +function createSourceChannelPageQuery(limit: number) { + return WebhooksBySourceChannel.select({ + columns: ['webhook_id', 'guild_id'], + where: [WebhooksBySourceChannel.where.eq('source_channel_id'), WebhooksBySourceChannel.where.gt('webhook_id')], + orderBy: {col: 'webhook_id', direction: 'ASC'}, + limit, + }); +} export class WebhookRepository extends IWebhookRepository { async findUnique(webhookId: WebhookID): Promise { @@ -58,6 +80,8 @@ export class WebhookRepository extends IWebhookRepository { creatorId: UserID | null; name: string; avatarHash: string | null; + sourceGuildId?: GuildID | null; + sourceChannelId?: ChannelID | null; }): Promise { const webhookData: WebhookRow = { webhook_id: data.webhookId, @@ -68,6 +92,8 @@ export class WebhookRepository extends IWebhookRepository { creator_id: data.creatorId, name: data.name, avatar_hash: data.avatarHash, + source_guild_id: data.sourceGuildId ?? null, + source_channel_id: data.sourceChannelId ?? null, version: 1, }; const result = await executeVersionedUpdate( @@ -97,6 +123,15 @@ export class WebhookRepository extends IWebhookRepository { }), ); } + if (data.sourceChannelId && data.guildId) { + batch.addPrepared( + WebhooksBySourceChannel.upsertAll({ + source_channel_id: data.sourceChannelId, + webhook_id: data.webhookId, + guild_id: data.guildId, + }), + ); + } await batch.execute(); return new Webhook({...webhookData, version: result.finalVersion ?? 1}); } @@ -125,6 +160,8 @@ export class WebhookRepository extends IWebhookRepository { creator_id: data.creatorId !== undefined ? data.creatorId : existing.creatorId, name: data.name ?? existing.name, avatar_hash: data.avatarHash !== undefined ? data.avatarHash : existing.avatarHash, + source_guild_id: existing.sourceGuildId, + source_channel_id: existing.sourceChannelId, version: existing.version, }; const result = await executeVersionedUpdate( @@ -203,9 +240,52 @@ export class WebhookRepository extends IWebhookRepository { }), ); } + if (webhook.sourceChannelId) { + batch.addPrepared( + WebhooksBySourceChannel.deleteByPk({ + source_channel_id: webhook.sourceChannelId, + webhook_id: webhookId, + }), + ); + } await batch.execute(); } + async findManyByIds(webhookIds: Array): Promise> { + return this.fetchWebhooksByIds(webhookIds); + } + + async listIdsBySourceChannel( + sourceChannelId: ChannelID, + options: {afterWebhookId?: WebhookID; limit: number}, + ): Promise> { + const rows = + options.afterWebhookId !== undefined + ? await fetchMany>( + createSourceChannelPageQuery(options.limit).bind({ + source_channel_id: sourceChannelId, + webhook_id: options.afterWebhookId, + }), + ) + : await fetchMany>( + createSourceChannelFirstPageQuery(options.limit).bind({ + source_channel_id: sourceChannelId, + }), + ); + return rows.map((row) => ({webhookId: row.webhook_id, guildId: row.guild_id})); + } + + async countBySourceChannel(sourceChannelId: ChannelID): Promise<{channelCount: number; guildCount: number}> { + const rows = await fetchMany>( + FETCH_WEBHOOK_GUILDS_BY_SOURCE_CHANNEL_CQL, + {source_channel_id: sourceChannelId}, + ); + return { + channelCount: rows.length, + guildCount: new Set(rows.map((row) => row.guild_id)).size, + }; + } + private async fetchWebhooksByIds(webhookIds: Array): Promise> { if (webhookIds.length === 0) { return []; diff --git a/fluxer_api/src/api/webhook/WebhookRequestService.ts b/fluxer_api/src/api/webhook/WebhookRequestService.ts index 9460b6c1e..d23b265e2 100644 --- a/fluxer_api/src/api/webhook/WebhookRequestService.ts +++ b/fluxer_api/src/api/webhook/WebhookRequestService.ts @@ -6,6 +6,7 @@ import {Config} from '@app/api/Config'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; import type {MessageUpdateRequest} from '@app/api/channel/MessageTypes'; import {createMessageResponseDataService} from '@app/api/channel/services/message/MessageResponseDataService'; +import type {IGatewayService} from '@app/api/infrastructure/IGatewayService'; import type {LiveKitWebhookService} from '@app/api/infrastructure/LiveKitWebhookService'; import type {UserCacheService} from '@app/api/infrastructure/UserCacheService'; import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware'; @@ -13,9 +14,11 @@ import type {Message} from '@app/api/models/Message'; import type {SweegoWebhookService} from '@app/api/webhook/SweegoWebhookService'; import {transformSlackWebhookRequest} from '@app/api/webhook/transformers/SlackTransformer'; import { + createWebhookSourceResolver, mapWebhooksToResponse, mapWebhookToResponseWithCache, mapWebhookToTokenResponse, + type WebhookSourceResolver, } from '@app/api/webhook/WebhookModel'; import type {WebhookExecuteMessageData, WebhookService} from '@app/api/webhook/WebhookService'; import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; @@ -167,8 +170,17 @@ export class WebhookRequestService { private readonly userCacheService: UserCacheService, private readonly liveKitWebhookService: LiveKitWebhookService | null, private readonly sweegoWebhookService: SweegoWebhookService, + private readonly gatewayService: IGatewayService, ) {} + private sourceResolver(requestCache: RequestCache): WebhookSourceResolver { + return createWebhookSourceResolver({ + channelRepository: this.channelRepository, + gatewayService: this.gatewayService, + requestCache, + }); + } + async listGuildWebhooks(params: WebhookListGuildParams): Promise> { const webhooks = await this.webhookService.getGuildWebhooks({ userId: params.userId, @@ -178,6 +190,7 @@ export class WebhookRequestService { webhooks, userCacheService: this.userCacheService, requestCache: params.requestCache, + resolveSource: this.sourceResolver(params.requestCache), }); } @@ -190,6 +203,7 @@ export class WebhookRequestService { webhooks, userCacheService: this.userCacheService, requestCache: params.requestCache, + resolveSource: this.sourceResolver(params.requestCache), }); } @@ -221,6 +235,7 @@ export class WebhookRequestService { webhook, userCacheService: this.userCacheService, requestCache: params.requestCache, + resolveSource: this.sourceResolver(params.requestCache), }); } @@ -247,6 +262,7 @@ export class WebhookRequestService { webhook, userCacheService: this.userCacheService, requestCache: params.requestCache, + resolveSource: this.sourceResolver(params.requestCache), }); } diff --git a/fluxer_api/src/api/webhook/WebhookService.ts b/fluxer_api/src/api/webhook/WebhookService.ts index f748a95ec..2df4f3a4b 100644 --- a/fluxer_api/src/api/webhook/WebhookService.ts +++ b/fluxer_api/src/api/webhook/WebhookService.ts @@ -6,7 +6,14 @@ import type {ChannelID, GuildID, MessageID, UserID, WebhookID, WebhookToken} fro import {createChannelID, createGuildID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; import type {MessageRequest, MessageUpdateRequest} from '@app/api/channel/MessageTypes'; +import {withChannelFollowLock} from '@app/api/channel/services/ChannelFollowers'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; +import {assertCrosspostContentRules} from '@app/api/channel/utils/CrosspostContentRules'; +import { + type ContentWarningChannelLike, + channelToContentWarningView, + guildResponseToContentWarningView, +} from '@app/api/channel/utils/EffectiveContentWarning'; import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService'; import type {GuildService} from '@app/api/guild/services/GuildService'; import type {AvatarService} from '@app/api/infrastructure/AvatarService'; @@ -28,15 +35,26 @@ import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository'; import {transform as GitHubTransform} from '@app/api/webhook/transformers/GitHubTransformer'; import {instatusDeliveryKey, transformInstatusWebhook} from '@app/api/webhook/transformers/InstatusTransformer'; import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType'; -import {GUILD_TEXT_BASED_CHANNEL_TYPES, Permissions} from '@fluxer/constants/src/ChannelConstants'; +import { + CHANNEL_FOLLOW_TARGET_TYPES, + ChannelTypes, + GUILD_TEXT_BASED_CHANNEL_TYPES, + Permissions, + WebhookTypes, +} from '@fluxer/constants/src/ChannelConstants'; import type {LimitKey} from '@fluxer/constants/src/LimitConfigMetadata'; import {MAX_WEBHOOKS_PER_CHANNEL, MAX_WEBHOOKS_PER_GUILD} from '@fluxer/constants/src/LimitConstants'; +import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import {ChannelAlreadyFollowedError} from '@fluxer/errors/src/domains/channel/ChannelAlreadyFollowedError'; +import {InvalidFollowTargetChannelError} from '@fluxer/errors/src/domains/channel/InvalidFollowTargetChannelError'; import {MaxWebhooksPerChannelError} from '@fluxer/errors/src/domains/channel/MaxWebhooksPerChannelError'; import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError'; import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError'; +import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError'; import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError'; import {MaxWebhooksPerGuildError} from '@fluxer/errors/src/domains/guild/MaxWebhooksPerGuildError'; import {UnknownWebhookError} from '@fluxer/errors/src/domains/webhook/UnknownWebhookError'; +import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; import type {AllowedMentionsRequest} from '@fluxer/schema/src/domains/message/SharedMessageSchemas'; import type {GitHubWebhook} from '@fluxer/schema/src/domains/webhook/GitHubWebhookSchemas'; import type {InstatusWebhook} from '@fluxer/schema/src/domains/webhook/InstatusWebhookSchemas'; @@ -184,20 +202,7 @@ export class WebhookService { }); await checkPermission(Permissions.MANAGE_WEBHOOKS); await this.assertChannelWebhookPermission({userId, guildId: channel.guildId, channelId}); - const guildLimit = this.resolveWebhookLimit(guildData.features, 'max_webhooks_per_guild', MAX_WEBHOOKS_PER_GUILD); - const guildWebhookCount = await this.repository.countByGuild(channel.guildId); - if (guildWebhookCount >= guildLimit) { - throw new MaxWebhooksPerGuildError(guildLimit); - } - const channelLimit = this.resolveWebhookLimit( - guildData.features, - 'max_webhooks_per_channel', - MAX_WEBHOOKS_PER_CHANNEL, - ); - const channelWebhookCount = await this.repository.countByChannel(channelId); - if (channelWebhookCount >= channelLimit) { - throw new MaxWebhooksPerChannelError(channelLimit); - } + await this.assertWebhookCapacity({guildId: channel.guildId, channelId, guildFeatures: guildData.features}); contentModerationService.scanText(data.name, { userId, guildId: channel.guildId, @@ -209,7 +214,7 @@ export class WebhookService { const webhook = await this.repository.create({ webhookId, token: createWebhookToken(RandomUtils.randomString(64)), - type: 1, + type: WebhookTypes.INCOMING, guildId: channel.guildId, channelId, creatorId: userId, @@ -242,11 +247,19 @@ export class WebhookService { guildId: webhook.guildId ? webhook.guildId : createGuildID(0n), }); await checkPermission(Permissions.MANAGE_WEBHOOKS); + const isFollower = webhook.type === WebhookTypes.CHANNEL_FOLLOWER; + if (isFollower && data.avatar !== undefined) { + throw InputValidationError.fromCode('avatar', ValidationErrorCodes.INVALID_FORMAT); + } + let followerMoveChannelId: ChannelID | null = null; if (data.channel_id && data.channel_id !== webhook.channelId) { const targetChannel = await this.channelService.channelData.operations.getChannel({ userId, channelId: createChannelID(data.channel_id), }); + if (isFollower && !CHANNEL_FOLLOW_TARGET_TYPES.has(targetChannel.type)) { + throw new InvalidFollowTargetChannelError(); + } this.assertWebhookTargetChannel(targetChannel); if (targetChannel.guildId !== webhook.guildId) { throw new UnknownChannelError(); @@ -269,18 +282,44 @@ export class WebhookService { if (channelWebhookCount >= channelLimit) { throw new MaxWebhooksPerChannelError(channelLimit); } + if (isFollower) { + await this.assertFollowerMoveContentRules({webhook, targetChannel, targetGuild: guildData, userId}); + followerMoveChannelId = targetChannel.id; + } } const updatedData = await this.updateWebhookData({webhook, data}); - const updatedWebhook = await this.repository.update(webhookId, { - name: updatedData.name, - avatarHash: updatedData.avatarHash, - channelId: updatedData.channelId, - }); + const writeUpdate = () => + this.repository.update(webhookId, { + name: updatedData.name, + avatarHash: updatedData.avatarHash, + channelId: updatedData.channelId, + }); + const moveChannelId = followerMoveChannelId; + const updatedWebhook = moveChannelId + ? await withChannelFollowLock(this.cacheService, moveChannelId, async () => { + const current = await this.channelRepository.findUnique(moveChannelId); + if (!current || !CHANNEL_FOLLOW_TARGET_TYPES.has(current.type)) { + throw new InvalidFollowTargetChannelError(); + } + await this.assertChannelNotFollowing({ + channelId: moveChannelId, + sourceChannelId: webhook.sourceChannelId, + excludeWebhookId: webhook.id, + }); + return writeUpdate(); + }) + : await writeUpdate(); if (!updatedWebhook) throw new UnknownWebhookError(); await this.dispatchWebhooksUpdate({ guildId: webhook.guildId, channelId: webhook.channelId, }); + if (updatedWebhook.channelId && updatedWebhook.channelId !== webhook.channelId) { + await this.dispatchWebhooksUpdate({ + guildId: updatedWebhook.guildId, + channelId: updatedWebhook.channelId, + }); + } if (webhook.guildId) { const previousSnapshot = this.serializeWebhookForAudit(webhook); await this.recordWebhookAuditLog({ @@ -479,6 +518,99 @@ export class WebhookService { return webhook; } + async assertWebhookCapacity({ + guildId, + channelId, + guildFeatures, + }: { + guildId: GuildID; + channelId: ChannelID; + guildFeatures: Iterable | null; + }): Promise { + const guildLimit = this.resolveWebhookLimit(guildFeatures, 'max_webhooks_per_guild', MAX_WEBHOOKS_PER_GUILD); + const guildWebhookCount = await this.repository.countByGuild(guildId); + if (guildWebhookCount >= guildLimit) { + throw new MaxWebhooksPerGuildError(guildLimit); + } + const channelLimit = this.resolveWebhookLimit(guildFeatures, 'max_webhooks_per_channel', MAX_WEBHOOKS_PER_CHANNEL); + const channelWebhookCount = await this.repository.countByChannel(channelId); + if (channelWebhookCount >= channelLimit) { + throw new MaxWebhooksPerChannelError(channelLimit); + } + } + + async assertChannelNotFollowing({ + channelId, + sourceChannelId, + excludeWebhookId, + }: { + channelId: ChannelID; + sourceChannelId: ChannelID | null; + excludeWebhookId?: WebhookID; + }): Promise { + if (!sourceChannelId) return; + const webhooks = await this.repository.listByChannel(channelId); + const duplicate = webhooks.some( + (candidate) => + candidate.type === WebhookTypes.CHANNEL_FOLLOWER && + candidate.sourceChannelId === sourceChannelId && + candidate.id !== excludeWebhookId, + ); + if (duplicate) throw new ChannelAlreadyFollowedError(); + } + + async assertFollowContentRules({ + sourceChannel, + sourceGuild, + targetChannel, + targetGuild, + }: { + sourceChannel: Channel; + sourceGuild: GuildResponse; + targetChannel: Channel; + targetGuild: GuildResponse; + }): Promise { + const [sourceParent, targetParent] = await Promise.all([ + this.resolveParentContentView(sourceChannel), + this.resolveParentContentView(targetChannel), + ]); + assertCrosspostContentRules({ + source: channelToContentWarningView(sourceChannel), + sourceParent, + sourceGuild: guildResponseToContentWarningView(sourceGuild), + target: channelToContentWarningView(targetChannel), + targetParent, + targetGuild: guildResponseToContentWarningView(targetGuild), + }); + } + + private async resolveParentContentView(channel: Channel): Promise { + if (!channel.parentId || channel.type === ChannelTypes.GUILD_CATEGORY) return null; + const parent = await this.channelRepository.findUnique(channel.parentId); + return parent ? channelToContentWarningView(parent) : null; + } + + private async assertFollowerMoveContentRules({ + webhook, + targetChannel, + targetGuild, + userId, + }: { + webhook: Webhook; + targetChannel: Channel; + targetGuild: GuildResponse; + userId: UserID; + }): Promise { + if (!webhook.sourceChannelId || !webhook.sourceGuildId) return; + const sourceChannel = await this.channelRepository.findUnique(webhook.sourceChannelId); + if (!sourceChannel) return; + const sourceGuild = await this.gatewayService + .getGuildData({guildId: webhook.sourceGuildId, userId, skipMembershipCheck: true}) + .catch(() => null); + if (!sourceGuild) return; + await this.assertFollowContentRules({sourceChannel, sourceGuild, targetChannel, targetGuild}); + } + private async canManageChannelWebhooks({ userId, guildId, @@ -495,7 +627,7 @@ export class WebhookService { return canView && canManage; } - private async assertChannelWebhookPermission(params: { + async assertChannelWebhookPermission(params: { userId: UserID; guildId: GuildID; channelId: ChannelID; @@ -506,7 +638,7 @@ export class WebhookService { private async getTokenAuthenticatedWebhook({webhookId, token}: WebhookTokenParams): Promise { const webhook = await this.repository.findByToken(webhookId, token); - if (!webhook) throw new UnknownWebhookError(); + if (!webhook || webhook.type !== WebhookTypes.INCOMING) throw new UnknownWebhookError(); return webhook; } @@ -644,7 +776,7 @@ export class WebhookService { if (!webhook.channelId) { return undefined; } - return {channel_id: webhook.channelId.toString()}; + return {channel_id: webhook.channelId.toString(), type: webhook.type.toString()}; } private serializeWebhookForAudit(webhook: Webhook): Record { @@ -659,7 +791,7 @@ export class WebhookService { }; } - private async recordWebhookAuditLog(params: { + async recordWebhookAuditLog(params: { guildId: GuildID; userId: UserID; action: 'create' | 'update' | 'delete'; @@ -707,6 +839,6 @@ export class WebhookService { private resolveWebhookLimit(guildFeatures: Iterable | null, key: LimitKey, fallback: number): number { const ctx = createLimitMatchContext({guildFeatures}); - return resolveLimitSafe(this.limitConfigService.getConfigSnapshot(), ctx, key, fallback); + return resolveLimitSafe(this.limitConfigService.getConfigSnapshot(), ctx, key, fallback, 'guild'); } } diff --git a/fluxer_api/src/api/webhook/tests/ChannelFollow.test.ts b/fluxer_api/src/api/webhook/tests/ChannelFollow.test.ts new file mode 100644 index 000000000..92cb2ad80 --- /dev/null +++ b/fluxer_api/src/api/webhook/tests/ChannelFollow.test.ts @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createGuildID, createUserID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import { + type AnnouncementWorld, + addGuildMember, + announcementWorld, + createAnnouncementSourceGuild, + createGuildChannel, + follow, + followRequest, + setGuildFeatures, + setLimitOverride, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {createGuild, createPermissionOverwrite, updateGuild} from '@app/api/channel/tests/ChannelTestUtils'; +import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils'; +import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils'; +import {phraseBlocklistCache} from '@app/api/middleware/PhraseBlocklistCache'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, getChannelWebhooks} from '@app/api/webhook/tests/WebhookTestUtils'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType'; +import {ChannelTypes, MessageTypes, Permissions, WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import {GuildFeatures} from '@fluxer/constants/src/GuildConstants'; +import {MAX_WEBHOOKS_PER_CHANNEL} from '@fluxer/constants/src/LimitConstants'; +import type {ChannelFollowerStatsResponse} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import type {GuildAuditLogListResponse} from '@fluxer/schema/src/domains/guild/GuildAuditLogSchemas'; +import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; +import {WebhookListResponse} from '@fluxer/schema/src/domains/webhook/WebhookSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +async function listMessages(harness: ApiTestHarness, token: string, channelId: string) { + return createBuilder>(harness, token).get(`/channels/${channelId}/messages`).execute(); +} + +async function listAuditLogs(harness: ApiTestHarness, token: string, guildId: string, actionType: number) { + return createBuilder(harness, token) + .get(`/guilds/${guildId}/audit-logs?action_type=${actionType}`) + .execute(); +} + +async function followerStats(harness: ApiTestHarness, token: string, channelId: string) { + return createBuilder(harness, token) + .get(`/channels/${channelId}/follower-stats`) + .execute(); +} + +describe('Channel follow', () => { + let harness: ApiTestHarness; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await announcementWorld(harness); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('creates a follower webhook with the source name and a copy of the source icon', async () => { + const {a, b} = world; + const withIcon = await updateGuild(harness, a.owner.token, a.guild.id, {icon: getPngDataUrl()}); + expect(withIcon.icon).toBeTruthy(); + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + expect(followed.channel_id).toBe(a.ann.id); + const webhooks = WebhookListResponse.parse(await getChannelWebhooks(harness, b.t1.id, b.owner.token)); + const webhook = webhooks.find((candidate) => candidate.id === followed.webhook_id); + expect(webhook).toBeDefined(); + expect(webhook?.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + expect(webhook).not.toHaveProperty('token'); + expect(webhook?.name).toBe(`${withIcon.name} #news`); + expect(webhook?.avatar).toBe(withIcon.icon); + expect(webhook?.source_guild).toEqual({id: a.guild.id, name: withIcon.name, icon: withIcon.icon}); + expect(webhook?.source_channel).toEqual({id: a.ann.id, name: 'news'}); + const iconKey = withIcon.icon!.replace(/^a_/, ''); + expect(harness.storageService.getCopiedObjects()).toContainEqual({ + sourceBucket: Config.s3.buckets.cdn, + sourceKey: `icons/${a.guild.id}/${iconKey}`, + destinationBucket: Config.s3.buckets.cdn, + destinationKey: `avatars/${followed.webhook_id}/${iconKey}`, + }); + const stored = await new WebhookRepository().findUnique(createWebhookID(BigInt(followed.webhook_id))); + expect(stored?.sourceGuildId?.toString()).toBe(a.guild.id); + expect(stored?.sourceChannelId?.toString()).toBe(a.ann.id); + expect(stored?.creatorId?.toString()).toBe(b.owner.userId); + }); + + test('uses a null avatar when the source guild has no icon', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + expect(webhooks.find((candidate) => candidate.id === followed.webhook_id)?.avatar ?? null).toBeNull(); + expect( + harness.storageService + .getCopiedObjects() + .filter((copy) => copy.destinationKey.startsWith(`avatars/${followed.webhook_id}/`)), + ).toHaveLength(0); + }); + + test('truncates long follower webhook names to 80 code points', async () => { + const {b} = world; + const longName = `${'\u{1F4E3}'.repeat(10)}${'x'.repeat(80)}`; + const source = await createAnnouncementSourceGuild(harness, longName.slice(0, 90)); + await addGuildMember(harness, source.owner, source.guild, b.owner); + const followed = await follow(harness, b.owner.token, source.ann.id, b.t1.id); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + const name = webhooks.find((candidate) => candidate.id === followed.webhook_id)!.name; + expect(name.length).toBeLessThanOrEqual(80); + expect(name.length).toBeGreaterThanOrEqual(79); + expect(name.startsWith('\u{1F4E3}'.repeat(10))).toBe(true); + expect(Array.from(name).every((codePoint) => codePoint.length === 2 || codePoint.charCodeAt(0) < 0xd800)).toBe( + true, + ); + }); + + test('falls back to the channel name when the combined name fails moderation', async () => { + const {b} = world; + const source = await createAnnouncementSourceGuild(harness, 'Forbidden Words Guild'); + await addGuildMember(harness, source.owner, source.guild, b.owner); + vi.spyOn(phraseBlocklistCache, 'containsBannedPhrase').mockImplementation((text) => + text.includes('Forbidden Words'), + ); + const followed = await follow(harness, b.owner.token, source.ann.id, b.t1.id); + vi.restoreAllMocks(); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + expect(webhooks.find((candidate) => candidate.id === followed.webhook_id)?.name).toBe('#news'); + }); + + test('rejects the follow when the fallback name also fails moderation', async () => { + const {a, b} = world; + vi.spyOn(phraseBlocklistCache, 'containsBannedPhrase').mockImplementation((text) => text.includes('news')); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.CONTENT_BLOCKED) + .execute(); + vi.restoreAllMocks(); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(0); + }); + + test('posts a CHANNEL_FOLLOW_ADD system message whose reference has no message id', async () => { + const {a, b} = world; + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const webhookUpdates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'WEBHOOKS_UPDATE'); + expect(webhookUpdates).toEqual([ + expect.objectContaining({guildId: createGuildID(BigInt(b.guild.id)), data: {channel_id: b.t1.id}}), + ]); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + const webhookName = webhooks.find((candidate) => candidate.id === followed.webhook_id)!.name; + const messages = await listMessages(harness, b.owner.token, b.t1.id); + const systemMessage = messages.find((message) => message.type === MessageTypes.CHANNEL_FOLLOW_ADD); + expect(systemMessage).toBeDefined(); + expect(systemMessage?.author.id).toBe(b.owner.userId); + expect(systemMessage?.content).toBe(webhookName); + expect(systemMessage?.mention_everyone).toBe(false); + expect(systemMessage?.message_reference).toEqual({type: 0, guild_id: a.guild.id, channel_id: a.ann.id}); + expect(systemMessage?.message_reference).not.toHaveProperty('message_id'); + expect(systemMessage).not.toHaveProperty('referenced_message'); + await ensureSessionStarted(harness, b.owner.token); + await createBuilder(harness, b.owner.token) + .patch(`/channels/${b.t1.id}/messages/${systemMessage!.id}`) + .body({content: 'edited'}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CANNOT_MODIFY_SYSTEM_WEBHOOK) + .execute(); + await createBuilder(harness, b.owner.token) + .delete(`/channels/${b.t1.id}/messages/${systemMessage!.id}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + const remaining = await listMessages(harness, b.owner.token, b.t1.id); + expect(remaining.some((message) => message.id === systemMessage!.id)).toBe(false); + }); + + test('records WEBHOOK_CREATE in the target guild only, with the reason and follower type', async () => { + const {a, b} = world; + const followed = await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .header('X-Audit-Log-Reason', 'release notes') + .execute(); + const targetLog = await listAuditLogs(harness, b.owner.token, b.guild.id, AuditLogActionType.WEBHOOK_CREATE); + const entry = targetLog.audit_log_entries.find((candidate) => candidate.target_id === followed.webhook_id); + expect(entry?.reason).toBe('release notes'); + expect(entry?.user_id).toBe(b.owner.userId); + expect(entry?.options).toEqual({channel_id: b.t1.id, type: WebhookTypes.CHANNEL_FOLLOWER}); + const sourceLog = await listAuditLogs(harness, a.owner.token, a.guild.id, AuditLogActionType.WEBHOOK_CREATE); + expect(sourceLog.audit_log_entries).toHaveLength(0); + }); + + test('allows following into a text channel of the same guild', async () => { + const {a} = world; + const local = await createGuildChannel(harness, a.owner.token, a.guild.id, { + name: 'local', + type: ChannelTypes.GUILD_TEXT, + }); + const followed = await follow(harness, a.owner.token, a.ann.id, local.id); + expect(followed.channel_id).toBe(a.ann.id); + }); + + test('rejects a duplicate follow and allows another target', async () => { + const {a, b} = world; + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CHANNEL_ALREADY_FOLLOWED) + .execute(); + await follow(harness, b.owner.token, a.ann.id, b.t2.id); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(1); + }); + + test('allows two sources to post into the same target', async () => { + const {a, b} = world; + const second = await createGuildChannel(harness, a.owner.token, a.guild.id, { + name: 'updates', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + }); + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await follow(harness, b.owner.token, second.id, b.t1.id); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + expect(webhooks.map((webhook) => webhook.source_channel?.id).sort()).toEqual([a.ann.id, second.id].sort()); + }); + + test('counts follower webhooks toward the channel webhook cap', async () => { + const {a, b} = world; + for (let index = 0; index < MAX_WEBHOOKS_PER_CHANNEL; index++) { + await createWebhook(harness, b.t1.id, b.owner.token, `Hook ${index}`); + } + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.MAX_WEBHOOKS_PER_CHANNEL) + .execute(); + }); + + test('counts follower webhooks toward the guild webhook cap', async () => { + const {a, b} = world; + const restore = await setLimitOverride(harness, {max_webhooks_per_guild: 1}); + try { + await createWebhook(harness, b.t2.id, b.owner.token, 'Existing'); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.MAX_WEBHOOKS_PER_GUILD) + .execute(); + } finally { + await restore(); + } + }); + + test('an announcement channel takes follows however many it already has', async () => { + const {a, b} = world; + vi.spyOn(WebhookRepository.prototype, 'countBySourceChannel').mockResolvedValue({ + channelCount: 100_000, + guildCount: 50_000, + }); + try { + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await follow(harness, b.owner.token, a.ann.id, b.t2.id); + } finally { + vi.restoreAllMocks(); + } + expect(await new WebhookRepository().countBySourceChannel(createChannelID(BigInt(a.ann.id)))).toEqual({ + channelCount: 2, + guildCount: 1, + }); + }); + + test('one user can create more than 20 follows in a row', async () => { + const {a, b} = world; + const sources = [a.ann]; + for (let index = 1; index < 7; index++) { + sources.push( + await createGuildChannel(harness, a.owner.token, a.guild.id, { + name: `news-${index}`, + type: ChannelTypes.GUILD_ANNOUNCEMENT, + }), + ); + } + const third = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'third', + type: ChannelTypes.GUILD_TEXT, + }); + const pairs: Array<[ChannelResponse, ChannelResponse]> = []; + for (const target of [b.t1, b.t2, third]) { + for (const source of sources) { + pairs.push([source, target]); + } + } + expect(pairs.length).toBeGreaterThan(20); + for (const [source, target] of pairs) { + await follow(harness, b.owner.token, source.id, target.id); + } + for (const target of [b.t1, b.t2, third]) { + const webhooks = await getChannelWebhooks(harness, target.id, b.owner.token); + expect(webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)).toHaveLength(sources.length); + } + }); + test('reports follower stats, cached for a minute and refreshed by a follow', async () => { + const {a, b} = world; + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await follow(harness, b.owner.token, a.ann.id, b.t2.id); + expect(await followerStats(harness, a.member.token, a.ann.id)).toEqual({channel_count: 2, guild_count: 1}); + await new WebhookRepository().create({ + webhookId: createWebhookID(BigInt(b.voice.id) + 7n), + token: createWebhookToken('s'.repeat(64)), + type: WebhookTypes.CHANNEL_FOLLOWER, + guildId: createGuildID(BigInt(b.guild.id)), + channelId: createChannelID(BigInt(b.ageRestricted.id)), + creatorId: createUserID(BigInt(b.owner.userId)), + name: 'Direct', + avatarHash: null, + sourceGuildId: createGuildID(BigInt(a.guild.id)), + sourceChannelId: createChannelID(BigInt(a.ann.id)), + }); + expect(await followerStats(harness, a.member.token, a.ann.id)).toEqual({channel_count: 2, guild_count: 1}); + const c = await createTestAccount(harness); + const guildC = await createGuild(harness, c.token, 'Guild C'); + await addGuildMember(harness, a.owner, a.guild, c); + await follow(harness, c.token, a.ann.id, guildC.system_channel_id!); + expect(await followerStats(harness, a.member.token, a.ann.id)).toEqual({channel_count: 4, guild_count: 2}); + }); + + test('rejects follower stats outside announcement channels and without access', async () => { + const {a, b} = world; + await createBuilder(harness, a.owner.token) + .get(`/channels/${a.guild.system_channel_id}/follower-stats`) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + const outsider = await createTestAccount(harness); + await createBuilder(harness, outsider.token) + .get(`/channels/${a.ann.id}/follower-stats`) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.ACCESS_DENIED) + .execute(); + await createPermissionOverwrite(harness, a.owner.token, a.ann.id, b.owner.userId, { + type: 1, + allow: '0', + deny: Permissions.VIEW_CHANNEL.toString(), + }); + await createBuilder(harness, b.owner.token) + .get(`/channels/${a.ann.id}/follower-stats`) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + }); + + test('lets exactly one of two concurrent identical follows succeed', async () => { + const {a, b} = world; + const attempts = await Promise.all([ + followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.OK) + .executeWithResponse() + .then( + () => 'ok', + (error: Error) => error.message, + ), + followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.OK) + .executeWithResponse() + .then( + () => 'ok', + (error: Error) => error.message, + ), + ]); + expect(attempts.filter((result) => result === 'ok')).toHaveLength(1); + const failure = attempts.find((result) => result !== 'ok')!; + expect(failure).toMatch(/CHANNEL_ALREADY_FOLLOWED|RESOURCE_LOCKED/); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(1); + }); + + test('never leaves a follower webhook inside a channel converted to announcement concurrently', async () => { + const {a, b} = world; + await Promise.allSettled([ + follow(harness, b.owner.token, a.ann.id, b.t1.id), + createBuilder(harness, b.owner.token).patch(`/channels/${b.t1.id}`).body({type: 5}).execute(), + ]); + const channel = await createBuilder(harness, b.owner.token).get(`/channels/${b.t1.id}`).execute(); + const webhooks = await new WebhookRepository().listByChannel(createChannelID(BigInt(b.t1.id))); + const followers = webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER); + if (channel.type === ChannelTypes.GUILD_ANNOUNCEMENT) { + expect(followers).toHaveLength(0); + } else { + expect(followers).toHaveLength(1); + } + }); + + test('undoes a follow whose source stopped being an announcement channel mid-request', async () => { + const {a, b} = world; + const originalCreate = WebhookRepository.prototype.create; + vi.spyOn(WebhookRepository.prototype, 'create').mockImplementation(async function (this: WebhookRepository, data) { + const created = await originalCreate.call(this, data); + await createBuilder(harness, a.owner.token).patch(`/channels/${a.ann.id}`).body({type: 0}).execute(); + return created; + }); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + vi.restoreAllMocks(); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(0); + expect(await new WebhookRepository().countBySourceChannel(createChannelID(BigInt(a.ann.id)))).toEqual({ + channelCount: 0, + guildCount: 0, + }); + const messages = await listMessages(harness, b.owner.token, b.t1.id); + expect(messages.some((message) => message.type === MessageTypes.CHANNEL_FOLLOW_ADD)).toBe(false); + }); + + test('refuses to follow while the source guild has announcement channels disabled', async () => { + const {a, b} = world; + await setGuildFeatures(harness, a.guild.id, {add: [GuildFeatures.ANNOUNCEMENT_CHANNELS_DISABLED]}); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.FEATURE_TEMPORARILY_DISABLED) + .execute(); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(0); + }); +}); diff --git a/fluxer_api/src/api/webhook/tests/ChannelFollowPermissions.test.ts b/fluxer_api/src/api/webhook/tests/ChannelFollowPermissions.test.ts new file mode 100644 index 000000000..3a9342f45 --- /dev/null +++ b/fluxer_api/src/api/webhook/tests/ChannelFollowPermissions.test.ts @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {authorizeBot, createTestBotAccount} from '@app/api/bot/tests/BotTestUtils'; +import { + type AnnouncementWorld, + addGuildMember, + announcementWorld, + createGuildChannel, + follow, + followRequest, + grantGuildRole, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import { + createDmChannel, + createFriendship, + createGuild, + createPermissionOverwrite, + updateGuild, +} from '@app/api/channel/tests/ChannelTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {getChannelWebhooks} from '@app/api/webhook/tests/WebhookTestUtils'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {ChannelTypes, Permissions, WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import {ContentWarningLevel} from '@fluxer/constants/src/GuildConstants'; +import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; + +describe('Channel follow permissions and content rules', () => { + let harness: ApiTestHarness; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await announcementWorld(harness); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function createAnnouncement(owner: TestAccount, guildId: string, body: Record = {}) { + return createGuildChannel(harness, owner.token, guildId, { + name: 'news-extra', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + ...body, + }); + } + + test('requires the source to be an announcement channel', async () => { + const {a, b} = world; + await followRequest(harness, b.owner.token, a.guild.system_channel_id!, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.ANNOUNCEMENT_CHANNEL_REQUIRED) + .execute(); + }); + + test('only accepts text channels as targets', async () => { + const {a, b} = world; + const category = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'category', + type: ChannelTypes.GUILD_CATEGORY, + }); + const link = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'link', + type: ChannelTypes.GUILD_LINK, + url: 'https://example.com', + }); + const announcement = await createAnnouncement(b.owner, b.guild.id); + for (const target of [b.voice, category, link, announcement]) { + await followRequest(harness, b.owner.token, a.ann.id, target.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FOLLOW_TARGET_CHANNEL) + .execute(); + } + }); + + test('rejects DM and group DM targets', async () => { + const {a, b} = world; + const friend = await createTestAccount(harness); + await createFriendship(harness, b.owner, friend); + const dm = await createDmChannel(harness, b.owner.token, friend.userId); + await followRequest(harness, b.owner.token, a.ann.id, dm.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FOLLOW_TARGET_CHANNEL) + .execute(); + const other = await createTestAccount(harness); + await createFriendship(harness, b.owner, other); + const groupDm = await createBuilder(harness, b.owner.token) + .post('/users/@me/channels') + .body({recipients: [friend.userId, other.userId]}) + .execute(); + await followRequest(harness, b.owner.token, a.ann.id, groupDm.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FOLLOW_TARGET_CHANNEL) + .execute(); + }); + + test('requires membership and View Channel on the source', async () => { + const {a, b} = world; + const outsider = await createTestAccount(harness); + const outsiderGuild = await createGuild(harness, outsider.token, 'Outsider Guild'); + await followRequest(harness, outsider.token, a.ann.id, outsiderGuild.system_channel_id!) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.ACCESS_DENIED) + .execute(); + await createPermissionOverwrite(harness, a.owner.token, a.ann.id, b.owner.userId, { + type: 1, + allow: '0', + deny: Permissions.VIEW_CHANNEL.toString(), + }); + await followRequest(harness, b.owner.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + }); + + test('requires Manage Webhooks in the target guild and channel', async () => { + const {a, b} = world; + const plain = await createTestAccount(harness); + await addGuildMember(harness, a.owner, a.guild, plain); + await addGuildMember(harness, b.owner, b.guild, plain); + await followRequest(harness, plain.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await grantGuildRole(harness, b.owner, b.guild.id, plain, Permissions.MANAGE_WEBHOOKS, 'Hooks'); + await createPermissionOverwrite(harness, b.owner.token, b.t2.id, plain.userId, { + type: 1, + allow: '0', + deny: Permissions.MANAGE_WEBHOOKS.toString(), + }); + await followRequest(harness, plain.token, a.ann.id, b.t2.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await follow(harness, plain.token, a.ann.id, b.t1.id); + }); + + test('a channel-only Manage Webhooks grant still needs the guild permission', async () => { + const {a, b} = world; + await followRequest(harness, b.webhookManager.token, a.ann.id, b.t1.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + await grantGuildRole(harness, b.owner, b.guild.id, b.webhookManager, Permissions.MANAGE_WEBHOOKS, 'Hooks'); + await createPermissionOverwrite(harness, b.owner.token, b.t2.id, b.webhookManager.userId, { + type: 1, + allow: '0', + deny: Permissions.MANAGE_WEBHOOKS.toString(), + }); + await follow(harness, b.webhookManager.token, a.ann.id, b.t1.id); + await followRequest(harness, b.webhookManager.token, a.ann.id, b.t2.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + }); + + test('an age-restricted source only posts into age-restricted targets', async () => { + const {a, b} = world; + const restrictedSource = await createAnnouncement(a.owner, a.guild.id, {nsfw_override: true}); + await followRequest(harness, b.owner.token, restrictedSource.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_NOT_AGE_RESTRICTED) + .execute(); + await followRequest(harness, b.owner.token, restrictedSource.id, b.contentWarning.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_NOT_AGE_RESTRICTED) + .execute(); + await follow(harness, b.owner.token, restrictedSource.id, b.ageRestricted.id); + }); + + test('age restriction inherited from a category or guild counts on both sides', async () => { + const {a, b} = world; + const sourceCategory = await createGuildChannel(harness, a.owner.token, a.guild.id, { + name: 'adult', + type: ChannelTypes.GUILD_CATEGORY, + nsfw_override: true, + }); + const inheritedSource = await createAnnouncement(a.owner, a.guild.id, {parent_id: sourceCategory.id}); + await followRequest(harness, b.owner.token, inheritedSource.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_NOT_AGE_RESTRICTED) + .execute(); + const targetCategory = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'adult-target', + type: ChannelTypes.GUILD_CATEGORY, + nsfw_override: true, + }); + const inheritedTarget = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'inherits', + type: ChannelTypes.GUILD_TEXT, + parent_id: targetCategory.id, + }); + await follow(harness, b.owner.token, inheritedSource.id, inheritedTarget.id); + + const adultOwner = await createTestAccount(harness); + const adultGuild = await createGuild(harness, adultOwner.token, 'Adult Guild'); + await updateGuild(harness, adultOwner.token, adultGuild.id, {nsfw: true}); + const guildSource = await createAnnouncement(adultOwner, adultGuild.id); + await addGuildMember(harness, adultOwner, adultGuild, b.owner); + await followRequest(harness, b.owner.token, guildSource.id, b.t2.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_NOT_AGE_RESTRICTED) + .execute(); + const adultTarget = await createGuildChannel(harness, adultOwner.token, adultGuild.id, { + name: 'adult-text', + type: ChannelTypes.GUILD_TEXT, + }); + await follow(harness, adultOwner.token, guildSource.id, adultTarget.id); + await addGuildMember(harness, a.owner, a.guild, adultOwner); + await follow(harness, adultOwner.token, inheritedSource.id, adultTarget.id); + }); + + test('a content-warning source only posts into warned or age-restricted targets', async () => { + const {a, b} = world; + const warnedSource = await createAnnouncement(a.owner, a.guild.id, { + content_warning_level: ContentWarningLevel.CONTENT_WARNING, + }); + await followRequest(harness, b.owner.token, warnedSource.id, b.t1.id) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_CONTENT_WARNING_REQUIRED) + .execute(); + await follow(harness, b.owner.token, warnedSource.id, b.contentWarning.id); + await follow(harness, b.owner.token, warnedSource.id, b.ageRestricted.id); + const warnedGuildOwner = await createTestAccount(harness); + const warnedGuild = await createGuild(harness, warnedGuildOwner.token, 'Warned Guild'); + await updateGuild(harness, warnedGuildOwner.token, warnedGuild.id, { + content_warning_level: ContentWarningLevel.CONTENT_WARNING, + }); + await addGuildMember(harness, a.owner, a.guild, warnedGuildOwner); + await follow(harness, warnedGuildOwner.token, warnedSource.id, warnedGuild.system_channel_id!); + }); + + test('an unrestricted source may post into an age-restricted target', async () => { + const {a, b} = world; + await follow(harness, b.owner.token, a.ann.id, b.ageRestricted.id); + }); + + test('an age-restricted source requires an age-verified caller', async () => { + const {a, b} = world; + const restrictedSource = await createAnnouncement(a.owner, a.guild.id, {nsfw_override: true}); + const minor = await createTestAccount(harness, {dateOfBirth: '2011-06-15'}); + await addGuildMember(harness, a.owner, a.guild, minor); + await addGuildMember(harness, b.owner, b.guild, minor); + await grantGuildRole(harness, b.owner, b.guild.id, minor, Permissions.MANAGE_WEBHOOKS, 'Hooks'); + await followRequest(harness, minor.token, restrictedSource.id, b.ageRestricted.id) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.NSFW_CONTENT_AGE_RESTRICTED) + .execute(); + }); + + test('a bot with Manage Webhooks can follow', async () => { + const {b} = world; + const bot = await createTestBotAccount(harness); + const sourceGuild = await createGuild(harness, bot.ownerToken, 'Bot Source'); + const source = await createGuildChannel(harness, bot.ownerToken, sourceGuild.id, { + name: 'bot-news', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + }); + await authorizeBot( + harness, + bot.ownerToken, + bot.appId, + ['bot'], + sourceGuild.id, + Permissions.VIEW_CHANNEL.toString(), + ); + await authorizeBot( + harness, + b.owner.token, + bot.appId, + ['bot'], + b.guild.id, + (Permissions.VIEW_CHANNEL | Permissions.MANAGE_WEBHOOKS).toString(), + ); + const followed = await follow(harness, `Bot ${bot.botToken}`, source.id, b.t1.id); + const webhooks = await getChannelWebhooks(harness, b.t1.id, b.owner.token); + const webhook = webhooks.find((candidate) => candidate.id === followed.webhook_id); + expect(webhook?.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + expect(webhook?.user.id).toBe(bot.botUserId); + }); +}); diff --git a/fluxer_api/src/api/webhook/tests/FollowerWebhookManagement.test.ts b/fluxer_api/src/api/webhook/tests/FollowerWebhookManagement.test.ts new file mode 100644 index 000000000..de6509876 --- /dev/null +++ b/fluxer_api/src/api/webhook/tests/FollowerWebhookManagement.test.ts @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {setUserACLs} from '@app/api/auth/tests/AuthTestUtils'; +import {createChannelID, createGuildID} from '@app/api/BrandedTypes'; +import { + type AnnouncementWorld, + addGuildMember, + announcementWorld, + createGuildChannel, + disableCrosspostWorker, + findWebhookRow, + follow, + grantGuildRole, +} from '@app/api/channel/tests/AnnouncementTestUtils'; +import {createPermissionOverwrite, deleteChannel, leaveGuild} from '@app/api/channel/tests/ChannelTestUtils'; +import { + copiesOf, + type FanoutWorld, + followInto, + postAndPublish, + publishAndDrain, + readRow, + sendWithImage, + setupFanoutWorld, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {createWebhook, getChannelWebhooks, getGuildWebhooks} from '@app/api/webhook/tests/WebhookTestUtils'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import {CROSSPOST_SOURCE_DELETED_CONTENT} from '@fluxer/constants/src/AnnouncementConstants'; +import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType'; +import {ChannelTypes, MessageFlags, Permissions, WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import type {GuildAuditLogListResponse} from '@fluxer/schema/src/domains/guild/GuildAuditLogSchemas'; +import {WebhookListResponse, WebhookResponse} from '@fluxer/schema/src/domains/webhook/WebhookSchemas'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; + +function webhooksUpdateChannels(spy: { + mock: {calls: Array>}; +}): Array { + return spy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'WEBHOOKS_UPDATE') + .map((params) => (params.data as {channel_id: string}).channel_id); +} + +async function sourceIndex(sourceChannelId: string) { + return new WebhookRepository().listIdsBySourceChannel(createChannelID(BigInt(sourceChannelId)), {limit: 100}); +} + +describe('Follower webhook management', () => { + let harness: ApiTestHarness; + let world: AnnouncementWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await announcementWorld(harness); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + test('rejects every token route for a follower webhook', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const row = await findWebhookRow(followed.webhook_id); + expect(row?.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + const base = `/webhooks/${followed.webhook_id}/${row!.token}`; + const messages = await createBuilder>(harness, b.owner.token) + .get(`/channels/${b.t1.id}/messages`) + .execute(); + const messageId = messages[0]!.id; + const attempts: Array<{method: 'get' | 'post' | 'patch' | 'delete'; path: string; body?: unknown}> = [ + {method: 'get', path: base}, + {method: 'patch', path: base, body: {name: 'Hijacked'}}, + {method: 'delete', path: base}, + {method: 'post', path: base, body: {content: 'hello'}}, + {method: 'post', path: `${base}?wait=true`, body: {content: 'hello'}}, + {method: 'post', path: `${base}/slack`, body: {text: 'hello'}}, + {method: 'post', path: `${base}/instatus`, body: {meta: {unsubscribe: '', documentation: ''}, page: {id: 'x'}}}, + {method: 'get', path: `${base}/messages/${messageId}`}, + {method: 'patch', path: `${base}/messages/${messageId}`, body: {content: 'edited'}}, + {method: 'delete', path: `${base}/messages/${messageId}`}, + ]; + for (const attempt of attempts) { + const builder = createBuilderWithoutAuth(harness)[attempt.method](attempt.path); + if (attempt.body !== undefined) builder.body(attempt.body); + await builder.expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_WEBHOOK).execute(); + } + const github = await harness.requestJson({ + path: `${base}/github`, + method: 'POST', + body: { + repository: {id: 789, full_name: 'test/repo', name: 'repo', html_url: 'https://github.com/test/repo'}, + sender: { + id: 123456, + login: 'testauthor', + html_url: 'https://github.com/testauthor', + avatar_url: 'https://avatars.githubusercontent.com/u/123456', + }, + }, + headers: {'X-GitHub-Event': 'ping', 'X-GitHub-Delivery': 'delivery-1'}, + }); + expect(github.status).toBe(HTTP_STATUS.NOT_FOUND); + expect(((await github.json()) as {code: string}).code).toBe(APIErrorCodes.UNKNOWN_WEBHOOK); + expect(await findWebhookRow(followed.webhook_id)).not.toBeNull(); + }); + + test('allows renaming a follower webhook', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const updated = await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${followed.webhook_id}`) + .body({name: 'Release feed'}) + .execute(); + WebhookResponse.parse(updated); + expect(updated.name).toBe('Release feed'); + expect(updated.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + expect(updated).not.toHaveProperty('token'); + expect(updated.source_channel).toEqual({id: a.ann.id, name: 'news'}); + const row = await findWebhookRow(followed.webhook_id); + expect(row?.name).toBe('Release feed'); + expect(row?.sourceChannelId?.toString()).toBe(a.ann.id); + expect(row?.sourceGuildId?.toString()).toBe(a.guild.id); + }); + + test.each([ + ['an image', () => getPngDataUrl()], + ['null', () => null], + ])('rejects %s as a follower webhook avatar', async (_label, avatar) => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const before = await findWebhookRow(followed.webhook_id); + const response = await createBuilder<{code: string; errors: Array<{path: string; code: string}>}>( + harness, + b.owner.token, + ) + .patch(`/webhooks/${followed.webhook_id}`) + .body({name: 'Release feed', channel_id: b.t2.id, avatar: avatar()}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FORM_BODY) + .execute(); + expect(response.errors[0]?.path).toBe('avatar'); + expect(response.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_FORMAT); + const after = await findWebhookRow(followed.webhook_id); + expect(after?.name).toBe(before?.name); + expect(after?.avatarHash).toBe(before?.avatarHash); + expect(after?.channelId?.toString()).toBe(b.t1.id); + }); + + test('still lets an incoming webhook change its avatar', async () => { + const {b} = world; + const webhook = await createWebhook(harness, b.t1.id, b.owner.token, 'Incoming'); + const updated = await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${webhook.id}`) + .body({avatar: getPngDataUrl()}) + .execute(); + expect(updated.avatar).toBeTruthy(); + const cleared = await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${webhook.id}`) + .body({avatar: null}) + .execute(); + expect(cleared.avatar).toBeNull(); + }); + + test('moves a follower webhook to another text channel and notifies both channels', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const moved = await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${followed.webhook_id}`) + .body({channel_id: b.t2.id}) + .execute(); + expect(moved.channel_id).toBe(b.t2.id); + expect(moved.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + expect(moved.source_guild?.id).toBe(a.guild.id); + expect(moved.source_channel?.id).toBe(a.ann.id); + expect(webhooksUpdateChannels(dispatchSpy).sort()).toEqual([b.t1.id, b.t2.id].sort()); + expect(await sourceIndex(a.ann.id)).toEqual([ + {webhookId: BigInt(followed.webhook_id), guildId: createGuildID(BigInt(b.guild.id))}, + ]); + expect(await getChannelWebhooks(harness, b.t1.id, b.owner.token)).toHaveLength(0); + }); + + test('validates follower webhook moves', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const announcement = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'b-news', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + }); + for (const target of [b.voice.id, announcement.id]) { + await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${followed.webhook_id}`) + .body({channel_id: target}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FOLLOW_TARGET_CHANNEL) + .execute(); + } + await follow(harness, b.owner.token, a.ann.id, b.t2.id); + await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${followed.webhook_id}`) + .body({channel_id: b.t2.id}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CHANNEL_ALREADY_FOLLOWED) + .execute(); + const restricted = await createGuildChannel(harness, a.owner.token, a.guild.id, { + name: 'adult-news', + type: ChannelTypes.GUILD_ANNOUNCEMENT, + nsfw_override: true, + }); + const restrictedFollow = await follow(harness, b.owner.token, restricted.id, b.ageRestricted.id); + await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${restrictedFollow.webhook_id}`) + .body({channel_id: b.t1.id}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.FOLLOW_TARGET_NOT_AGE_RESTRICTED) + .execute(); + await grantGuildRole(harness, b.owner, b.guild.id, b.webhookManager, Permissions.MANAGE_WEBHOOKS, 'Hooks'); + await createPermissionOverwrite(harness, b.owner.token, b.contentWarning.id, b.webhookManager.userId, { + type: 1, + allow: '0', + deny: Permissions.MANAGE_WEBHOOKS.toString(), + }); + const third = await createGuildChannel(harness, b.owner.token, b.guild.id, { + name: 'third', + type: ChannelTypes.GUILD_TEXT, + }); + const managed = await follow(harness, b.owner.token, a.ann.id, third.id); + await createBuilder(harness, b.webhookManager.token) + .patch(`/webhooks/${managed.webhook_id}`) + .body({channel_id: b.contentWarning.id}) + .expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.MISSING_PERMISSIONS) + .execute(); + expect((await findWebhookRow(followed.webhook_id))?.channelId?.toString()).toBe(b.t1.id); + }); + + test('a follower webhook move loses to a concurrent conversion of the target to announcement', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const originalCount = WebhookRepository.prototype.countByChannel; + let converted = false; + vi.spyOn(WebhookRepository.prototype, 'countByChannel').mockImplementation(async function ( + this: WebhookRepository, + channelId, + ) { + const result = await originalCount.call(this, channelId); + if (!converted && channelId.toString() === b.t2.id) { + converted = true; + await createBuilder(harness, b.owner.token) + .patch(`/channels/${b.t2.id}`) + .body({type: ChannelTypes.GUILD_ANNOUNCEMENT}) + .expect(HTTP_STATUS.OK) + .execute(); + } + return result; + }); + await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${followed.webhook_id}`) + .body({channel_id: b.t2.id}) + .expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INVALID_FOLLOW_TARGET_CHANNEL) + .execute(); + vi.restoreAllMocks(); + expect(converted).toBe(true); + expect((await findWebhookRow(followed.webhook_id))?.channelId?.toString()).toBe(b.t1.id); + const t2Webhooks = await new WebhookRepository().listByChannel(createChannelID(BigInt(b.t2.id))); + expect(t2Webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)).toHaveLength(0); + }); + + test('dispatches WEBHOOKS_UPDATE for both channels when an incoming webhook moves', async () => { + const {b} = world; + const webhook = await createWebhook(harness, b.t1.id, b.owner.token, 'Incoming'); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await createBuilder(harness, b.owner.token) + .patch(`/webhooks/${webhook.id}`) + .body({channel_id: b.t2.id}) + .execute(); + expect(webhooksUpdateChannels(dispatchSpy).sort()).toEqual([b.t1.id, b.t2.id].sort()); + }); + + test('unfollows by deleting the follower webhook', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await createBuilder(harness, b.owner.token) + .delete(`/webhooks/${followed.webhook_id}`) + .header('X-Audit-Log-Reason', 'no longer needed') + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + expect(webhooksUpdateChannels(dispatchSpy)).toEqual([b.t1.id]); + expect(await findWebhookRow(followed.webhook_id)).toBeNull(); + expect(await sourceIndex(a.ann.id)).toEqual([]); + const auditLog = await createBuilder(harness, b.owner.token) + .get(`/guilds/${b.guild.id}/audit-logs?action_type=${AuditLogActionType.WEBHOOK_DELETE}`) + .execute(); + const entry = auditLog.audit_log_entries.find((candidate) => candidate.target_id === followed.webhook_id); + expect(entry?.reason).toBe('no longer needed'); + expect(entry?.options).toEqual({channel_id: b.t1.id, type: WebhookTypes.CHANNEL_FOLLOWER}); + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + }); + + test('deleting the target channel removes the follower webhook and its index row', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await deleteChannel(harness, b.owner.token, b.t1.id); + expect(await findWebhookRow(followed.webhook_id)).toBeNull(); + expect(await sourceIndex(a.ann.id)).toEqual([]); + }); + + test('deleting the target guild removes all of its follower index rows', async () => { + const {a, b} = world; + await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await follow(harness, b.owner.token, a.ann.id, b.t2.id); + expect(await sourceIndex(a.ann.id)).toHaveLength(2); + await createBuilder(harness, b.owner.token) + .post(`/guilds/${b.guild.id}/delete`) + .body({password: b.owner.password}) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + expect(await sourceIndex(a.ann.id)).toEqual([]); + }); + + test('lists both webhook types, with a token only on incoming webhooks', async () => { + const {a, b} = world; + const incoming = await createWebhook(harness, b.t1.id, b.owner.token, 'Incoming'); + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + const guildList = WebhookListResponse.parse(await getGuildWebhooks(harness, b.guild.id, b.owner.token)); + const channelList = WebhookListResponse.parse(await getChannelWebhooks(harness, b.t1.id, b.owner.token)); + for (const list of [guildList, channelList]) { + const incomingEntry = list.find((webhook) => webhook.id === incoming.id); + const followerEntry = list.find((webhook) => webhook.id === followed.webhook_id); + expect(incomingEntry?.type).toBe(WebhookTypes.INCOMING); + expect(incomingEntry?.token).toBe(incoming.token); + expect(incomingEntry).not.toHaveProperty('source_guild'); + expect(followerEntry?.type).toBe(WebhookTypes.CHANNEL_FOLLOWER); + expect(followerEntry).not.toHaveProperty('token'); + expect(followerEntry?.source_guild?.id).toBe(a.guild.id); + } + const single = WebhookResponse.parse( + await createBuilder(harness, b.owner.token).get(`/webhooks/${followed.webhook_id}`).execute(), + ); + expect(single).not.toHaveProperty('token'); + expect(single.source_channel?.id).toBe(a.ann.id); + }); + + test('hides the source while the creator cannot view it and shows it again after they rejoin', async () => { + const {a, b} = world; + const followed = await follow(harness, b.owner.token, a.ann.id, b.t1.id); + await leaveGuild(harness, b.owner.token, a.guild.id); + const paused = (await getChannelWebhooks(harness, b.t1.id, b.owner.token)).find( + (webhook) => webhook.id === followed.webhook_id, + ); + expect(paused).toBeDefined(); + expect(paused).not.toHaveProperty('source_guild'); + expect(paused).not.toHaveProperty('source_channel'); + await addGuildMember(harness, a.owner, a.guild, b.owner); + const resumed = (await getChannelWebhooks(harness, b.t1.id, b.owner.token)).find( + (webhook) => webhook.id === followed.webhook_id, + ); + expect(resumed?.source_guild?.id).toBe(a.guild.id); + expect(resumed?.source_channel?.id).toBe(a.ann.id); + await createPermissionOverwrite(harness, a.owner.token, a.ann.id, b.owner.userId, { + type: 1, + allow: '0', + deny: Permissions.VIEW_CHANNEL.toString(), + }); + const hidden = (await getChannelWebhooks(harness, b.t1.id, b.owner.token)).find( + (webhook) => webhook.id === followed.webhook_id, + ); + expect(hidden).not.toHaveProperty('source_guild'); + }); +}); + +describe('Follower webhook delivery', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + harness.storageService.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + const DELETED_COPY_FLAGS = MessageFlags.IS_CROSSPOST | MessageFlags.SOURCE_MESSAGE_DELETED; + + async function publishedCopyWithImage(channelId: string) { + const source = await sendWithImage(harness, world.a.owner.token, world.a.ann.id, {content: 'with image'}); + await publishAndDrain(harness, world, source.id); + const [copy] = await copiesOf(harness, world.b.owner.token, channelId, source.id); + const [sourceAttachment] = (await readRow(world.a.ann.id, source.id))!.attachments; + const [copyAttachment] = (await readRow(channelId, copy!.id))!.attachments; + expect(copyAttachment!.id).toBe(sourceAttachment!.id); + return { + source, + copy: copy!, + sourceKey: `attachments/${world.a.ann.id}/${sourceAttachment!.id}/${sourceAttachment!.filename}`, + }; + } + + function expectNoTargetObjectsDeleted(): void { + for (const channelId of [world.b.t1.id, world.b.t2.id]) { + expect( + harness.storageService.getDeletedObjects().some((entry) => entry.key.startsWith(`attachments/${channelId}/`)), + ).toBe(false); + } + } + + test('the next copy uses the renamed follower webhook', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + const before = await postAndPublish(harness, world, {content: 'old identity'}); + await createBuilder(harness, world.b.owner.token) + .patch(`/webhooks/${webhookId}`) + .body({name: 'Release feed'}) + .execute(); + const after = await postAndPublish(harness, world, {content: 'new identity'}); + const [oldCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, before.id); + const [newCopy] = await copiesOf(harness, world.b.owner.token, world.b.t1.id, after.id); + expect(oldCopy!.author.username).not.toBe('Release feed'); + expect(newCopy!.author.username).toBe('Release feed'); + }); + + test('a moved follower webhook delivers the next publish into its new channel', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + await createBuilder(harness, world.b.owner.token) + .patch(`/webhooks/${webhookId}`) + .body({channel_id: world.b.t2.id}) + .expect(HTTP_STATUS.OK) + .execute(); + const source = await postAndPublish(harness, world, {content: 'after the move'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t2.id, source.id)).toHaveLength(1); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(0); + }); + + test('after an unfollow the next publish makes no copy there', async () => { + const webhookId = await followInto(harness, world, world.b.t1.id); + await createBuilder(harness, world.b.owner.token) + .delete(`/webhooks/${webhookId}`) + .expect(HTTP_STATUS.NO_CONTENT) + .execute(); + const source = await postAndPublish(harness, world, {content: 'nobody listening'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, source.id)).toHaveLength(0); + expect(world.worker.deadLetters).toHaveLength(0); + }); + + test('deleting the source channel removes followers and marks earlier copies deleted', async () => { + await followInto(harness, world, world.b.t1.id); + const {copy, sourceKey} = await publishedCopyWithImage(world.b.t1.id); + await deleteChannel(harness, world.a.owner.token, world.a.ann.id); + const [removal] = world.worker.byTask('removeChannelFollowers'); + expect(removal!.payload).toEqual({sourceChannelId: world.a.ann.id, reason: 'deleted', copyMode: 'source_deleted'}); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + const webhooks = await getChannelWebhooks(harness, world.b.t1.id, world.b.owner.token); + expect(webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)).toHaveLength(0); + const marked = (await readRow(world.b.t1.id, copy.id))!; + expect(marked.flags).toBe(DELETED_COPY_FLAGS); + expect(marked.content).toBe(CROSSPOST_SOURCE_DELETED_CONTENT); + expect(marked.attachments).toHaveLength(0); + expect(harness.storageService.getDeletedObjects().filter((entry) => entry.key === sourceKey)).toHaveLength(1); + expectNoTargetObjectsDeleted(); + }); + + test('a user deleting the source guild marks copies deleted', async () => { + await followInto(harness, world, world.b.t1.id); + const {copy} = await publishedCopyWithImage(world.b.t1.id); + await createBuilder(harness, world.a.owner.token) + .post(`/guilds/${world.a.guild.id}/delete`) + .body({password: world.a.owner.password}) + .expect(HTTP_STATUS.NO_CONTENT) + .executeWithResponse(); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + expect((await readRow(world.b.t1.id, copy.id))!.flags).toBe(DELETED_COPY_FLAGS); + }); + + test('an admin deleting the source guild deletes the copies', async () => { + await followInto(harness, world, world.b.t1.id); + const {copy} = await publishedCopyWithImage(world.b.t1.id); + const admin = await setUserACLs(harness, world.a.member, ['admin:authenticate', 'guild:lookup', 'guild:delete']); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + await createBuilder(harness, admin.token) + .delete(`/admin/guilds/${world.a.guild.id}`) + .body(null) + .expect(HTTP_STATUS.OK) + .execute(); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + expect(await readRow(world.b.t1.id, copy.id)).toBeNull(); + expectNoTargetObjectsDeleted(); + expect( + dispatchSpy.mock.calls.some( + ([params]) => + params.event === 'MESSAGE_DELETE' && + params.guildId.toString() === world.b.guild.id && + (params.data as {id: string}).id === copy.id, + ), + ).toBe(true); + }); + + test('delivery pauses while the creator cannot view the source and resumes when they rejoin', async () => { + await followInto(harness, world, world.b.t1.id); + await leaveGuild(harness, world.b.owner.token, world.a.guild.id); + const paused = await postAndPublish(harness, world, {content: 'while away'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, paused.id)).toHaveLength(0); + await addGuildMember(harness, world.a.owner, world.a.guild, world.b.owner); + const resumed = await postAndPublish(harness, world, {content: 'welcome back'}); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, resumed.id)).toHaveLength(1); + expect(await copiesOf(harness, world.b.owner.token, world.b.t1.id, paused.id)).toHaveLength(0); + }); +}); diff --git a/fluxer_api/src/api/webhook/tests/WebhookTestUtils.ts b/fluxer_api/src/api/webhook/tests/WebhookTestUtils.ts index 0e8aef252..0cc70fa85 100644 --- a/fluxer_api/src/api/webhook/tests/WebhookTestUtils.ts +++ b/fluxer_api/src/api/webhook/tests/WebhookTestUtils.ts @@ -6,15 +6,22 @@ import type {ApiTestHarness} from '@app/api/test/ApiTestHarness'; import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; import type {GuildEmojiWithUserResponse} from '@fluxer/schema/src/domains/guild/GuildEmojiSchemas'; import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas'; -import type {WebhookResponse, WebhookTokenResponse} from '@fluxer/schema/src/domains/webhook/WebhookSchemas'; +import type { + WebhookCreateResponse, + WebhookResponse, + WebhookTokenResponse, +} from '@fluxer/schema/src/domains/webhook/WebhookSchemas'; export async function createWebhook( harness: ApiTestHarness, channelId: string, token: string, name: string, -): Promise { - return createBuilder(harness, token).post(`/channels/${channelId}/webhooks`).body({name}).execute(); +): Promise { + return createBuilder(harness, token) + .post(`/channels/${channelId}/webhooks`) + .body({name}) + .execute(); } export async function getWebhook(harness: ApiTestHarness, webhookId: string, token: string): Promise { diff --git a/fluxer_api/src/api/worker/WorkerDependencies.ts b/fluxer_api/src/api/worker/WorkerDependencies.ts index b4408d254..1b23ed190 100644 --- a/fluxer_api/src/api/worker/WorkerDependencies.ts +++ b/fluxer_api/src/api/worker/WorkerDependencies.ts @@ -107,6 +107,7 @@ import {UserHarvestRepository} from '@app/api/user/UserHarvestRepository'; import type {UserPermissionUtils} from '@app/api/utils/UserPermissionUtils'; import type {VoiceRepository} from '@app/api/voice/VoiceRepository'; import type {VoiceTopology} from '@app/api/voice/VoiceTopology'; +import type {WebhookRepository} from '@app/api/webhook/WebhookRepository'; import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig'; import type {ICacheService} from '@pkgs/cache/src/ICacheService'; import type {IEmailService} from '@pkgs/email/src/IEmailService'; @@ -149,6 +150,7 @@ export interface WorkerDependencies { emailService: IEmailService; instanceConfigRepository: InstanceConfigRepository; inviteService: InviteService; + webhookRepository: WebhookRepository; workerService: IWorkerService; unfurlerService: IUnfurlerService; embedService: EmbedService; @@ -305,6 +307,7 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS emailService, instanceConfigRepository, inviteService, + webhookRepository, workerService, unfurlerService, embedService, diff --git a/fluxer_api/src/api/worker/WorkerLaneConfig.ts b/fluxer_api/src/api/worker/WorkerLaneConfig.ts index 9b396eaf9..d06e89809 100644 --- a/fluxer_api/src/api/worker/WorkerLaneConfig.ts +++ b/fluxer_api/src/api/worker/WorkerLaneConfig.ts @@ -96,6 +96,21 @@ const LANE_CONFIG = { ackWaitMs: 120000, maxDeliver: 25, }, + crosspost: { + consumerName: 'workers_crosspost', + tasks: [ + 'crosspostMessage', + 'crosspostMessageChunk', + 'syncCrosspostedMessage', + 'syncCrosspostCopies', + 'removeChannelFollowers', + ] as const, + retiredTasks: [], + concurrency: 8, + maxAckPending: 64, + ackWaitMs: 60000, + maxDeliver: 25, + }, } satisfies Record; export type WorkerTaskName = (typeof LANE_CONFIG)[keyof typeof LANE_CONFIG]['tasks'][number]; @@ -137,6 +152,7 @@ const WORKER_LANES: ReadonlyArray = [ makeLane('unfurl'), makeLane('lifecycle'), makeLane('batch'), + makeLane('crosspost'), ]; const WORKER_LANE_MAP = buildWorkerLaneMap(); const WORKER_LANE_NAMES = Object.freeze(WORKER_LANES.map((lane) => lane.name)); diff --git a/fluxer_api/src/api/worker/WorkerMain.ts b/fluxer_api/src/api/worker/WorkerMain.ts index 3e0eeb6c3..4ae68868f 100644 --- a/fluxer_api/src/api/worker/WorkerMain.ts +++ b/fluxer_api/src/api/worker/WorkerMain.ts @@ -17,6 +17,7 @@ import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRep import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository'; import {Logger} from '@app/api/Logger'; import type {LimitConfigService} from '@app/api/limits/LimitConfigService'; +import {startContentBlocklistCaches, stopContentBlocklistCaches} from '@app/api/middleware/ContentBlocklistCaches'; import { closeOwnedKVClient, createSnowflakeService, @@ -148,6 +149,7 @@ export async function startWorkerMain(): Promise { await jsConnectionManager?.drain(); jsConnectionManager = null; }); + await cleanupStep('content blocklist caches', stopContentBlocklistCaches); await cleanupStep('worker dependencies', () => { dependencies = null; clearWorkerDependencies(); @@ -268,6 +270,8 @@ export async function startWorkerMain(): Promise { } dependencies = await initializeWorkerDependencies(snowflakeService); setWorkerDependencies(dependencies); + await startContentBlocklistCaches({kvClient: dependencies.kvClient, storageService: dependencies.storageService}); + Logger.info('Content blocklist caches initialised for worker backend'); await queueBlocklistFeedStartupJobs(dependencies.kvClient, workerService, Config.blocklistFeeds.enabled); setActivityProcessChannel('worker'); await startActivityEvents({ diff --git a/fluxer_api/src/api/worker/WorkerTaskRegistry.ts b/fluxer_api/src/api/worker/WorkerTaskRegistry.ts index d92b4be34..0b79fc5fe 100644 --- a/fluxer_api/src/api/worker/WorkerTaskRegistry.ts +++ b/fluxer_api/src/api/worker/WorkerTaskRegistry.ts @@ -12,6 +12,8 @@ import batchGuildAuditLogMessageDeletes from '@app/api/worker/tasks/BatchGuildAu import bulkDeleteSelfMessagesImmediate from '@app/api/worker/tasks/BulkDeleteSelfMessagesImmediate'; import bulkDeleteUserMessages from '@app/api/worker/tasks/BulkDeleteUserMessages'; import bulkDeleteUserMessagesScoped from '@app/api/worker/tasks/BulkDeleteUserMessagesScoped'; +import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage'; +import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk'; import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime'; import drainActivitySpool from '@app/api/worker/tasks/DrainActivitySpool'; import expireAttachments from '@app/api/worker/tasks/ExpireAttachments'; @@ -42,7 +44,10 @@ import prunePostgresKvTtl from '@app/api/worker/tasks/PrunePostgresKvTtl'; import reconcileUserPayments from '@app/api/worker/tasks/ReconcileUserPayments'; import refreshSearchIndex from '@app/api/worker/tasks/RefreshSearchIndex'; import refreshStorePurchase from '@app/api/worker/tasks/RefreshStorePurchase'; +import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers'; import {sendSystemDm} from '@app/api/worker/tasks/SendSystemDm'; +import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies'; +import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage'; import syncDiscoveryIndex from '@app/api/worker/tasks/SyncDiscoveryIndex'; import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists'; import syncUrlBlocklists from '@app/api/worker/tasks/SyncUrlBlocklists'; @@ -64,6 +69,8 @@ export const workerTasks: Record = { bulkUpdateGuildFeatures: bulkUpdateGuildFeatures, bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags, bulkUpdateUserFlags: bulkUpdateUserFlags, + crosspostMessage, + crosspostMessageChunk, deleteUserMessagesInGuildByTime, drainActivitySpool, expireAttachments, @@ -93,10 +100,13 @@ export const workerTasks: Record = { pollAppStoreNotificationHistory, prunePostgresKvTtl, refreshSearchIndex, + removeChannelFollowers, sendSystemDm, syncFileShaBlocklists, syncUrlBlocklists, syncDiscoveryIndex, + syncCrosspostCopies, + syncCrosspostedMessage, flushUserActivityBuffer, userProcessPendingDeletion, userProcessPendingDeletions, diff --git a/fluxer_api/src/api/worker/tasks/BulkDeleteSelfMessagesImmediate.ts b/fluxer_api/src/api/worker/tasks/BulkDeleteSelfMessagesImmediate.ts index d0685e859..368023a80 100644 --- a/fluxer_api/src/api/worker/tasks/BulkDeleteSelfMessagesImmediate.ts +++ b/fluxer_api/src/api/worker/tasks/BulkDeleteSelfMessagesImmediate.ts @@ -48,6 +48,7 @@ const bulkDeleteSelfMessagesImmediate: WorkerTaskHandler = async (payload, helpe gatewayService, storageService, purgeQueue, + workerService, }); const result = await deletionService.deleteUserMessagesFiltered( userId, diff --git a/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessages.ts b/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessages.ts index f054a5160..6f338d0f9 100644 --- a/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessages.ts +++ b/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessages.ts @@ -16,7 +16,8 @@ const bulkDeleteUserMessages: WorkerTaskHandler = async (payload, helpers) => { helpers.logger.debug({payload: validated}, 'Processing bulkDeleteUserMessages task'); const userId = createUserID(BigInt(validated.userId)); const scheduledAtMs = validated.scheduledAt ?? Number.POSITIVE_INFINITY; - const {channelRepository, gatewayService, userRepository, storageService, purgeQueue} = getWorkerDependencies(); + const {channelRepository, gatewayService, userRepository, storageService, purgeQueue, workerService} = + getWorkerDependencies(); const user = await userRepository.findUniqueAssert(userId); if (!user.pendingBulkMessageDeletionAt) { Logger.debug({userId}, 'User has no pending bulk message deletion, skipping (already completed)'); @@ -27,6 +28,7 @@ const bulkDeleteUserMessages: WorkerTaskHandler = async (payload, helpers) => { gatewayService, storageService, purgeQueue, + workerService, }); const totalDeleted = await deletionService.deleteUserMessagesBulk(userId, { beforeTimestamp: scheduledAtMs, diff --git a/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessagesScoped.ts b/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessagesScoped.ts index 9476e27cc..1fd89b900 100644 --- a/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessagesScoped.ts +++ b/fluxer_api/src/api/worker/tasks/BulkDeleteUserMessagesScoped.ts @@ -16,12 +16,13 @@ const bulkDeleteUserMessagesScoped: WorkerTaskHandler = async (payload, helpers) const validated = PayloadSchema.parse(payload); helpers.logger.debug({payload: validated}, 'Processing bulkDeleteUserMessagesScoped task'); const userId = createUserID(BigInt(validated.userId)); - const {channelRepository, gatewayService, storageService, purgeQueue} = getWorkerDependencies(); + const {channelRepository, gatewayService, storageService, purgeQueue, workerService} = getWorkerDependencies(); const deletionService = new UserMessageDeletionService({ channelRepository, gatewayService, storageService, purgeQueue, + workerService, }); const totalDeleted = await deletionService.deleteUserMessagesInScope( userId, diff --git a/fluxer_api/src/api/worker/tasks/CrosspostMessage.ts b/fluxer_api/src/api/worker/tasks/CrosspostMessage.ts new file mode 100644 index 000000000..d65ade15d --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/CrosspostMessage.ts @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes'; +import { + CrosspostMessagePayloadSchema, + CrosspostTaskNames, + crosspostChunkJobKey, + crosspostFanoutJobKey, + isCrosspostedMessage, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import {chunkArray} from '@app/api/worker/tasks/utils/CrosspostWorker'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import {CROSSPOST_FANOUT_CHUNK_SIZE, CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +const CHUNK_ENQUEUE_CONCURRENCY = 16; + +const crosspostMessage: WorkerTaskHandler = async (payload, helpers) => { + const validated = CrosspostMessagePayloadSchema.parse(payload); + const {channelRepository, webhookRepository} = getWorkerDependencies(); + const channelId = createChannelID(BigInt(validated.channelId)); + const messageId = createMessageID(BigInt(validated.messageId)); + const source = await channelRepository.messages.getMessage(channelId, messageId); + if (!source || !isCrosspostedMessage(source)) { + helpers.logger.info({messageId: validated.messageId}, 'Skipping crosspost fan-out: source is not published'); + return; + } + const pageSize = CROSSPOST_FANOUT_PAGE_SIZE; + const page = await webhookRepository.listIdsBySourceChannel(channelId, { + afterWebhookId: validated.afterWebhookId ? createWebhookID(BigInt(validated.afterWebhookId)) : undefined, + limit: pageSize, + }); + const chunks = chunkArray( + page.map((entry) => entry.webhookId.toString()), + CROSSPOST_FANOUT_CHUNK_SIZE, + ); + for (let offset = 0; offset < chunks.length; offset += CHUNK_ENQUEUE_CONCURRENCY) { + await Promise.all( + chunks + .slice(offset, offset + CHUNK_ENQUEUE_CONCURRENCY) + .map((webhookIds) => + helpers.addJob( + CrosspostTaskNames.CROSSPOST_MESSAGE_CHUNK, + {channelId: validated.channelId, messageId: validated.messageId, webhookIds, attempt: 0}, + {jobKey: crosspostChunkJobKey(validated.messageId, webhookIds[0]!, 0), skipLedger: true}, + ), + ), + ); + } + if (page.length === pageSize && page.length > 0) { + const afterWebhookId = page[page.length - 1]!.webhookId.toString(); + await helpers.addJob( + CrosspostTaskNames.CROSSPOST_MESSAGE, + {channelId: validated.channelId, messageId: validated.messageId, afterWebhookId}, + {jobKey: crosspostFanoutJobKey(validated.messageId, afterWebhookId), skipLedger: true}, + ); + } +}; + +export default crosspostMessage; diff --git a/fluxer_api/src/api/worker/tasks/CrosspostMessageChunk.ts b/fluxer_api/src/api/worker/tasks/CrosspostMessageChunk.ts new file mode 100644 index 000000000..ce26d5a75 --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/CrosspostMessageChunk.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes'; +import { + CrosspostMessageChunkPayloadSchema, + CrosspostTaskNames, + crosspostChunkJobKey, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import {createCrosspostDeliveryService, mapWithConcurrency} from '@app/api/worker/tasks/utils/CrosspostWorker'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import { + CROSSPOST_CHUNK_RETRY_DELAYS_MS, + CROSSPOST_FANOUT_CONCURRENCY, +} from '@fluxer/constants/src/AnnouncementConstants'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +const crosspostMessageChunk: WorkerTaskHandler = async (payload, helpers) => { + const validated = CrosspostMessageChunkPayloadSchema.parse(payload); + const channelId = createChannelID(BigInt(validated.channelId)); + const messageId = createMessageID(BigInt(validated.messageId)); + const delivery = createCrosspostDeliveryService(); + const loaded = await delivery.loadSourceForDelivery(channelId, messageId); + if (loaded.kind === 'missing') { + const {channelRepository} = getWorkerDependencies(); + const mode = await delivery.removalModeFor(messageId); + for (const webhookId of validated.webhookIds) { + const row = await channelRepository.crossposts.get(messageId, createWebhookID(BigInt(webhookId))); + if (row) { + await delivery.syncCopy(row, mode, null); + } + } + return; + } + if (loaded.kind === 'inactive') { + helpers.logger.info({messageId: validated.messageId}, 'Skipping crosspost chunk: source is not deliverable'); + return; + } + const failed: Array = []; + await mapWithConcurrency(validated.webhookIds, CROSSPOST_FANOUT_CONCURRENCY, async (webhookId) => { + try { + await delivery.deliverToWebhook(loaded.context, createWebhookID(BigInt(webhookId))); + } catch (error) { + failed.push(webhookId); + helpers.logger.warn({error, messageId: validated.messageId, webhookId}, 'Crosspost delivery failed'); + } + }); + if (failed.length === 0) { + return; + } + const orderedFailed = validated.webhookIds.filter((webhookId) => failed.includes(webhookId)); + if (validated.attempt >= CROSSPOST_CHUNK_RETRY_DELAYS_MS.length) { + helpers.logger.error( + {messageId: validated.messageId, webhookIds: orderedFailed, attempt: validated.attempt}, + 'Dropping crosspost deliveries after the last retry', + ); + return; + } + const nextAttempt = validated.attempt + 1; + await helpers.addJob( + CrosspostTaskNames.CROSSPOST_MESSAGE_CHUNK, + { + channelId: validated.channelId, + messageId: validated.messageId, + webhookIds: orderedFailed, + attempt: nextAttempt, + }, + { + jobKey: crosspostChunkJobKey(validated.messageId, orderedFailed[0]!, nextAttempt), + runAt: new Date(Date.now() + CROSSPOST_CHUNK_RETRY_DELAYS_MS[validated.attempt]!), + skipLedger: true, + }, + ); +}; + +export default crosspostMessageChunk; diff --git a/fluxer_api/src/api/worker/tasks/ExtractEmbeds.ts b/fluxer_api/src/api/worker/tasks/ExtractEmbeds.ts index 190ffb928..5dd3ea835 100644 --- a/fluxer_api/src/api/worker/tasks/ExtractEmbeds.ts +++ b/fluxer_api/src/api/worker/tasks/ExtractEmbeds.ts @@ -3,6 +3,11 @@ import type {ChannelID, GuildID, MessageID} from '@app/api/BrandedTypes'; import {createChannelID, createGuildID, createMessageID, createUserID} from '@app/api/BrandedTypes'; import type {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import { + enqueueCrosspostSourceRemoval, + enqueueCrosspostSync, + isCrosspostedMessage, +} from '@app/api/channel/services/message/CrosspostPropagation'; import {buildBroadcastMessageData} from '@app/api/channel/services/message/MessageGatewayDispatch'; import type {MessageEmbed, MessageEmbedChild} from '@app/api/database/types/MessageTypes'; import type {ModerationContext} from '@app/api/infrastructure/ContentModerationService'; @@ -457,6 +462,7 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => { return; } const orderedEmbeds = buildOrderedEmbeds(urls, unfurledEmbedsByUrl); + let propagated: Message | null = null; const handled = await withMessageWriteLock(cacheService, channelId, messageId, async () => { const latestExpectedMessage = await channelRepository.getMessage(channelId, messageId); if (!latestExpectedMessage) { @@ -496,6 +502,10 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => { await deleteMessageSearchDocuments([messageId], {context: {source: 'blocked_embed_unfurl'}}); const eventDispatcher = new ChannelEventDispatcher({gatewayService}); await eventDispatcher.dispatchMessageDelete(channel, messageId); + await enqueueCrosspostSourceRemoval(getWorkerDependencies().workerService, { + messages: [latestExpectedMessage], + mode: 'purge', + }); return true; } Logger.error( @@ -507,6 +517,9 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => { if (!latestMessage) { return false; } + if (latestMessage !== latestExpectedMessage) { + propagated = latestMessage; + } if (!(latestMessage.flags & MessageFlags.SUPPRESS_EMBEDS)) { await dispatchEmbedUpdate({ latestMessage, @@ -519,6 +532,9 @@ const extractEmbeds: WorkerTaskHandler = async (payload, helpers) => { } return true; }); + if (propagated && isCrosspostedMessage(propagated)) { + await enqueueCrosspostSync(getWorkerDependencies().workerService, {channelId, messageId, mode: 'update'}); + } if (!handled) { return; } diff --git a/fluxer_api/src/api/worker/tasks/MessageShred.ts b/fluxer_api/src/api/worker/tasks/MessageShred.ts index 186e18f6d..150a0ec41 100644 --- a/fluxer_api/src/api/worker/tasks/MessageShred.ts +++ b/fluxer_api/src/api/worker/tasks/MessageShred.ts @@ -2,6 +2,7 @@ import type {ChannelID, MessageID} from '@app/api/BrandedTypes'; import {createChannelID, createMessageID, createUserID} from '@app/api/BrandedTypes'; +import {enqueueCrosspostSourceRemoval} from '@app/api/channel/services/message/CrosspostPropagation'; import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers'; import type {Message} from '@app/api/models/Message'; import {deleteMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup'; @@ -30,7 +31,8 @@ const STATUS_TTL_SECONDS = seconds('1 hour'); const messageShredTask: WorkerTaskHandler = async (payload, helpers) => { const data = PayloadSchema.parse(payload); helpers.logger.debug({payload: data}, 'Processing messageShred task'); - const {kvClient, channelRepository, gatewayService, storageService, purgeQueue} = getWorkerDependencies(); + const {kvClient, channelRepository, gatewayService, storageService, purgeQueue, workerService} = + getWorkerDependencies(); const progressKey = `message_shred_status:${data.job_id}`; const requestedEntries = data.entries.length; const startedAt = new Date().toISOString(); @@ -127,9 +129,7 @@ const messageShredTask: WorkerTaskHandler = async (payload, helpers) => { await Promise.all( deletionChunk.map( async ({channelId, messageId, message}: {channelId: ChannelID; messageId: MessageID; message: Message}) => { - if (message.attachments.length > 0) { - await purgeMessageAttachments(message, storageService, purgeQueue); - } + await purgeMessageAttachments(message, storageService, purgeQueue); return channelRepository.deleteMessage(channelId, messageId, authorId); }, ), @@ -139,6 +139,10 @@ const messageShredTask: WorkerTaskHandler = async (payload, helpers) => { deletionChunk.map(({messageId}) => messageId), {context: {source: 'message_shred', jobId: data.job_id}}, ); + await enqueueCrosspostSourceRemoval(workerService, { + messages: deletionChunk.map(({message}) => message), + mode: 'purge', + }); await persistStatus('in_progress'); for (const {channelId, messageId} of deletionChunk) { bulkDeleteDispatcher.track(channelId, messageId); diff --git a/fluxer_api/src/api/worker/tasks/RemoveChannelFollowers.ts b/fluxer_api/src/api/worker/tasks/RemoveChannelFollowers.ts new file mode 100644 index 000000000..b9271223d --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/RemoveChannelFollowers.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, type MessageID, type WebhookID} from '@app/api/BrandedTypes'; +import { + CrosspostTaskNames, + crosspostSyncJobKey, + RemoveChannelFollowersPayloadSchema, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import {CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants'; +import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +const removeChannelFollowers: WorkerTaskHandler = async (payload, helpers) => { + const validated = RemoveChannelFollowersPayloadSchema.parse(payload); + const {channelRepository, webhookRepository, gatewayService} = getWorkerDependencies(); + const sourceChannelId = createChannelID(BigInt(validated.sourceChannelId)); + if (validated.reason === 'converted') { + const channel = await channelRepository.findUnique(sourceChannelId); + if (channel?.type === ChannelTypes.GUILD_ANNOUNCEMENT) { + helpers.logger.info( + {sourceChannelId: validated.sourceChannelId}, + 'Keeping channel followers: the channel is an announcement channel again', + ); + return; + } + } + const pageSize = CROSSPOST_FANOUT_PAGE_SIZE; + let afterWebhookId: WebhookID | undefined; + let removed = 0; + for (;;) { + const page = await webhookRepository.listIdsBySourceChannel(sourceChannelId, {afterWebhookId, limit: pageSize}); + for (const entry of page) { + const webhook = await webhookRepository.findUnique(entry.webhookId); + await webhookRepository.delete(entry.webhookId); + removed++; + const channelId = webhook?.channelId; + if (channelId) { + await gatewayService.dispatchGuild({ + guildId: webhook.guildId ?? entry.guildId, + event: 'WEBHOOKS_UPDATE', + data: {channel_id: channelId.toString()}, + }); + } + } + if (page.length < pageSize || page.length === 0) break; + afterWebhookId = page[page.length - 1]!.webhookId; + } + let requeued = 0; + if (validated.copyMode) { + let afterMessageId: MessageID | undefined; + for (;;) { + const sources = await channelRepository.crossposts.listSourcesByChannel(sourceChannelId, { + afterMessageId, + limit: pageSize, + }); + for (const messageId of sources) { + await helpers.addJob( + CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE, + {channelId: validated.sourceChannelId, messageId: messageId.toString(), mode: validated.copyMode}, + {jobKey: crosspostSyncJobKey(messageId.toString(), validated.copyMode), skipLedger: true}, + ); + requeued++; + } + if (sources.length < pageSize || sources.length === 0) break; + afterMessageId = sources[sources.length - 1]; + } + } + helpers.logger.info( + {sourceChannelId: validated.sourceChannelId, reason: validated.reason, removed, requeued}, + 'Removed channel followers', + ); +}; + +export default removeChannelFollowers; diff --git a/fluxer_api/src/api/worker/tasks/SyncCrosspostCopies.ts b/fluxer_api/src/api/worker/tasks/SyncCrosspostCopies.ts new file mode 100644 index 000000000..1b0e83917 --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/SyncCrosspostCopies.ts @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createMessageID, createWebhookID} from '@app/api/BrandedTypes'; +import {SyncCrosspostCopiesPayloadSchema} from '@app/api/channel/services/message/CrosspostPropagation'; +import {createCrosspostDeliveryService} from '@app/api/worker/tasks/utils/CrosspostWorker'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +const syncCrosspostCopies: WorkerTaskHandler = async (payload, helpers) => { + const validated = SyncCrosspostCopiesPayloadSchema.parse(payload); + const {channelRepository} = getWorkerDependencies(); + const channelId = createChannelID(BigInt(validated.channelId)); + const messageId = createMessageID(BigInt(validated.messageId)); + const delivery = createCrosspostDeliveryService(); + const source = validated.mode === 'update' ? await delivery.loadSource(channelId, messageId) : null; + if (validated.mode === 'update' && !source) { + return; + } + const errors: Array = []; + for (const webhookId of validated.webhookIds) { + const row = await channelRepository.crossposts.get(messageId, createWebhookID(BigInt(webhookId))); + if (!row) continue; + try { + await delivery.syncCopy(row, validated.mode, source); + } catch (error) { + errors.push(error); + helpers.logger.warn( + {error, messageId: validated.messageId, webhookId, mode: validated.mode}, + 'Crosspost copy sync failed', + ); + } + } + if (errors.length > 0) { + throw errors[0]; + } +}; + +export default syncCrosspostCopies; diff --git a/fluxer_api/src/api/worker/tasks/SyncCrosspostedMessage.ts b/fluxer_api/src/api/worker/tasks/SyncCrosspostedMessage.ts new file mode 100644 index 000000000..93c43b932 --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/SyncCrosspostedMessage.ts @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createMessageID, type WebhookID} from '@app/api/BrandedTypes'; +import { + CrosspostTaskNames, + crosspostSyncBucket, + crosspostSyncChunkJobKey, + SyncCrosspostedMessagePayloadSchema, +} from '@app/api/channel/services/message/CrosspostPropagation'; +import {chunkArray, createCrosspostDeliveryService} from '@app/api/worker/tasks/utils/CrosspostWorker'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import {CROSSPOST_FANOUT_CHUNK_SIZE, CROSSPOST_FANOUT_PAGE_SIZE} from '@fluxer/constants/src/AnnouncementConstants'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; + +const syncCrosspostedMessage: WorkerTaskHandler = async (payload, helpers) => { + const validated = SyncCrosspostedMessagePayloadSchema.parse(payload); + const {channelRepository} = getWorkerDependencies(); + const channelId = createChannelID(BigInt(validated.channelId)); + const messageId = createMessageID(BigInt(validated.messageId)); + if (validated.deleteSource) { + await createCrosspostDeliveryService().deleteSourceMessage(channelId, messageId); + } + const bucketOrMode = validated.mode === 'update' ? String(crosspostSyncBucket(Date.now())) : validated.mode; + const pageSize = CROSSPOST_FANOUT_PAGE_SIZE; + let afterWebhookId: WebhookID | undefined; + for (;;) { + const rows = await channelRepository.crossposts.listBySourceMessage(messageId, {afterWebhookId, limit: pageSize}); + for (const chunk of chunkArray(rows, CROSSPOST_FANOUT_CHUNK_SIZE)) { + const webhookIds = chunk.map((row) => row.webhook_id.toString()); + await helpers.addJob( + CrosspostTaskNames.SYNC_CROSSPOST_COPIES, + {channelId: validated.channelId, messageId: validated.messageId, mode: validated.mode, webhookIds}, + { + jobKey: crosspostSyncChunkJobKey({ + messageId: validated.messageId, + mode: validated.mode, + bucketOrMode, + firstWebhookId: webhookIds[0]!, + }), + skipLedger: true, + }, + ); + } + if (rows.length < pageSize || rows.length === 0) break; + afterWebhookId = rows[rows.length - 1]!.webhook_id; + } + if (validated.mode !== 'update') { + await channelRepository.crossposts.deleteSource({sourceChannelId: channelId, sourceMessageId: messageId}); + } +}; + +export default syncCrosspostedMessage; diff --git a/fluxer_api/src/api/worker/tasks/admin_bulk/BulkDeleteMessagesForUsers.ts b/fluxer_api/src/api/worker/tasks/admin_bulk/BulkDeleteMessagesForUsers.ts index ddc25dd66..d8ddd8163 100644 --- a/fluxer_api/src/api/worker/tasks/admin_bulk/BulkDeleteMessagesForUsers.ts +++ b/fluxer_api/src/api/worker/tasks/admin_bulk/BulkDeleteMessagesForUsers.ts @@ -26,6 +26,7 @@ const handler: WorkerTaskHandler = async (rawPayload, helpers) => { gatewayService: deps.gatewayService, storageService: deps.storageService, purgeQueue: deps.purgeQueue, + workerService: deps.workerService, }); const adminUserId = createUserID(BigInt(payload.admin_user_id)); const total = payload.user_ids.length; diff --git a/fluxer_api/src/api/worker/tasks/utils/CrosspostWorker.ts b/fluxer_api/src/api/worker/tasks/utils/CrosspostWorker.ts new file mode 100644 index 000000000..0a84e464a --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/utils/CrosspostWorker.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {CrosspostDeliveryService} from '@app/api/channel/services/message/CrosspostDeliveryService'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; + +export function createCrosspostDeliveryService(): CrosspostDeliveryService { + const deps = getWorkerDependencies(); + return new CrosspostDeliveryService({ + channelRepository: deps.channelRepository, + webhookRepository: deps.webhookRepository, + userRepository: deps.userRepository, + guildRepository: deps.guildRepository, + gatewayService: deps.gatewayService, + storageService: deps.storageService, + avatarService: deps.avatarService, + purgeQueue: deps.purgeQueue, + snowflakeService: deps.snowflakeService, + cacheService: deps.cacheService, + limitConfigService: deps.limitConfigService, + persistenceService: deps.channelService.messages.persistence, + searchService: deps.channelService.messages.search, + }); +} + +export async function mapWithConcurrency( + items: ReadonlyArray, + concurrency: number, + fn: (item: T) => Promise, +): Promise> { + const results: Array = new Array(items.length); + let nextIndex = 0; + const workers = Array.from({length: Math.min(concurrency, items.length)}, async () => { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await fn(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + +export function chunkArray(items: ReadonlyArray, size: number): Array> { + const chunks: Array> = []; + for (let index = 0; index < items.length; index += size) { + chunks.push(items.slice(index, index + size)); + } + return chunks; +} diff --git a/fluxer_api/src/api/worker/tests/CrosspostTasks.test.ts b/fluxer_api/src/api/worker/tests/CrosspostTasks.test.ts new file mode 100644 index 000000000..48d03d10b --- /dev/null +++ b/fluxer_api/src/api/worker/tests/CrosspostTasks.test.ts @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createChannelID, createGuildID, createMessageID, createWebhookID} from '@app/api/BrandedTypes'; +import {ChannelRepository} from '@app/api/channel/ChannelRepository'; +import {CrosspostTaskNames} from '@app/api/channel/services/message/CrosspostPropagation'; +import {disableCrosspostWorker, follow} from '@app/api/channel/tests/AnnouncementTestUtils'; +import { + type FanoutWorld, + followInto, + patchChannelRow, + postAndPublish, + setupFanoutWorld, + sourceIndex, + workerHelpers, +} from '@app/api/channel/tests/CrosspostWorkerTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopGatewayService} from '@app/api/test/NoopGatewayService'; +import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService'; +import {WebhookRepository} from '@app/api/webhook/WebhookRepository'; +import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage'; +import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk'; +import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers'; +import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies'; +import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage'; +import { + findLaneForTask, + resolveWorkerLanes, + validateLaneCompleteness, + WORKER_LANES, +} from '@app/api/worker/WorkerLaneConfig'; +import {workerTasks} from '@app/api/worker/WorkerTaskRegistry'; +import {ChannelTypes, WebhookTypes} from '@fluxer/constants/src/ChannelConstants'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; +import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; +import {ZodError} from 'zod'; + +const pageSizes = vi.hoisted(() => ({chunk: null as number | null, page: null as number | null})); + +vi.mock('@fluxer/constants/src/AnnouncementConstants', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + get CROSSPOST_FANOUT_CHUNK_SIZE() { + return pageSizes.chunk ?? actual.CROSSPOST_FANOUT_CHUNK_SIZE; + }, + get CROSSPOST_FANOUT_PAGE_SIZE() { + return pageSizes.page ?? actual.CROSSPOST_FANOUT_PAGE_SIZE; + }, + }; +}); + +const CROSSPOST_TASKS = Object.values(CrosspostTaskNames); + +describe('crosspost worker lane', () => { + test('every crosspost task runs in the crosspost lane', () => { + for (const task of CROSSPOST_TASKS) { + expect(findLaneForTask(task)).toBe('crosspost'); + expect(workerTasks).toHaveProperty(task); + } + const lane = WORKER_LANES.find((entry) => entry.name === 'crosspost'); + expect(lane).toMatchObject({ + consumerName: 'workers_crosspost', + concurrency: 8, + maxAckPending: 64, + ackWaitMs: 60000, + maxDeliver: 25, + }); + expect([...lane!.taskTypes].sort()).toEqual([...CROSSPOST_TASKS].sort()); + expect(() => validateLaneCompleteness(workerTasks)).not.toThrow(); + }); + + test('the lane resolves on its own and takes a concurrency override', () => { + const [lane] = resolveWorkerLanes({ + mode: 'single_lane', + laneName: 'crosspost', + laneConcurrencyOverrides: {crosspost: 3}, + }); + expect(lane!.name).toBe('crosspost'); + expect(lane!.concurrency).toBe(3); + const [single] = resolveWorkerLanes({ + mode: 'single_task', + taskName: 'crosspostMessageChunk', + laneConcurrencyOverrides: {}, + }); + expect(single!.name).toBe('crosspost'); + expect(single!.taskTypes).toEqual(['crosspostMessageChunk']); + }); +}); + +describe('crosspost task payloads', () => { + const cases: Array<[string, WorkerTaskHandler, Record]> = [ + ['crosspostMessage', crosspostMessage, {channelId: 1}], + ['crosspostMessageChunk', crosspostMessageChunk, {channelId: '1', messageId: '2', webhookIds: [], attempt: 0}], + ['crosspostMessageChunk', crosspostMessageChunk, {channelId: '1', messageId: '2', webhookIds: ['3'], attempt: -1}], + ['syncCrosspostedMessage', syncCrosspostedMessage, {channelId: '1', messageId: '2', mode: 'rewrite'}], + ['syncCrosspostCopies', syncCrosspostCopies, {channelId: '1', messageId: '2', mode: 'update', webhookIds: []}], + ['removeChannelFollowers', removeChannelFollowers, {sourceChannelId: '1', reason: 'archived'}], + ['removeChannelFollowers', removeChannelFollowers, {sourceChannelId: '1', reason: 'deleted', copyMode: 'update'}], + ]; + + test.each(cases)('%s rejects a malformed payload', async (_name, handler, payload) => { + const worker = new SyncTaskWorkerService({}, {deferred: true}); + await expect(handler(payload, workerHelpers(worker))).rejects.toBeInstanceOf(ZodError); + }); +}); + +describe('deferred SyncTaskWorkerService', () => { + test('nested jobs go through the same queue and job keys dedupe', async () => { + const seen: Array = []; + const worker = new SyncTaskWorkerService( + { + parent: async (payload, helpers) => { + seen.push(`parent:${payload.n}`); + await helpers.addJob('child', {n: payload.n}, {jobKey: 'child-key'}); + await helpers.addJob('child', {n: 99}, {jobKey: 'child-key'}); + }, + child: async (payload) => { + seen.push(`child:${payload.n}`); + }, + }, + {deferred: true}, + ); + await worker.addJob('parent', {n: 1}, {jobKey: 'parent-key', runAt: new Date(Date.now() + 60_000)}); + await worker.addJob('parent', {n: 2}, {jobKey: 'parent-key'}); + expect(seen).toEqual([]); + expect(worker.pending).toHaveLength(1); + const result = await worker.drain(); + expect(result).toEqual({ran: 2, failed: 0, remaining: 0}); + expect(seen).toEqual(['parent:1', 'child:1']); + expect(worker.recorded.map((job) => job.jobKey)).toEqual(['parent-key', 'child-key']); + expect(worker.recorded[0]!.runAt).toBeInstanceOf(Date); + expect(worker.deduped.map((job) => job.payload)).toEqual([{n: 2}, {n: 99}]); + await worker.addJob('parent', {n: 3}, {jobKey: 'parent-key'}); + await worker.drain(); + expect(seen.at(-1)).toBe('child:3'); + }); + + test('a failing job retries and then lands in dead letters', async () => { + let flakyCalls = 0; + let onceCalls = 0; + const worker = new SyncTaskWorkerService( + { + flaky: async () => { + flakyCalls++; + throw new Error('always fails'); + }, + once: async () => { + onceCalls++; + if (onceCalls === 1) throw new Error('fails once'); + }, + }, + {deferred: true}, + ); + await worker.addJob('flaky', {}); + await worker.addJob('once', {}); + const result = await worker.drain({maxAttempts: 3}); + expect(result.failed).toBe(4); + expect(flakyCalls).toBe(3); + expect(onceCalls).toBe(2); + expect(worker.deadLetters).toHaveLength(1); + expect(worker.deadLetters[0]!.taskType).toBe('flaky'); + expect(worker.deadLetters[0]!.attempts).toBe(3); + }); + + test('drain stops after maxJobs and resumes later', async () => { + const ran: Array = []; + const worker = new SyncTaskWorkerService( + {step: async (payload) => void ran.push(payload.n as number)}, + {deferred: true}, + ); + for (const n of [1, 2, 3]) { + await worker.addJob('step', {n}); + } + expect(await worker.drain({maxJobs: 2})).toEqual({ran: 2, failed: 0, remaining: 1}); + expect(ran).toEqual([1, 2]); + await worker.drain(); + expect(ran).toEqual([1, 2, 3]); + }); + + test('the default mode still runs inline and drops nested jobs', async () => { + const seen: Array = []; + const worker = new SyncTaskWorkerService({ + parent: async (_payload, helpers) => { + seen.push('parent'); + expect(await helpers.addJob('child', {})).toBe(0n); + }, + child: async () => { + seen.push('child'); + }, + }); + await worker.addJob('parent', {}, {jobKey: 'k'}); + await worker.addJob('parent', {}, {jobKey: 'k'}); + expect(seen).toEqual(['parent', 'parent']); + expect(worker.recorded).toHaveLength(0); + }); +}); + +describe('crosspost tasks against storage', () => { + let harness: ApiTestHarness; + let world: FanoutWorld; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + world = await setupFanoutWorld(harness); + }); + + afterEach(() => { + pageSizes.chunk = null; + pageSizes.page = null; + disableCrosspostWorker(); + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + async function followerIds(): Promise> { + const entries = await new WebhookRepository().listIdsBySourceChannel(createChannelID(BigInt(world.a.ann.id)), { + limit: 100, + }); + return entries.map((entry) => entry.webhookId.toString()); + } + + test('removeChannelFollowers removes every follower once and announces each target', async () => { + await followInto(harness, world, world.b.t1.id); + await followInto(harness, world, world.b.t2.id); + await patchChannelRow(world.a.ann.id, {type: ChannelTypes.GUILD_TEXT}); + const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild'); + const payload = {sourceChannelId: world.a.ann.id, reason: 'converted'}; + await removeChannelFollowers(payload, workerHelpers(world.worker)); + expect(await followerIds()).toHaveLength(0); + const updates = dispatchSpy.mock.calls + .map(([params]) => params) + .filter((params) => params.event === 'WEBHOOKS_UPDATE'); + expect(updates.map((params) => (params.data as {channel_id: string}).channel_id).sort()).toEqual( + [world.b.t1.id, world.b.t2.id].sort(), + ); + expect(updates.every((params) => params.guildId.toString() === world.b.guild.id)).toBe(true); + dispatchSpy.mockClear(); + await removeChannelFollowers(payload, workerHelpers(world.worker)); + expect(dispatchSpy.mock.calls.filter(([params]) => params.event === 'WEBHOOKS_UPDATE')).toHaveLength(0); + const webhooks = await new WebhookRepository().listByChannel(createChannelID(BigInt(world.b.t1.id))); + expect(webhooks.filter((webhook) => webhook.type === WebhookTypes.CHANNEL_FOLLOWER)).toHaveLength(0); + }); + + test('removeChannelFollowers keeps followers when the channel is an announcement channel again', async () => { + await followInto(harness, world, world.b.t1.id); + await removeChannelFollowers({sourceChannelId: world.a.ann.id, reason: 'converted'}, workerHelpers(world.worker)); + expect(await followerIds()).toHaveLength(1); + }); + + test('removeChannelFollowers with a copy mode queues one sync per published message, across pages', async () => { + await followInto(harness, world, world.b.t1.id); + const published = []; + for (const content of ['one', 'two', 'three']) { + published.push(await postAndPublish(harness, world, {content})); + } + expect((await sourceIndex(world.a.ann.id)).sort()).toEqual(published.map((message) => message.id).sort()); + pageSizes.page = 2; + const before = world.worker.recorded.length; + await removeChannelFollowers( + {sourceChannelId: world.a.ann.id, reason: 'deleted', copyMode: 'purge'}, + workerHelpers(world.worker), + ); + const queued = world.worker.recorded.slice(before); + expect(queued.every((job) => job.taskType === CrosspostTaskNames.SYNC_CROSSPOSTED_MESSAGE)).toBe(true); + expect(queued.map((job) => (job.payload as {messageId: string}).messageId).sort()).toEqual( + published.map((message) => message.id).sort(), + ); + expect(queued.map((job) => job.payload.mode)).toEqual(['purge', 'purge', 'purge']); + expect(queued.map((job) => job.jobKey).sort()).toEqual( + published.map((message) => `crosspost-sync:${message.id}:purge`).sort(), + ); + expect(await followerIds()).toHaveLength(0); + }); + + test('syncCrosspostedMessage pages through every mapping row', async () => { + const source = await postAndPublish(harness, world, {content: 'many followers'}); + const repository = new ChannelRepository(); + const webhookIds = [ + '900000000000000001', + '900000000000000002', + '900000000000000003', + '900000000000000004', + '900000000000000005', + ]; + for (const webhookId of webhookIds) { + await repository.crossposts.insertPending({ + source_message_id: createMessageID(BigInt(source.id)), + webhook_id: createWebhookID(BigInt(webhookId)), + source_channel_id: createChannelID(BigInt(world.a.ann.id)), + target_guild_id: createGuildID(BigInt(world.b.guild.id)), + target_channel_id: createChannelID(BigInt(world.b.t1.id)), + target_message_id: createMessageID(BigInt(webhookId)), + state: 'pending', + reserved_at: new Date(Date.now() - 120_000), + source_fingerprint: null, + created_at: new Date(), + }); + } + pageSizes.page = 2; + pageSizes.chunk = 1; + const before = world.worker.recorded.length; + await syncCrosspostedMessage( + {channelId: world.a.ann.id, messageId: source.id, mode: 'source_deleted'}, + workerHelpers(world.worker), + ); + const chunks = world.worker.recorded.slice(before); + expect(chunks.map((job) => job.taskType)).toEqual(Array(5).fill(CrosspostTaskNames.SYNC_CROSSPOST_COPIES)); + expect(chunks.map((job) => (job.payload as {webhookIds: Array}).webhookIds)).toEqual( + webhookIds.map((webhookId) => [webhookId]), + ); + expect(chunks.map((job) => job.jobKey)).toEqual( + webhookIds.map((webhookId) => `crosspost-sync-chunk:${source.id}:source_deleted:source_deleted:${webhookId}`), + ); + expect(await sourceIndex(world.a.ann.id)).not.toContain(source.id); + await world.worker.drain(); + expect(world.worker.deadLetters).toHaveLength(0); + const remaining = await repository.crossposts.listBySourceMessage(createMessageID(BigInt(source.id)), { + limit: 100, + }); + expect(remaining).toHaveLength(0); + }); + + test('crosspostMessage does nothing for a message that is not published', async () => { + await follow(harness, world.b.owner.token, world.a.ann.id, world.b.t1.id); + await world.worker.drain(); + const before = world.worker.recorded.length; + await crosspostMessage({channelId: world.a.ann.id, messageId: '123456789012345678'}, workerHelpers(world.worker)); + expect(world.worker.recorded.length).toBe(before); + }); +}); diff --git a/fluxer_api/src/api/worker/tests/WorkerLaneConfig.test.ts b/fluxer_api/src/api/worker/tests/WorkerLaneConfig.test.ts index 0b838b219..81ed15ce8 100644 --- a/fluxer_api/src/api/worker/tests/WorkerLaneConfig.test.ts +++ b/fluxer_api/src/api/worker/tests/WorkerLaneConfig.test.ts @@ -72,6 +72,7 @@ describe('WorkerLaneConfig', () => { unfurl: [], lifecycle: ['sendScheduledMessage'], batch: [], + crosspost: [], }); for (const lane of lanes) { for (const task of lane.retiredTaskTypes) expect(lane.taskTypes).not.toContain(task); diff --git a/fluxer_app/src/features/app/components/alerts/RateLimitedConfirmModal.tsx b/fluxer_app/src/features/app/components/alerts/RateLimitedConfirmModal.tsx index b281aa671..a9fc68719 100644 --- a/fluxer_app/src/features/app/components/alerts/RateLimitedConfirmModal.tsx +++ b/fluxer_app/src/features/app/components/alerts/RateLimitedConfirmModal.tsx @@ -27,10 +27,11 @@ interface RateLimitedConfirmModalProps { retryAfter?: number; onRetry?: () => void; hideCloseButton?: boolean; + describe?: (duration: string | undefined) => string; } export const RateLimitedConfirmModal = observer( - ({title, retryAfter, onRetry, hideCloseButton}: RateLimitedConfirmModalProps) => { + ({title, retryAfter, onRetry, hideCloseButton, describe}: RateLimitedConfirmModalProps) => { const {i18n} = useLingui(); const hasRetryAfter = retryAfter != null; const formatRateLimitTime = (totalSeconds: number): string => { @@ -48,11 +49,13 @@ export const RateLimitedConfirmModal = observer( - channel.type === ChannelTypes.GUILD_TEXT || channel.type === ChannelTypes.GUILD_LINK; + channel.type === ChannelTypes.GUILD_TEXT || + channel.type === ChannelTypes.GUILD_ANNOUNCEMENT || + channel.type === ChannelTypes.GUILD_LINK; const isCategoryChannel = (channel: Channel) => channel.type === ChannelTypes.GUILD_CATEGORY; const gatherCategoryBlock = (channels: ReadonlyArray, categoryId: string) => { return channels.filter((ch) => ch.id === categoryId || ch.parentId === categoryId); diff --git a/fluxer_app/src/features/app/components/layout/utils/ChannelOrganization.ts b/fluxer_app/src/features/app/components/layout/utils/ChannelOrganization.ts index f0d9d9568..5f11e4c1d 100644 --- a/fluxer_app/src/features/app/components/layout/utils/ChannelOrganization.ts +++ b/fluxer_app/src/features/app/components/layout/utils/ChannelOrganization.ts @@ -5,7 +5,9 @@ import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; import {compareChannelOrdering} from '@fluxer/schema/src/domains/channel/GuildChannelOrdering'; export const isTextChannel = (ch: Channel) => - ch.type === ChannelTypes.GUILD_TEXT || ch.type === ChannelTypes.GUILD_LINK; + ch.type === ChannelTypes.GUILD_TEXT || + ch.type === ChannelTypes.GUILD_ANNOUNCEMENT || + ch.type === ChannelTypes.GUILD_LINK; const isVoiceChannel = (ch: Channel) => ch.type === ChannelTypes.GUILD_VOICE; export const isCategory = (ch: Channel) => ch.type === ChannelTypes.GUILD_CATEGORY; diff --git a/fluxer_app/src/features/app/constants/Endpoints.ts b/fluxer_app/src/features/app/constants/Endpoints.ts index 17b99e8e1..20b28d91f 100644 --- a/fluxer_app/src/features/app/constants/Endpoints.ts +++ b/fluxer_app/src/features/app/constants/Endpoints.ts @@ -68,6 +68,10 @@ export const Endpoints = { CHANNEL_MESSAGE_ATTACHMENT: (channelId: string, messageId: string, attachmentId: string) => `/channels/${channelId}/messages/${messageId}/attachments/${attachmentId}`, CHANNEL_MESSAGE_ACK: (channelId: string, messageId: string) => `/channels/${channelId}/messages/${messageId}/ack`, + CHANNEL_MESSAGE_CROSSPOST: (channelId: string, messageId: string) => + `/channels/${channelId}/messages/${messageId}/crosspost`, + CHANNEL_MESSAGE_CROSSPOST_SOURCE: (channelId: string, messageId: string) => + `/channels/${channelId}/messages/${messageId}/crosspost-source`, CHANNEL_MESSAGE_REACTION: (channelId: string, messageId: string, emoji: string) => `/channels/${channelId}/messages/${messageId}/reactions/${emoji}`, CHANNEL_MESSAGE_REACTION_USERS: (channelId: string, messageId: string, emoji: string) => @@ -82,6 +86,8 @@ export const Endpoints = { CHANNEL_PINS_ACK: (channelId: string) => `/channels/${channelId}/pins/ack`, CHANNEL_TYPING: (channelId: string) => `/channels/${channelId}/typing`, CHANNEL_WEBHOOKS: (channelId: string) => `/channels/${channelId}/webhooks`, + CHANNEL_FOLLOWERS: (channelId: string) => `/channels/${channelId}/followers`, + CHANNEL_FOLLOWER_STATS: (channelId: string) => `/channels/${channelId}/follower-stats`, CHANNEL_RTC_REGIONS: (channelId: string) => `/channels/${channelId}/rtc-regions`, CHANNEL_SLOWMODE: (channelId: string) => `/channels/${channelId}/slowmode`, CHANNEL_CALL: (channelId: string) => `/channels/${channelId}/call`, diff --git a/fluxer_app/src/features/channel/commands/ChannelCommands.ts b/fluxer_app/src/features/channel/commands/ChannelCommands.ts index c39ec91cb..6adb62e2c 100644 --- a/fluxer_app/src/features/channel/commands/ChannelCommands.ts +++ b/fluxer_app/src/features/channel/commands/ChannelCommands.ts @@ -8,6 +8,7 @@ import {Logger} from '@app/features/platform/utils/AppLogger'; import Slowmode from '@app/features/slowmode/state/Slowmode'; import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; import {CHANNEL_RATE_LIMIT_PER_USER_MAX} from '@fluxer/constants/src/LimitConstants'; +import type {ChannelFollowerStatsResponse} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; import type {Channel, ChannelSlowmodeStateResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; import type {Invite} from '@fluxer/schema/src/domains/invite/InviteSchemas'; @@ -31,6 +32,7 @@ type ChannelUpdateParams = Partial< Pick< Channel, | 'name' + | 'type' | 'topic' | 'url' | 'nsfw' @@ -155,6 +157,11 @@ export async function update(channelId: string, params: ChannelUpdateParams): Pr } } +export async function fetchFollowerStats(channelId: string): Promise { + const response = await http.get(Endpoints.CHANNEL_FOLLOWER_STATS(channelId)); + return response.body; +} + export async function updateGroupDMNickname( channelId: string, userId: string, diff --git a/fluxer_app/src/features/channel/commands/ChannelFollowCommands.ts b/fluxer_app/src/features/channel/commands/ChannelFollowCommands.ts new file mode 100644 index 000000000..8a2a21e6c --- /dev/null +++ b/fluxer_app/src/features/channel/commands/ChannelFollowCommands.ts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Endpoints} from '@app/features/app/constants/Endpoints'; +import {http} from '@app/features/platform/transport/RestTransport'; +import {Logger} from '@app/features/platform/utils/AppLogger'; +import type {FollowedChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelFollowSchemas'; + +const logger = new Logger('ChannelFollowCommands'); + +export async function followChannel( + sourceChannelId: string, + targetChannelId: string, +): Promise { + try { + const response = await http.post(Endpoints.CHANNEL_FOLLOWERS(sourceChannelId), { + body: {webhook_channel_id: targetChannelId}, + }); + return response.body; + } catch (error) { + logger.error(`Failed to follow channel ${sourceChannelId} into ${targetChannelId}:`, error); + throw error; + } +} diff --git a/fluxer_app/src/features/channel/commands/CrosspostSourceCommands.ts b/fluxer_app/src/features/channel/commands/CrosspostSourceCommands.ts new file mode 100644 index 000000000..91828609d --- /dev/null +++ b/fluxer_app/src/features/channel/commands/CrosspostSourceCommands.ts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Endpoints} from '@app/features/app/constants/Endpoints'; +import {http} from '@app/features/platform/transport/RestTransport'; +import {Logger} from '@app/features/platform/utils/AppLogger'; +import type {CrosspostSourceResponse} from '@fluxer/schema/src/domains/message/CrosspostSourceSchemas'; + +const logger = new Logger('CrosspostSourceCommands'); + +export async function fetchCrosspostSource(channelId: string, messageId: string): Promise { + try { + const response = await http.get( + Endpoints.CHANNEL_MESSAGE_CROSSPOST_SOURCE(channelId, messageId), + ); + return response.body; + } catch (error) { + logger.error(`Failed to fetch crosspost source for message ${messageId} in ${channelId}:`, error); + throw error; + } +} diff --git a/fluxer_app/src/features/channel/components/ChannelFollowAddMessage.tsx b/fluxer_app/src/features/channel/components/ChannelFollowAddMessage.tsx new file mode 100644 index 000000000..f7c6cb094 --- /dev/null +++ b/fluxer_app/src/features/channel/components/ChannelFollowAddMessage.tsx @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {CrosspostCommunityPopout} from '@app/features/channel/components/CrosspostCommunityPopout'; +import joinStyles from '@app/features/channel/components/GuildJoinMessage.module.css'; +import {SystemMessage} from '@app/features/channel/components/SystemMessage'; +import {SystemMessageUsername} from '@app/features/channel/components/SystemMessageUsername'; +import {useSystemMessageData} from '@app/features/messaging/hooks/useSystemMessageData'; +import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import styles from '@app/features/theme/styles/Message.module.css'; +import FocusRing from '@app/features/ui/focus_ring/FocusRing'; +import {Trans} from '@lingui/react/macro'; +import {ArrowRightIcon} from '@phosphor-icons/react'; +import {observer} from 'mobx-react-lite'; + +interface ChannelFollowAddMessageProps { + message: Message; +} + +export const ChannelFollowAddMessage = observer(({message}: ChannelFollowAddMessageProps) => { + const {author, guild} = useSystemMessageData(message); + const sourceComponent = ( + + + + + + ); + const messageContent = ( + + {' '} + followed {sourceComponent} into this channel. Messages published there will appear here. + + ); + return ( + + ); +}); diff --git a/fluxer_app/src/features/channel/components/ChannelHeader.module.css b/fluxer_app/src/features/channel/components/ChannelHeader.module.css index 401d38a23..8dc15c8e6 100644 --- a/fluxer_app/src/features/channel/components/ChannelHeader.module.css +++ b/fluxer_app/src/features/channel/components/ChannelHeader.module.css @@ -386,6 +386,11 @@ overflow: hidden; } +.followButton { + margin-left: var(--channel-header-name-gap); + flex-shrink: 0; +} + .topicDivider { margin: 0 var(--channel-header-topic-divider-gap); flex-shrink: 0; diff --git a/fluxer_app/src/features/channel/components/ChannelHeader.tsx b/fluxer_app/src/features/channel/components/ChannelHeader.tsx index adfea847e..8eef32140 100644 --- a/fluxer_app/src/features/channel/components/ChannelHeader.tsx +++ b/fluxer_app/src/features/channel/components/ChannelHeader.tsx @@ -39,6 +39,7 @@ import { } from '@app/features/channel/components/channel_header/shared'; import {useChannelHeaderData} from '@app/features/channel/components/channel_header/useChannelHeaderData'; import {CallButtons} from '@app/features/channel/components/channel_header_components/CallButtons'; +import {ChannelFollowButton} from '@app/features/channel/components/channel_header_components/ChannelFollowButton'; import {ChannelHeaderIcon} from '@app/features/channel/components/channel_header_components/ChannelHeaderIcon'; import {ChannelNotificationSettingsButton} from '@app/features/channel/components/channel_header_components/ChannelNotificationSettingsButton'; import {ChannelPinsButton} from '@app/features/channel/components/channel_header_components/ChannelPinsButton'; @@ -860,6 +861,13 @@ export const ChannelHeader = observer( {channelName} + {!isMobile && ( + + )} {channel.topic && ( <> diff --git a/fluxer_app/src/features/channel/components/ChannelMessage.tsx b/fluxer_app/src/features/channel/components/ChannelMessage.tsx index c80cb833a..b9c23e435 100644 --- a/fluxer_app/src/features/channel/components/ChannelMessage.tsx +++ b/fluxer_app/src/features/channel/components/ChannelMessage.tsx @@ -569,6 +569,7 @@ export const Message: React.FC = observer((props) => { canDeleteAttachment: true, canPinMessage: true, canForwardMessage: true, + canCrosspostMessage: false, canSuppressEmbeds: true, shouldRenderSuppressEmbeds: false, } @@ -716,7 +717,11 @@ export const Message: React.FC = observer((props) => { data-flx-author-self={message.author.id === Users.currentUserId ? 'true' : undefined} data-flx-author-bot={message.author.bot ? 'true' : undefined} data-flx-author-webhook={message.webhookId != null ? 'true' : undefined} - data-flx-reply={message.type === MessageTypes.REPLY || message.messageReference != null ? 'true' : undefined} + data-flx-reply={ + message.type === MessageTypes.REPLY || (message.messageReference != null && !message.isCrosspostCopy) + ? 'true' + : undefined + } data-flx-blocked={message.blocked ? 'true' : undefined} data-flx-pinned={message.pinned ? 'true' : undefined} data-flx-call={message.type === MessageTypes.CALL ? 'true' : undefined} @@ -764,6 +769,7 @@ export const Message: React.FC = observer((props) => { canDeleteMessage: true, canPinMessage: true, canForwardMessage: true, + canCrosspostMessage: false, shouldRenderSuppressEmbeds: true, }} developerMode={false} diff --git a/fluxer_app/src/features/channel/components/ChannelMessages.module.css b/fluxer_app/src/features/channel/components/ChannelMessages.module.css index 99aced3f6..a4f809fc3 100644 --- a/fluxer_app/src/features/channel/components/ChannelMessages.module.css +++ b/fluxer_app/src/features/channel/components/ChannelMessages.module.css @@ -31,13 +31,11 @@ inset-block-end: 0; inset-inline: 0 1rem; z-index: 1; - height: 1rem; - background: linear-gradient(to bottom, transparent, var(--messages-bottom-fade-fill)); pointer-events: none; } .bottomFadeStatusAtBottom { - height: var(--composer-status-safe-area); + height: 1rem; background: linear-gradient( to bottom, transparent, @@ -47,7 +45,7 @@ } .bottomFadeStatusScrolled { - height: calc(var(--composer-status-safe-area) + 4rem); + height: 5rem; background: linear-gradient( to bottom, transparent, diff --git a/fluxer_app/src/features/channel/components/ChannelMessages.tsx b/fluxer_app/src/features/channel/components/ChannelMessages.tsx index d20d46478..654e5fd2c 100644 --- a/fluxer_app/src/features/channel/components/ChannelMessages.tsx +++ b/fluxer_app/src/features/channel/components/ChannelMessages.tsx @@ -745,15 +745,16 @@ export const Messages = observer(function Messages({ - diff --git a/fluxer_app/src/features/channel/components/ChannelUserTag.tsx b/fluxer_app/src/features/channel/components/ChannelUserTag.tsx index 90d0d5023..0d29f7973 100644 --- a/fluxer_app/src/features/channel/components/ChannelUserTag.tsx +++ b/fluxer_app/src/features/channel/components/ChannelUserTag.tsx @@ -8,11 +8,27 @@ import React from 'react'; interface UserTagProps extends React.ComponentPropsWithoutRef<'span'> { className?: string; system?: boolean; + variant?: 'bot' | 'system' | 'community'; size?: 'sm' | 'lg'; } +function renderTagLabel(variant: 'bot' | 'system' | 'community') { + switch (variant) { + case 'system': + return System; + case 'community': + return ( + + Community + + ); + default: + return Bot; + } +} + export const UserTag = React.forwardRef( - ({className, system, size = 'sm', ...props}, ref) => { + ({className, system, variant, size = 'sm', ...props}, ref) => { return ( ( className={clsx(styles.text, size === 'lg' ? styles.textLg : styles.textSm)} data-flx="channel.user-tag.text" > - {system ? System : Bot} + {renderTagLabel(variant ?? (system ? 'system' : 'bot'))} ); diff --git a/fluxer_app/src/features/channel/components/CompactMessageLayout.tsx b/fluxer_app/src/features/channel/components/CompactMessageLayout.tsx index e08a04e4a..57c6d6809 100644 --- a/fluxer_app/src/features/channel/components/CompactMessageLayout.tsx +++ b/fluxer_app/src/features/channel/components/CompactMessageLayout.tsx @@ -176,10 +176,11 @@ export function CompactAuthorPrefix({ {' '} - {author.bot && ( + {(author.bot || message.isCrosspostCopy) && ( )} diff --git a/fluxer_app/src/features/channel/components/CrosspostCommunityCard.module.css b/fluxer_app/src/features/channel/components/CrosspostCommunityCard.module.css new file mode 100644 index 000000000..13b4acbf9 --- /dev/null +++ b/fluxer_app/src/features/channel/components/CrosspostCommunityCard.module.css @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: AGPL-3.0-or-later */ + +.card { + --crosspost-card-surface: var(--background-secondary); + --crosspost-card-banner-height: 7.5rem; + position: relative; + display: flex; + flex-direction: column; + width: 18.75rem; + max-width: 100%; + overflow: hidden; + background: var(--crosspost-card-surface); + color: var(--text-primary); + border: 0.0625rem solid var(--background-modifier-accent); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-xl); +} + +.sheetCard { + width: 100%; + border: none; + border-radius: 0; + box-shadow: none; +} + +.banner { + position: relative; + display: block; + height: var(--crosspost-card-banner-height); + flex: 0 0 auto; + overflow: hidden; + background-color: var(--background-tertiary); +} + +.bannerImage { + position: absolute; + inset: 0; + display: block; + background-position: center center; + background-repeat: no-repeat; + background-size: cover; +} + +.bannerScrim { + position: absolute; + inset: 0; + display: block; + background: linear-gradient(to bottom, rgba(0, 0, 0, 0.04) 0%, rgba(0, 0, 0, 0.42) 100%); +} + +.icon { + position: absolute; + top: calc(var(--crosspost-card-banner-height) - 1.75rem); + left: var(--spacing-4); + z-index: 1; + box-shadow: 0 0 0 0.1875rem var(--crosspost-card-surface); + pointer-events: none; +} + +.body { + display: flex; + flex-direction: column; + gap: var(--spacing-2); + padding: var(--spacing-10) var(--spacing-4) var(--spacing-4); + min-width: 0; +} + +.titleRow { + display: flex; + align-items: center; + gap: 0.25rem; + height: 1.25rem; + min-width: 0; +} + +.nameText { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 1rem; + font-weight: 600; + line-height: 1.25; + color: var(--text-primary); +} + +.description { + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 4; + overflow: hidden; + font-size: 0.875rem; + line-height: 1.5; + color: var(--text-secondary); + overflow-wrap: anywhere; +} + +.footer { + display: flex; + align-items: center; + gap: var(--spacing-3); + flex: 0 0 auto; + min-width: 0; + padding: var(--spacing-3) var(--spacing-4); + border-top: 0.0625rem solid var(--background-modifier-accent); +} + +.stat { + display: flex; + align-items: center; + gap: var(--count-indicator-gap); + flex: 0 1 auto; + height: 0.9rem; + min-width: 0; +} + +.statDot { + height: 0.5rem; + width: 0.5rem; + border-radius: var(--radius-full); + flex: 0 0 auto; +} + +.statDotOnline { + composes: statDot; + background-color: var(--status-online); +} + +.statDotMembers { + composes: statDot; + background-color: var(--text-tertiary-secondary); +} + +.statText { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--text-tertiary); + font-size: 0.75rem; + line-height: 1.2; +} + +.notice { + color: var(--text-tertiary); + font-size: 0.875rem; + line-height: 1.3; +} + +.action { + padding: 0 var(--spacing-4) var(--spacing-4); +} + +.loading { + display: flex; + justify-content: center; + padding-block: var(--spacing-1); +} diff --git a/fluxer_app/src/features/channel/components/CrosspostCommunityCard.tsx b/fluxer_app/src/features/channel/components/CrosspostCommunityCard.tsx new file mode 100644 index 000000000..5435f4a0b --- /dev/null +++ b/fluxer_app/src/features/channel/components/CrosspostCommunityCard.tsx @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import RuntimeConfig from '@app/features/app/state/RuntimeConfig'; +import styles from '@app/features/channel/components/CrosspostCommunityCard.module.css'; +import CrosspostSourceCards from '@app/features/channel/state/CrosspostSourceCards'; +import {getCrosspostSourceDisplayName, getCrosspostSourceGuildId} from '@app/features/channel/utils/CrosspostUtils'; +import {joinDiscoveryGuild} from '@app/features/discovery/commands/DiscoveryJoinCommands'; +import {resolveBannerTintClassName} from '@app/features/discovery/utils/DiscoveryBannerTint'; +import {GuildBadge} from '@app/features/guild/components/GuildBadge'; +import {GuildIcon} from '@app/features/guild/components/popouts/GuildIcon'; +import GuildCount from '@app/features/guild/state/GuildCount'; +import Guilds from '@app/features/guild/state/Guilds'; +import {JOIN_COMMUNITY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; +import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import * as NavigationCommands from '@app/features/navigation/commands/NavigationCommands'; +import {Button} from '@app/features/ui/button/Button'; +import {Spinner} from '@app/features/ui/components/Spinner'; +import * as AvatarUtils from '@app/features/user/utils/AvatarUtils'; +import {getCurrentLocale} from '@app/features/user/utils/LocaleUtils'; +import {msg} from '@lingui/core/macro'; +import {Plural, Trans, useLingui} from '@lingui/react/macro'; +import {formatNumber} from '@pkgs/number_utils/src/NumberFormatting'; +import {clsx} from 'clsx'; +import {observer} from 'mobx-react-lite'; +import {useCallback, useEffect, useState} from 'react'; + +const GO_TO_COMMUNITY_DESCRIPTOR = msg({ + message: 'Go to community', + comment: + 'Button on the community card that opens from a published message copy, shown when the viewer is already a member of the source community.', +}); +const ONLINE_COUNT_DESCRIPTOR = msg({ + message: '{renderedOnlineCount} online', + comment: + 'Presence summary on the community card that opens from a published message copy. renderedOnlineCount is a formatted count of online members.', +}); +const COMMUNITY_UNAVAILABLE_DESCRIPTOR = msg({ + message: 'This community is no longer available', + comment: + 'Shown on the community card that opens from a published message copy when the source community was deleted or cannot be viewed.', +}); +const COMMUNITY_LOAD_FAILED_DESCRIPTOR = msg({ + message: "Couldn't load this community. Try again in a moment.", + comment: + 'Shown on the community card that opens from a published message copy when loading the source community failed.', +}); + +const ICON_SIZE_PX = 56; + +interface CommunityCardData { + id: string; + name: string; + icon: string | null; + banner: string | null; + description: string | null; + features: ReadonlySet | ReadonlyArray; + onlineCount: number | null; + memberCount: number | null; + isMember: boolean; + canJoin: boolean; +} + +type CommunityCardView = + | {kind: 'card'; data: CommunityCardData} + | {kind: 'loading'} + | {kind: 'unavailable'} + | {kind: 'failed'}; + +function useCommunityCardView(message: Message, sourceGuildId: string | null): CommunityCardView { + const localGuild = sourceGuildId != null ? Guilds.getGuild(sourceGuildId) : null; + useEffect(() => { + if (sourceGuildId == null) { + return; + } + CrosspostSourceCards.ensure(sourceGuildId, message.channelId, message.id); + }, [sourceGuildId, message.channelId, message.id]); + if (sourceGuildId == null) { + return {kind: 'unavailable'}; + } + const entry = CrosspostSourceCards.get(sourceGuildId); + if (localGuild != null) { + const counts = GuildCount.getCounts(localGuild.id); + const fetched = entry?.status === 'ready' ? entry.card : null; + return { + kind: 'card', + data: { + id: localGuild.id, + name: localGuild.name, + icon: localGuild.icon, + banner: localGuild.banner, + description: fetched?.description ?? null, + features: localGuild.features, + onlineCount: counts?.onlineCount ?? null, + memberCount: counts?.memberCount ?? null, + isMember: true, + canJoin: false, + }, + }; + } + if (entry == null || entry.status === 'loading') { + return {kind: 'loading'}; + } + if (entry.status === 'unavailable') { + return {kind: 'unavailable'}; + } + if (entry.status === 'failed') { + return {kind: 'failed'}; + } + const {card} = entry; + return { + kind: 'card', + data: { + id: card.id, + name: card.name, + icon: card.icon ?? null, + banner: card.banner ?? null, + description: card.description ?? null, + features: card.features, + onlineCount: card.approximate_presence_count, + memberCount: card.approximate_member_count, + isMember: false, + canJoin: card.discoverable && !RuntimeConfig.singleCommunityEnabled, + }, + }; +} + +interface CrosspostCommunityCardProps { + message: Message; + onClose: () => void; + variant?: 'popout' | 'sheet'; +} + +export const CrosspostCommunityCard = observer(function CrosspostCommunityCard({ + message, + onClose, + variant = 'popout', +}: CrosspostCommunityCardProps) { + const {i18n} = useLingui(); + const sourceGuildId = getCrosspostSourceGuildId(message); + const view = useCommunityCardView(message, sourceGuildId); + const [joining, setJoining] = useState(false); + const data = view.kind === 'card' ? view.data : null; + const guildId = data?.id ?? sourceGuildId ?? message.author.id; + const name = data?.name ?? getCrosspostSourceDisplayName(message); + const handleGoToCommunity = useCallback(() => { + onClose(); + NavigationCommands.selectGuild(guildId); + }, [onClose, guildId]); + const handleJoin = useCallback(async () => { + setJoining(true); + const joined = await joinDiscoveryGuild(guildId); + setJoining(false); + if (joined) { + onClose(); + } + }, [guildId, onClose]); + const bannerUrl = data?.banner ? AvatarUtils.getGuildBannerURL({id: data.id, banner: data.banner}) : null; + const locale = getCurrentLocale(); + const hasCounts = data != null && data.onlineCount != null && data.memberCount != null; + const renderedOnlineCount = hasCounts ? formatNumber(data.onlineCount ?? 0, locale) : ''; + const renderedMemberCount = hasCounts ? formatNumber(data.memberCount ?? 0, locale) : ''; + const memberCount = data?.memberCount ?? 0; + return ( +
+
+ {bannerUrl != null ? ( + <> + + + + ) : ( + + )} +
+ +
+
+ + {name} + + {data != null && ( + + )} +
+ {data?.description && ( + + {data.description} + + )} + {view.kind === 'loading' && ( +
+ +
+ )} + {view.kind === 'unavailable' && ( + + {i18n._(COMMUNITY_UNAVAILABLE_DESCRIPTOR)} + + )} + {view.kind === 'failed' && ( + + {i18n._(COMMUNITY_LOAD_FAILED_DESCRIPTOR)} + + )} +
+ {hasCounts && ( +
+ + + + {i18n._(ONLINE_COUNT_DESCRIPTOR, {renderedOnlineCount})} + + + + + + + {renderedMemberCount}{' '} + + + + +
+ )} + {data?.isMember && ( +
+ +
+ )} + {data != null && !data.isMember && data.canJoin && ( +
+ +
+ )} +
+ ); +}); diff --git a/fluxer_app/src/features/channel/components/CrosspostCommunityPopout.tsx b/fluxer_app/src/features/channel/components/CrosspostCommunityPopout.tsx new file mode 100644 index 000000000..d2fc7b4fc --- /dev/null +++ b/fluxer_app/src/features/channel/components/CrosspostCommunityPopout.tsx @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {CrosspostCommunityCard} from '@app/features/channel/components/CrosspostCommunityCard'; +import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import {WebhookContextMenu} from '@app/features/ui/action_menu/WebhookContextMenu'; +import {BottomSheet} from '@app/features/ui/bottom_sheet/BottomSheet'; +import * as ContextMenuCommands from '@app/features/ui/commands/ContextMenuCommands'; +import {Popout} from '@app/features/ui/popover/PopoverPopout'; +import MobileLayout from '@app/features/ui/state/MobileLayout'; +import {observer} from 'mobx-react-lite'; +import React, {useCallback, useState} from 'react'; + +type CrosspostChildProps = React.HTMLAttributes & React.RefAttributes; + +interface CrosspostCommunityPopoutProps { + message: Message; + children: React.ReactElement; + onPopoutOpen?: () => void; + onPopoutClose?: () => void; +} + +export const CrosspostCommunityPopout = observer(function CrosspostCommunityPopout({ + message, + children, + onPopoutOpen, + onPopoutClose, +}: CrosspostCommunityPopoutProps) { + const [sheetOpen, setSheetOpen] = useState(false); + const webhookId = message.webhookId; + const handleContextMenu = useCallback( + (event: React.MouseEvent) => { + if (!webhookId) return; + event.preventDefault(); + event.stopPropagation(); + ContextMenuCommands.openFromEvent(event, ({onClose}) => ( + + )); + }, + [webhookId], + ); + const handleSheetClose = useCallback(() => setSheetOpen(false), []); + if (MobileLayout.enabled) { + const {onClick: originalOnClick} = children.props; + const clonedChild = React.cloneElement(children, { + onClick: (event: React.MouseEvent) => { + originalOnClick?.(event); + setSheetOpen(true); + }, + onContextMenu: handleContextMenu, + }); + return ( + <> + {clonedChild} + {sheetOpen && ( + + + + )} + + ); + } + return ( + ( + + )} + position="right-start" + animationType="profile-slide" + constrainHeight={false} + freezePosition + keepOpenOnTargetUnmount + onOpen={onPopoutOpen} + onClose={onPopoutClose} + data-flx="channel.crosspost-community-popout.popout" + > + {React.cloneElement(children, {onContextMenu: handleContextMenu})} + + ); +}); diff --git a/fluxer_app/src/features/channel/components/CrosspostPublishNudge.module.css b/fluxer_app/src/features/channel/components/CrosspostPublishNudge.module.css new file mode 100644 index 000000000..65e522eeb --- /dev/null +++ b/fluxer_app/src/features/channel/components/CrosspostPublishNudge.module.css @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: AGPL-3.0-or-later */ + +.row { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.25rem 1rem; + margin-block-start: 0.25rem; + min-inline-size: 0; + font-size: 0.875rem; + line-height: 1.25rem; + user-select: none; + -webkit-user-select: none; +} + +.pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + min-inline-size: 0; + padding: 0.375rem 0.625rem; + border-radius: var(--radius-md); + background: var(--background-secondary); + color: var(--text-primary); +} + +.icon { + width: 1rem; + height: 1rem; + flex: 0 0 auto; + color: var(--text-tertiary); +} + +.label { + min-inline-size: 0; + font-weight: 400; +} + +.publishButton, +.dismissButton, +.hideForeverButton { + display: inline-flex; + align-items: center; + padding: 0; + border: none; + border-radius: var(--radius-sm); + background: transparent; + font: inherit; + cursor: pointer; + flex-shrink: 0; +} + +.publishButton { + color: var(--text-link); + font-weight: 500; +} + +.dismissButton { + color: var(--text-tertiary); +} + +.dismissIcon { + width: 1.125rem; + height: 1.125rem; +} + +.hideForeverButton { + color: var(--text-chat-muted); +} + +@media (hover: hover) { + .publishButton:hover, + .hideForeverButton:hover { + text-decoration: underline; + } + + .dismissButton:hover { + color: var(--text-primary); + } +} diff --git a/fluxer_app/src/features/channel/components/CrosspostPublishNudge.tsx b/fluxer_app/src/features/channel/components/CrosspostPublishNudge.tsx new file mode 100644 index 000000000..4aa7a19a2 --- /dev/null +++ b/fluxer_app/src/features/channel/components/CrosspostPublishNudge.tsx @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import styles from '@app/features/channel/components/CrosspostPublishNudge.module.css'; +import {requestMessageCrosspost, useMessagePermissions} from '@app/features/channel/components/MessageActionUtils'; +import type {Channel} from '@app/features/channel/models/Channel'; +import PublishNudge from '@app/features/channel/state/PublishNudge'; +import {DISMISS_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; +import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import {AnnouncementChannelIcon} from '@app/features/ui/components/icons/AnnouncementChannelIcon'; +import FocusRing from '@app/features/ui/focus_ring/FocusRing'; +import {msg} from '@lingui/core/macro'; +import {useLingui} from '@lingui/react/macro'; +import {XCircleIcon} from '@phosphor-icons/react'; +import {observer} from 'mobx-react-lite'; +import type React from 'react'; + +const SHARE_WITH_FOLLOWERS_DESCRIPTOR = msg({ + message: 'Not sent to followers yet.', + comment: + 'Prompt under the newest message the current user sent in an announcement channel, suggesting they publish it to the communities that follow the channel.', +}); +const PUBLISH_DESCRIPTOR = msg({ + message: 'Publish', + comment: + 'Link-style button in the prompt under a new announcement channel message. Publishes the message to the communities that follow the channel.', +}); +const DONT_SHOW_AGAIN_DESCRIPTOR = msg({ + message: "Don't show again", + comment: + 'Muted button next to the publish prompt under a new announcement channel message. Hides the prompt for every future message.', +}); + +interface CrosspostPublishNudgeProps { + message: Message; + channel: Channel; +} + +export const CrosspostPublishNudge = observer(({message, channel}: CrosspostPublishNudgeProps) => { + const {i18n} = useLingui(); + const permissions = useMessagePermissions(message, channel); + if (!PublishNudge.shouldShow(message, permissions?.canCrosspostMessage ?? false)) { + return null; + } + const handlePublish = (event: React.MouseEvent) => { + requestMessageCrosspost(message, i18n, {shiftKey: event.shiftKey}); + }; + return ( +
+
+ + + {i18n._(SHARE_WITH_FOLLOWERS_DESCRIPTOR)} + + + + + + + +
+ + + +
+ ); +}); diff --git a/fluxer_app/src/features/channel/components/MessageActionBar.module.css b/fluxer_app/src/features/channel/components/MessageActionBar.module.css index aefefdfdb..332c9c450 100644 --- a/fluxer_app/src/features/channel/components/MessageActionBar.module.css +++ b/fluxer_app/src/features/channel/components/MessageActionBar.module.css @@ -84,8 +84,17 @@ display: none; } +.button.disabled { + color: var(--text-chat-muted); + cursor: default; +} + +.button.disabled .actionBarIcon { + opacity: 0.5; +} + :global(:where(html.window-focused, html.unfocused-fully-interactive):not(.window-focus-activation-guard)) - .button:not(.active):not(.danger):hover { + .button:not(.active):not(.danger):not(.disabled):hover { color: var(--text-primary); background: var(--message-action-bar-hover-background); } diff --git a/fluxer_app/src/features/channel/components/MessageActionBar.tsx b/fluxer_app/src/features/channel/components/MessageActionBar.tsx index 1484d70e1..17626d6e8 100644 --- a/fluxer_app/src/features/channel/components/MessageActionBar.tsx +++ b/fluxer_app/src/features/channel/components/MessageActionBar.tsx @@ -38,6 +38,7 @@ import { AddReactionIcon, CopyIdIcon, CopyLinkIcon, + CrosspostIcon, DebugMessageIcon, DeleteIcon, EditMessageIcon, @@ -82,6 +83,16 @@ const DEBUG_MESSAGE_DESCRIPTOR = msg({ message: 'Debug message', comment: 'Developer-mode item in the message action bar overflow menu. Opens the message debug modal.', }); +const PUBLISH_DESCRIPTOR = msg({ + message: 'Publish', + comment: + 'Tooltip on the publish button in the inline message hover action bar. Publishing sends an announcement channel message to every channel that follows it.', +}); +const PUBLISHED_DESCRIPTOR = msg({ + message: 'Published', + comment: + 'Tooltip on the disabled publish button in the inline message hover action bar for a message that was already published.', +}); const FORWARD_DESCRIPTOR = msg({ message: 'Forward', comment: 'Tooltip on the forward button in the inline message hover action bar.', @@ -95,18 +106,22 @@ interface MessageActionBarButtonProps { danger?: boolean; isActive?: boolean; hidden?: boolean; + disabled?: boolean; dataAction?: string; } const MessageActionBarButton = React.forwardRef( - ({label, icon, onClick, onPointerDownCapture, danger, isActive, hidden, dataAction}, ref) => { + ({label, icon, onClick, onPointerDownCapture, danger, isActive, hidden, disabled, dataAction}, ref) => { const handleClick = useCallback( (event: React.MouseEvent | React.KeyboardEvent) => { event.preventDefault(); event.stopPropagation(); + if (disabled) { + return; + } onClick?.(event); }, - [onClick], + [onClick, disabled], ); const handlePointerDownCapture = useCallback( (event: React.PointerEvent) => { @@ -115,8 +130,8 @@ const MessageActionBarButton = React.forwardRef clsx(styles.button, danger && styles.danger, isActive && styles.active), - [danger, isActive], + () => clsx(styles.button, danger && styles.danger, isActive && styles.active, disabled && styles.disabled), + [danger, isActive, disabled], ); return ( @@ -125,6 +140,7 @@ const MessageActionBarButton = React.forwardRef