chore(moderation): remove the built-in NCMEC integration (#3332)

This commit is contained in:
Hampus authored and GitHub committed 2026-10-09 01:39:04 +02:00
1 parent f390e610b9
commit 454fefdb92
146 files changed
+234 -4075

No files matched your search

-7
View File
@@ -383,13 +383,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
port: master.integrations.clamav.port,
failOpen: master.integrations.clamav.fail_open,
},
ncmec: {
enabled: master.integrations.ncmec.enabled,
baseUrl: master.integrations.ncmec.base_url,
username: master.integrations.ncmec.username,
password: master.integrations.ncmec.password,
reporterEmail: master.integrations.ncmec.reporter_email ?? '',
},
admin: {
oauthClientSecret: master.services.admin.oauth_client_secret,
},
-16
View File
@@ -149,12 +149,6 @@ import {
type WebhooksBySourceChannelRow,
} from '@app/api/database/types/ChannelTypes';
import {USER_CONNECTION_STORAGE_COLUMNS, type UserConnectionStorageRow} from '@app/api/database/types/ConnectionTypes';
import {
NCMEC_ATTACHMENT_SUBMISSION_COLUMNS,
NCMEC_USER_WORKFLOW_COLUMNS,
type NcmecAttachmentSubmissionRow,
type NcmecUserWorkflowRow,
} from '@app/api/database/types/CsamTypes';
import {
GUILD_DISCOVERY_BY_STATUS_COLUMNS,
GUILD_DISCOVERY_COLUMNS,
@@ -1294,16 +1288,6 @@ export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTr
primaryKey: ['attachment_id'],
defaultTtlSeconds: seconds('30 days'),
});
export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
name: 'ncmec_attachment_submissions',
columns: NCMEC_ATTACHMENT_SUBMISSION_COLUMNS,
primaryKey: ['attachment_id'],
});
export const NcmecUserWorkflows = defineTable<NcmecUserWorkflowRow, 'user_id'>({
name: 'ncmec_user_workflows',
columns: NCMEC_USER_WORKFLOW_COLUMNS,
primaryKey: ['user_id'],
});
export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
name: 'billing_customers',
columns: BILLING_CUSTOMER_COLUMNS,
+1 -7
View File
@@ -31,11 +31,7 @@ import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import type {InviteRepository} from '@app/api/invite/InviteRepository';
import type {IJobLedgerRepository} from '@app/api/jobs/IJobLedgerRepository';
import {JobAdminService} from '@app/api/jobs/JobAdminService';
import {
getGuildDiscoveryRepository,
getKVAccountDeletionQueue,
getNcmecSubmissionService,
} from '@app/api/middleware/ServiceSingletons';
import {getGuildDiscoveryRepository, getKVAccountDeletionQueue} from '@app/api/middleware/ServiceSingletons';
import type {IApplicationRepository} from '@app/api/oauth/repositories/IApplicationRepository';
import type {ReportService} from '@app/api/report/ReportService';
import type {StoreEntitlementService} from '@app/api/store_billing/StoreEntitlementService';
@@ -140,7 +136,6 @@ export class AdminService {
channelRepository: this.channelRepository,
guildRepository: this.guildRepository,
auditService: this.auditService,
ncmecSubmissionService: getNcmecSubmissionService(),
});
this.messageShredService = new AdminMessageShredService({
apiContext: this.apiContext,
@@ -159,7 +154,6 @@ export class AdminService {
storageService: this.storageService,
auditService: this.auditService,
userCacheService: this.userCacheService,
ncmecSubmissionService: getNcmecSubmissionService(),
reporterResolutionNotifier,
});
this.voiceService = new AdminVoiceService({
@@ -2,7 +2,6 @@
import {AdminAuditReadActions} from '@app/api/admin/AdminAuditActions';
import {recordAdminRead} from '@app/api/admin/AdminAuditRecorder';
import {createAttachmentID, createChannelID, createMessageID, createReportID} from '@app/api/BrandedTypes';
import {requireAdminACL} from '@app/api/middleware/AdminMiddleware';
import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
@@ -25,13 +24,11 @@ import {
DeleteAllUserMessagesResponse,
MessageShredJobIdParam,
MessageShredResponse,
ReportAttachmentToNcmecRequest,
} from '@fluxer/schema/src/domains/admin/AdminMessageSchemas';
import {
DeleteMessageResponse,
LookupMessageResponse,
MessageShredStatusResponse,
NcmecAttachmentSubmitResultResponse,
} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {
ChannelIdMessageIdParam,
@@ -124,40 +121,6 @@ export function MessageAdminController(app: HonoApp) {
return ctx.json(response);
},
);
app.post(
'/admin/messages/ncmec-reports',
RateLimitMiddleware(RateLimitConfigs.ADMIN_MESSAGE_OPERATION),
requireAdminACL(AdminACLs.CSAM_SUBMIT_NCMEC),
requireAdminACL(AdminACLs.MESSAGE_DELETE),
requireAdminACL(AdminACLs.USER_DELETE),
requireAdminACL(AdminACLs.ARCHIVE_TRIGGER_USER),
Validator('json', ReportAttachmentToNcmecRequest),
OpenAPI({
operationId: 'create_admin_ncmec_report',
summary: 'Report an attachment to NCMEC',
description:
'Submits a specific image attachment to NCMEC, creates an audit log entry, silently disables the user, triggers one archive for the user, and schedules content deletion after the archive completes.',
responseSchema: NcmecAttachmentSubmitResultResponse,
statusCode: 200,
security: 'adminApiKey',
tags: 'Admin',
}),
async (ctx) => {
const service = ctx.get('ncmecSubmissionService');
const adminUserId = ctx.get('adminUserId');
const body = ctx.req.valid('json');
const result = await service.submitAttachmentToNcmec({
channelId: createChannelID(body.channel_id),
messageId: createMessageID(body.message_id),
attachmentId: createAttachmentID(body.attachment_id),
filename: body.filename,
reporterFullName: body.reporter_full_name,
adminUserId,
sourceReportId: body.source_report_id ? createReportID(body.source_report_id) : null,
});
return ctx.json(result);
},
);
app.get(
'/admin/messages/shreds/:job_id',
RateLimitMiddleware(RateLimitConfigs.ADMIN_MESSAGE_OPERATION),
@@ -26,7 +26,6 @@ import {
messageResponseAccessForGuild,
} from '@app/api/channel/services/message/MessageResponseDataService';
import {resolveNsfwScopeChannel} from '@app/api/channel/utils/ThreadNsfwScope';
import type {NcmecAttachmentStatusResponse, NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import {getPurgeQueue, getStorageService} from '@app/api/middleware/ServiceSingletons';
import {getMessageSearchService} from '@app/api/SearchFactory';
@@ -49,7 +48,6 @@ interface AdminMessageServiceDeps {
channelRepository: IChannelRepository;
guildRepository: IGuildRepositoryAggregate;
auditService: AdminAuditService;
ncmecSubmissionService: NcmecSubmissionService;
}
interface ChannelNsfwContext {
@@ -92,10 +90,8 @@ export class AdminMessageService {
around: messageId,
})
).reverse();
const attachmentStatuses = await this.getAttachmentStatusesForMessages(messageResponses);
const priorReports = await this.getPriorReportsForMessages(messageResponses);
const adminMessages = messageResponses.map((message) =>
this.mapMessageResponseToAdminMessage(message, channelNsfwContext, attachmentStatuses, priorReports),
this.mapMessageResponseToAdminMessage(message, channelNsfwContext),
);
return {
messages: adminMessages,
@@ -207,10 +203,8 @@ export class AdminMessageService {
after: afterId,
});
const messageResponses = afterId ? messages : [...messages].reverse();
const attachmentStatuses = await this.getAttachmentStatusesForMessages(messageResponses);
const priorReports = await this.getPriorReportsForMessages(messageResponses);
const adminMessages = messageResponses.map((message) =>
this.mapMessageResponseToAdminMessage(message, channelNsfwContext, attachmentStatuses, priorReports),
this.mapMessageResponseToAdminMessage(message, channelNsfwContext),
);
return {
messages: adminMessages,
@@ -241,10 +235,8 @@ export class AdminMessageService {
messages: result.messages,
access: await this.getMessageResponseAccessForAdmin(channelId),
});
const attachmentStatuses = await this.getAttachmentStatusesForMessages(messageResponses);
const priorReports = await this.getPriorReportsForMessages(messageResponses);
const adminMessages = messageResponses.map((message) =>
this.mapMessageResponseToAdminMessage(message, channelNsfwContext, attachmentStatuses, priorReports),
this.mapMessageResponseToAdminMessage(message, channelNsfwContext),
);
return {
messages: adminMessages,
@@ -253,15 +245,6 @@ export class AdminMessageService {
};
}
private async getAttachmentStatusesForMessages(
messages: Array<MessageResponse>,
): Promise<Map<string, NcmecAttachmentStatusResponse>> {
const attachmentIds = messages.flatMap((message) =>
(message.attachments ?? []).map((attachment) => createAttachmentID(BigInt(attachment.id))),
);
return this.deps.ncmecSubmissionService.getAttachmentStatuses(attachmentIds);
}
private async getMessageResponseAccessForAdmin(channelId: ChannelID): Promise<MessageResponseAccessContext> {
const channel = await this.deps.channelRepository.findUnique(channelId);
const access = channel ? messageResponseAccessForChannel(channel) : messageResponseAccessForGuild(null);
@@ -287,17 +270,7 @@ export class AdminMessageService {
});
}
private async getPriorReportsForMessages(messages: Array<MessageResponse>): Promise<Map<string, Array<string>>> {
const authorIds = messages.map((message) => createUserID(BigInt(message.author.id)));
return this.deps.ncmecSubmissionService.getUserPriorReportIds(authorIds);
}
private mapMessageResponseToAdminMessage(
message: MessageResponse,
channelNsfwContext: ChannelNsfwContext,
attachmentStatuses: Map<string, NcmecAttachmentStatusResponse>,
priorReports: Map<string, Array<string>>,
) {
private mapMessageResponseToAdminMessage(message: MessageResponse, channelNsfwContext: ChannelNsfwContext) {
return {
id: message.id,
channel_id: message.channel_id ?? '',
@@ -313,7 +286,6 @@ export class AdminMessageService {
author_avatar: message.author.avatar,
content: message.content ?? '',
timestamp: message.timestamp,
user_prior_ncmec_report_ids: priorReports.get(message.author.id) ?? [],
attachments:
message.attachments?.map((attachment) => ({
id: attachment.id,
@@ -324,9 +296,6 @@ export class AdminMessageService {
width: attachment.width ?? null,
height: attachment.height ?? null,
size: attachment.size == null ? null : assertSafeByteSize(attachment.size, 'admin message attachment size'),
ncmec_status: attachmentStatuses.get(attachment.id)?.status ?? 'not_submitted',
ncmec_report_id: attachmentStatuses.get(attachment.id)?.ncmec_report_id ?? null,
ncmec_failure_reason: attachmentStatuses.get(attachment.id)?.failure_reason ?? null,
})) ?? [],
};
}
@@ -24,7 +24,6 @@ import {
messageResponseAccessForGuild,
} from '@app/api/channel/services/message/MessageResponseDataService';
import {resolveNsfwScopeChannel} from '@app/api/channel/utils/ThreadNsfwScope';
import type {NcmecAttachmentStatusResponse, NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
@@ -63,7 +62,6 @@ interface AdminReportServiceDeps {
storageService: IStorageService;
auditService: AdminAuditService;
userCacheService: UserCacheService;
ncmecSubmissionService: NcmecSubmissionService;
reporterResolutionNotifier: ReporterResolutionNotifier;
}
@@ -378,8 +376,6 @@ export class AdminReportService {
if (!includeContext) {
return baseResponse;
}
const attachmentStatusesById = await this.getAttachmentStatusesById(report);
const priorReportsByAuthor = await this.getPriorReportsForContext(report);
const authorBotFlags = await this.getContextAuthorBotFlags(report, requestCache);
const messageContext = (
await Promise.all(
@@ -389,8 +385,6 @@ export class AdminReportService {
report.reportedChannelId ?? null,
report.reportedGuildId ?? report.guildContextId ?? null,
reportLookupCache,
attachmentStatusesById,
priorReportsByAuthor,
authorBotFlags,
),
),
@@ -566,8 +560,6 @@ export class AdminReportService {
fallbackChannelId: ChannelID | null,
fallbackGuildId: GuildID | null,
reportLookupCache: ReportLookupCache,
attachmentStatusesById: Map<string, NcmecAttachmentStatusResponse>,
priorReportsByAuthor: Map<string, Array<string>>,
authorBotFlags: Map<string, boolean | null>,
) {
const channelId = message.channelId ?? fallbackChannelId;
@@ -581,9 +573,7 @@ export class AdminReportService {
message.attachments && message.attachments.length > 0
? (
await Promise.all(
message.attachments.map((attachment) =>
this.mapReportAttachmentToResponse(attachment, channelId, attachmentStatusesById),
),
message.attachments.map((attachment) => this.mapReportAttachmentToResponse(attachment, channelId)),
)
).filter(
(
@@ -597,9 +587,6 @@ export class AdminReportService {
width: number | null;
height: number | null;
size: number | null;
ncmec_status: string;
ncmec_report_id: string | null;
ncmec_failure_reason: string | null;
} => attachment !== null,
)
: [];
@@ -627,21 +614,9 @@ export class AdminReportService {
missing_attachments: message.missingAttachments
.filter((attachment) => attachment.attachment_id != null && attachment.filename)
.map(mapMissingAttachmentToResponse),
user_prior_ncmec_report_ids: message.authorId ? (priorReportsByAuthor.get(authorId) ?? []) : [],
};
}
private async getPriorReportsForContext(report: IARSubmission): Promise<Map<string, Array<string>>> {
const authorIds = new Set<string>();
for (const message of report.messageContext ?? []) {
if (message.authorId) authorIds.add(message.authorId.toString());
}
if (report.reportedUserId) authorIds.add(report.reportedUserId.toString());
if (authorIds.size === 0) return new Map();
const userIds = [...authorIds].map((value) => createUserID(BigInt(value)));
return this.deps.ncmecSubmissionService.getUserPriorReportIds(userIds);
}
private async getChannelNsfwState(
channelId: ChannelID | null,
reportLookupCache: ReportLookupCache,
@@ -700,7 +675,6 @@ export class AdminReportService {
private async mapReportAttachmentToResponse(
attachment: MessageAttachment,
channelId: ChannelID | null,
attachmentStatusesById: Map<string, NcmecAttachmentStatusResponse>,
): Promise<{
id: string;
filename: string;
@@ -710,9 +684,6 @@ export class AdminReportService {
width: number | null;
height: number | null;
size: number | null;
ncmec_status: string;
ncmec_report_id: string | null;
ncmec_failure_reason: string | null;
} | null> {
if (!attachment || attachment.attachment_id == null || !attachment.filename || !channelId) {
return null;
@@ -736,9 +707,6 @@ export class AdminReportService {
width: attachment.width ?? null,
height: attachment.height ?? null,
size: attachment.size != null ? assertSafeByteSize(attachment.size, 'admin report attachment size') : null,
ncmec_status: attachmentStatusesById.get(attachment.attachment_id.toString())?.status ?? 'not_submitted',
ncmec_report_id: attachmentStatusesById.get(attachment.attachment_id.toString())?.ncmec_report_id ?? null,
ncmec_failure_reason: attachmentStatusesById.get(attachment.attachment_id.toString())?.failure_reason ?? null,
};
} catch (error) {
Logger.error(
@@ -749,13 +717,6 @@ export class AdminReportService {
return null;
}
private async getAttachmentStatusesById(report: IARSubmission) {
const attachmentIds = (report.messageContext ?? []).flatMap((message) =>
message.attachments.map((attachment) => attachment.attachment_id),
);
return this.deps.ncmecSubmissionService.getAttachmentStatuses(attachmentIds);
}
private async buildUserTag(userId: UserID | null, requestCache: RequestCache): Promise<UserTagInfo | null> {
if (!userId) {
return null;
@@ -61,7 +61,6 @@ describe('Admin audit log access filter', () => {
expect(getAdminAuditAccess(action)).toBe('read');
}
expect(getAdminAuditAccess('update_flags')).toBe('write');
expect(getAdminAuditAccess('NCMEC Report')).toBe('write');
});
for (const search of ['disabled', 'enabled'] as const) {
@@ -116,47 +116,6 @@ export const MessageAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [
};
},
},
{
method: 'POST',
route: '/admin/messages/ncmec-reports',
auditLogReason: expect.stringMatching(/^NCMEC Report \S+ - \S+$/),
async prepare({harness}) {
const {author, channelId} = await createAuthorChannel(harness);
const message = await createMessageWithImageAttachment(harness, author.token, channelId);
const attachment = message.attachments[0];
return {
request: {
path: '/admin/messages/ncmec-reports',
body: {
channel_id: channelId,
message_id: message.id,
attachment_id: attachment.id,
filename: attachment.filename,
reporter_full_name: 'Audit Coverage Reporter',
confirmed_viewed: true,
},
},
expected: {
action: 'NCMEC Report',
targetType: 'user',
targetId: author.userId,
metadata: {
channel_id: channelId,
message_id: message.id,
attachment_id: attachment.id,
reported_user_email: author.email,
reported_user_email_verified: 'true',
reported_user_date_of_birth: '2000-01-01',
reported_user_last_active_ip: '127.0.0.1',
attachment_upload_mode: 'form_data',
attachment_upload_request_ip: '127.0.0.1',
attachment_upload_requested_at: expect.any(String),
attachment_upload_ip_source: 'attachment upload request handled directly by Fluxer API',
},
},
};
},
},
{
method: 'GET',
route: '/admin/messages/shreds/:job_id',
@@ -12,10 +12,6 @@ const GET_UPLOAD_TRACE_BY_KEY_QUERY = AttachmentUploadTracesByKey.select({
where: AttachmentUploadTracesByKey.where.eq('upload_key'),
limit: 1,
});
const GET_UPLOAD_TRACE_BY_ATTACHMENT_QUERY = AttachmentUploadTracesByAttachment.select({
where: AttachmentUploadTracesByAttachment.where.eq('attachment_id'),
limit: 1,
});
export type AttachmentUploadMode = 'form_data' | 'presigned_singlepart' | 'presigned_multipart';
@@ -48,12 +44,6 @@ export class AttachmentUploadTraceRepository {
return await fetchOne<AttachmentUploadTraceByKeyRow>(GET_UPLOAD_TRACE_BY_KEY_QUERY.bind({upload_key: uploadKey}));
}
async getByAttachmentId(attachmentId: AttachmentID): Promise<AttachmentUploadTraceByAttachmentRow | null> {
return await fetchOne<AttachmentUploadTraceByAttachmentRow>(
GET_UPLOAD_TRACE_BY_ATTACHMENT_QUERY.bind({attachment_id: BigInt(attachmentId)}),
);
}
async getPendingUpload(input: GetPendingAttachmentUploadInput): Promise<AttachmentUploadTraceByKeyRow | null> {
const existing = await this.getByUploadKey(input.uploadKey);
if (
@@ -1,16 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
type ChannelID,
createAttachmentID,
createChannelID,
createGuildID,
createMessageID,
createUserID,
type MessageID,
type UserID,
} from '@app/api/BrandedTypes';
import {createChannelID, createGuildID, createMessageID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {ChannelRepository} from '@app/api/channel/ChannelRepository';
import {CrosspostTaskNames} from '@app/api/channel/services/message/CrosspostPropagation';
@@ -26,7 +17,6 @@ import {
import {loadFixture, sendMessageWithAttachments} from '@app/api/channel/tests/AttachmentTestUtils';
import {sendChannelMessage} from '@app/api/channel/tests/ChannelTestUtils';
import {readMessageRow, writeMessageRow} from '@app/api/channel/tests/CrosspostTestUtils';
import {NcmecRepository} from '@app/api/csam/NcmecRepository';
import type {MessageEmbed} from '@app/api/database/types/MessageTypes';
import type {EmbedService} from '@app/api/infrastructure/EmbedService';
import {
@@ -38,7 +28,6 @@ import {
import {
getCacheService,
getChannelRepository,
getNcmecSubmissionService,
getPurgeQueue,
getStorageService,
} from '@app/api/middleware/ServiceSingletons';
@@ -157,11 +146,6 @@ class CrosspostQueueWorker implements IWorkerService {
}
}
interface FabricatedCopy {
copyId: string;
attachment: {id: string; filename: string} | null;
}
async function fabricateCopy(params: {
harness: ApiTestHarness;
owner: TestAccount;
@@ -169,8 +153,7 @@ async function fabricateCopy(params: {
world: AnnouncementWorld;
sourceId: string;
extraFlags?: number;
withImage?: boolean;
}): Promise<FabricatedCopy> {
}): Promise<{copyId: string}> {
const webhook = await createWebhook(params.harness, params.target, params.owner.token, 'Follower');
const {json: message} = await executeWebhook(
params.harness,
@@ -179,14 +162,10 @@ async function fabricateCopy(params: {
{content: 'copied update', wait: true},
200,
);
const sourceAttachments = params.withImage
? (await readMessageRow(params.world.a.ann.id, params.sourceId))!.attachments
: [];
const row = await readMessageRow(params.target, message!.id);
await writeMessageRow(row!, {
flags: MessageFlags.IS_CROSSPOST | (params.extraFlags ?? 0),
attachments:
sourceAttachments.length > 0 ? sourceAttachments.map((attachment) => attachment.toMessageAttachment()) : null,
attachments: null,
message_reference: {
channel_id: createChannelID(BigInt(params.world.a.ann.id)),
guild_id: createGuildID(BigInt(params.world.a.guild.id)),
@@ -194,11 +173,7 @@ async function fabricateCopy(params: {
type: MessageReferenceTypes.DEFAULT,
},
});
const attachment = sourceAttachments[0];
return {
copyId: message!.id,
attachment: attachment ? {id: attachment.id.toString(), filename: attachment.filename} : null,
};
return {copyId: message!.id};
}
async function createAdmin(harness: ApiTestHarness): Promise<TestAccount> {
@@ -206,9 +181,6 @@ async function createAdmin(harness: ApiTestHarness): Promise<TestAccount> {
AdminACLs.AUTHENTICATE,
AdminACLs.MESSAGE_DELETE,
AdminACLs.MESSAGE_SHRED,
AdminACLs.CSAM_SUBMIT_NCMEC,
AdminACLs.USER_DELETE,
AdminACLs.ARCHIVE_TRIGGER_USER,
]);
}
@@ -219,17 +191,6 @@ async function adminDeleteMessage(harness: ApiTestHarness, admin: TestAccount, c
.execute();
}
async function deleteMessageSilently(channelId: string, messageId: string, fallbackUserId: string): Promise<void> {
const service = getNcmecSubmissionService() as unknown as {
deleteMessageSilently(channelId: ChannelID, messageId: MessageID, fallbackUserId: UserID): Promise<void>;
};
await service.deleteMessageSilently(
createChannelID(BigInt(channelId)),
createMessageID(BigInt(messageId)),
createUserID(BigInt(fallbackUserId)),
);
}
function blockedEmbedService(): EmbedService {
return {
processUrlWithCachePolicy: async (url: string) => ({
@@ -331,73 +292,6 @@ describe('Crosspost moderation', () => {
expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'});
});
test('an NCMEC report on a copy records the source author and its takedown purges the family', async () => {
const {json: source} = await sendMessageWithAttachments(
harness,
world.a.member.token,
world.a.ann.id,
{content: 'update', attachments: [{id: 0, filename: 'yeah.png'}]},
[{index: 0, filename: 'yeah.png', data: loadFixture('yeah.png')}],
);
const {copyId, attachment} = await fabricateCopy({
harness,
owner: world.b.owner,
target: world.b.t1.id,
world,
sourceId: source.id,
withImage: true,
});
expect(attachment).not.toBeNull();
const admin = await createAdmin(harness);
await createBuilder(harness, admin.token)
.post('/admin/messages/ncmec-reports')
.body({
channel_id: world.b.t1.id,
message_id: copyId,
attachment_id: attachment!.id,
filename: attachment!.filename,
reporter_full_name: 'Crosspost Reporter',
confirmed_viewed: true,
})
.expect(HTTP_STATUS.OK)
.execute();
const submission = await new NcmecRepository().getAttachmentSubmission(
createAttachmentID(BigInt(attachment!.id)),
);
expect(submission?.user_id?.toString()).toBe(world.a.member.userId);
expect(submission?.message_id.toString()).toBe(copyId);
expect(submission?.channel_id.toString()).toBe(world.b.t1.id);
worker.clear();
const deletedBefore = deletedKeys().length;
await deleteMessageSilently(world.b.t1.id, copyId, world.a.member.userId);
expect(await readMessageRow(world.b.t1.id, copyId)).toBeNull();
expect(deletedKeys().slice(deletedBefore)).toEqual([]);
expect(
harness.storageService.hasObject(
Config.s3.buckets.cdn,
`attachments/${world.a.ann.id}/${attachment!.id}/${attachment!.filename}`,
),
).toBe(true);
const jobs = worker.syncJobs();
expect(jobs).toHaveLength(1);
expect(jobs[0]!.payload).toEqual({
channelId: world.a.ann.id,
messageId: source.id,
mode: 'purge',
deleteSource: true,
});
});
test('an NCMEC takedown of a published source purges its copies', async () => {
const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update');
await publish(harness, world.a.member.token, world.a.ann.id, source.id);
worker.clear();
await deleteMessageSilently(world.a.ann.id, source.id, world.a.member.userId);
const jobs = worker.syncJobs();
expect(jobs).toHaveLength(1);
expect(jobs[0]!.payload).toEqual({channelId: world.a.ann.id, messageId: source.id, mode: 'purge'});
});
test('a message shred of a published source purges its copies', async () => {
const source = await sendChannelMessage(harness, world.a.member.token, world.a.ann.id, 'update');
await publish(harness, world.a.member.token, world.a.ann.id, source.id);
@@ -585,35 +479,6 @@ describe('Crosspost moderation', () => {
expectNoTargetObjectsDeleted();
});
test('an NCMEC takedown of a delivered copy records the source author and removes the family', async () => {
const sourceId = await publishAndDeliver('image update', true);
const sourceKey = await sourceAttachmentKey(sourceId);
const [copyId] = await listCopies(c.t1.id, sourceId);
const copy = await readMessageRow(c.t1.id, copyId!);
const attachment = copy!.attachments[0]!;
expect(`attachments/${world.a.ann.id}/${attachment.id}/${attachment.filename}`).toBe(sourceKey);
const admin = await createAdmin(harness);
await createBuilder(harness, admin.token)
.post('/admin/messages/ncmec-reports')
.body({
channel_id: c.t1.id,
message_id: copyId,
attachment_id: attachment.id.toString(),
filename: attachment.filename,
reporter_full_name: 'Crosspost Reporter',
confirmed_viewed: true,
})
.expect(HTTP_STATUS.OK)
.execute();
const submission = await new NcmecRepository().getAttachmentSubmission(attachment.id);
expect(submission?.user_id?.toString()).toBe(world.a.member.userId);
await deleteMessageSilently(c.t1.id, copyId!, world.a.member.userId);
await worker.drain();
await expectFamilyGone(sourceId);
expect(deletedKeys().filter((key) => key === sourceKey)).toHaveLength(1);
expectNoTargetObjectsDeleted();
});
test('a blocked unfurl of a published source deletes every copy', async () => {
const sourceId = await publishAndDeliver(`see https://${BLOCKED_DOMAIN}/page`);
expect(await listCopies(c.t1.id, sourceId)).toHaveLength(1);
@@ -1,15 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
type ChannelID,
createChannelID,
createMessageID,
createReportID,
createUserID,
type MessageID,
type UserID,
} from '@app/api/BrandedTypes';
import {createReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {
createChannel,
@@ -18,7 +10,6 @@ import {
sendMessageWithAttachments,
} from '@app/api/channel/tests/AttachmentTestUtils';
import {acceptInvite, createChannelInvite} from '@app/api/channel/tests/ChannelTestUtils';
import {getNcmecSubmissionService} from '@app/api/middleware/ServiceSingletons';
import {ReportRepository} from '@app/api/report/ReportRepository';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
@@ -210,13 +201,7 @@ describe('Embed attachment purge', () => {
}
async function createAdmin(): Promise<TestAccount> {
return setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.MESSAGE_DELETE,
AdminACLs.CSAM_SUBMIT_NCMEC,
AdminACLs.USER_DELETE,
AdminACLs.ARCHIVE_TRIGGER_USER,
]);
return setUserACLs(harness, await createTestAccount(harness), [AdminACLs.AUTHENTICATE, AdminACLs.MESSAGE_DELETE]);
}
function embedKeys(message: MessageResponse): Array<string> {
@@ -274,24 +259,9 @@ describe('Embed attachment purge', () => {
expect(deletedKeys()).toEqual(expect.arrayContaining(keys));
});
it('removes files shown only through an embed on a silent NCMEC delete', async () => {
const message = await sendEmbedOnly();
const keys = embedKeys(message);
const service = getNcmecSubmissionService() as unknown as {
deleteMessageSilently(channelId: ChannelID, messageId: MessageID, fallbackUserId: UserID): Promise<void>;
};
await service.deleteMessageSilently(
createChannelID(BigInt(channelId)),
createMessageID(BigInt(message.id)),
createUserID(BigInt(account.userId)),
);
expect(deletedKeys()).toEqual(expect.arrayContaining(keys));
});
it('keeps reported embed files as report evidence after the author deletes the message', async () => {
const message = await sendEmbedOnly();
const [imageKey] = embedKeys(message);
const [, , attachmentId, filename] = imageKey!.split('/');
const reporter = await createTestAccount(harness);
const invite = await createChannelInvite(harness, account.token, channelId);
await acceptInvite(harness, reporter.token, invite.code);
@@ -306,20 +276,6 @@ describe('Embed attachment purge', () => {
const row = await new ReportRepository().getReport(createReportID(BigInt(report.report_id)));
const context = row!.messageContext!.find((entry) => entry.messageId.toString() === message.id);
expect(context?.attachments.map((attachment) => attachment.filename).sort()).toEqual(['image.png', 'thumb.png']);
const admin = await createAdmin();
await createBuilder(harness, admin.token)
.post('/admin/messages/ncmec-reports')
.body({
channel_id: channelId,
message_id: message.id,
attachment_id: attachmentId,
filename,
reporter_full_name: 'Embed Reporter',
confirmed_viewed: true,
source_report_id: report.report_id,
})
.expect(HTTP_STATUS.OK)
.execute();
});
it('does not expose the ownership marker on embed media flags', async () => {
-7
View File
@@ -363,13 +363,6 @@ export interface APIConfig {
crosspost?: number;
};
};
ncmec: {
enabled: boolean;
baseUrl?: string;
username?: string;
password?: string;
reporterEmail?: string;
};
}
export interface BlueskyOAuthKeyConfig {
-498
View File
@@ -1,498 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {Logger} from '@app/api/Logger';
import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
import * as FetchUtils from '@app/api/utils/FetchUtils';
import {ms} from 'itty-time';
const NCMEC_REQUEST_TIMEOUT_MS = ms('2 minutes');
type NcmecOperation = 'report' | 'evidence' | 'fileinfo' | 'finish' | 'retract';
type NcmecApiConfig =
| {
enabled: true;
baseUrl: string;
username: string;
password: string;
}
| {
enabled: false;
baseUrl: null;
username: null;
password: null;
};
interface NcmecApiDeps {
config: NcmecApiConfig;
fetch: typeof fetch;
}
class NcmecRequestError extends Error {
constructor(
message: string,
public readonly httpStatus: number,
public readonly responseCode: number | null,
public readonly responseDescription: string | null,
public readonly requestId: string | null,
public readonly body: string | null,
) {
super(message);
}
}
interface NcmecResponseMeta {
responseCode: number | null;
responseDescription: string | null;
requestId: string | null;
}
export interface NcmecApiClient {
submitReport(reportXml: string): Promise<string>;
uploadEvidence(
reportId: string,
body: Uint8Array,
filename: string,
): Promise<{
fileId: string;
md5: string | null;
}>;
submitFileDetails(fileDetailsXml: string): Promise<void>;
finish(reportId: string): Promise<{
reportId: string;
fileIds: Array<string>;
}>;
retract(reportId: string): Promise<void>;
}
export class NcmecReporter implements NcmecApiClient {
constructor(private readonly deps: NcmecApiDeps) {}
async submitReport(reportXml: string): Promise<string> {
return this.telemetered('report', async () => {
const {res, text} = await this.postXml('/submit', reportXml);
this.assertOk(res, text);
const reportId = takeTag(text, 'reportId');
if (!reportId) throw this.fail('NCMEC /submit returned no reportId.', res, text);
return reportId;
});
}
async uploadEvidence(
reportId: string,
body: Uint8Array,
filename: string,
): Promise<{
fileId: string;
md5: string | null;
}> {
return this.telemetered('evidence', async () => {
const form = new FormData();
form.append('id', reportId);
const blobBytes = new Uint8Array(body.byteLength);
blobBytes.set(body);
form.append('file', new Blob([blobBytes]), filename);
const {res, text} = await this.postForm('/upload', form);
this.assertOk(res, text);
const fileId = takeTag(text, 'fileId');
if (!fileId) throw this.fail('NCMEC /upload returned no fileId.', res, text);
return {fileId, md5: takeTag(text, 'hash')};
});
}
async submitFileDetails(fileDetailsXml: string): Promise<void> {
await this.telemetered('fileinfo', async () => {
const {res, text} = await this.postXml('/fileinfo', fileDetailsXml);
this.assertOk(res, text);
});
}
async finish(reportId: string): Promise<{
reportId: string;
fileIds: Array<string>;
}> {
return this.telemetered('finish', async () => {
const form = new FormData();
form.append('id', reportId);
const {res, text} = await this.postForm('/finish', form);
this.assertOk(res, text);
const returned = takeTag(text, 'reportId');
if (!returned) throw this.fail('NCMEC /finish returned no reportId.', res, text);
return {reportId: returned, fileIds: takeTags(text, 'fileId')};
});
}
async retract(reportId: string): Promise<void> {
await this.telemetered('retract', async () => {
const form = new FormData();
form.append('id', reportId);
const {res, text} = await this.postForm('/retract', form);
this.assertOk(res, text);
});
}
private postXml(
path: string,
body: string,
): Promise<{
res: Response;
text: string;
}> {
return this.post(path, {body, headers: {'Content-Type': 'text/xml; charset=utf-8'}});
}
private postForm(
path: string,
form: FormData,
): Promise<{
res: Response;
text: string;
}> {
return this.post(path, {body: form});
}
private async post(
path: string,
init: RequestInit,
): Promise<{
res: Response;
text: string;
}> {
const cfg = this.requireEnabled();
const res = await this.deps.fetch(`${cfg.baseUrl}${path}`, {
method: 'POST',
...init,
headers: {
Authorization: basicAuth(cfg.username, cfg.password),
...(init.headers ?? {}),
},
signal: AbortSignal.timeout(NCMEC_REQUEST_TIMEOUT_MS),
});
const text = await FetchUtils.streamToStringWithLimit(res.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.ncmecResponseBytes,
headers: res.headers,
url: res.url,
description: 'NCMEC response',
}).catch(() => '');
return {res, text};
}
private assertOk(res: Response, body: string): void {
const meta = parseResponseMeta(res, body);
if (res.ok && meta.responseCode === 0) return;
throw this.fail(
meta.responseCode === null
? `NCMEC request failed (http ${res.status}).`
: `NCMEC request failed (http ${res.status}, responseCode ${meta.responseCode}).`,
res,
body,
meta,
);
}
private fail(message: string, res: Response, body: string, meta = parseResponseMeta(res, body)): NcmecRequestError {
Logger.warn(
{
status: res.status,
responseCode: meta.responseCode,
description: meta.responseDescription,
body: body || '<no body>',
},
'NCMEC request failed',
);
return new NcmecRequestError(
message,
res.status,
meta.responseCode,
meta.responseDescription,
meta.requestId,
body || null,
);
}
private requireEnabled(): Extract<
NcmecApiConfig,
{
enabled: true;
}
> {
if (!this.deps.config.enabled) throw new Error('NCMEC reporting is disabled.');
return this.deps.config;
}
private async telemetered<T>(operation: NcmecOperation, fn: () => Promise<T>): Promise<T> {
if (!this.deps.config.enabled) {
throw new Error('NCMEC reporting is disabled.');
}
try {
const out = await fn();
return out;
} catch (error) {
Logger.error({error, operation}, 'NCMEC operation failed');
throw error;
}
}
}
export function createNcmecApiConfig(): NcmecApiConfig {
if (!Config.ncmec.enabled) return {enabled: false, baseUrl: null, username: null, password: null};
return {
enabled: true,
baseUrl: normaliseBaseUrl(Config.ncmec.baseUrl),
username: requireConfigValue(Config.ncmec.username, 'NCMEC username'),
password: requireConfigValue(Config.ncmec.password, 'NCMEC password'),
};
}
function basicAuth(user: string, pass: string): string {
return `Basic ${Buffer.from(`${user}:${pass}`, 'utf-8').toString('base64')}`;
}
function normaliseBaseUrl(raw: string | undefined | null): string {
const trimmed = (raw ?? '').trim();
if (!trimmed) throw new Error('NCMEC base URL is required when reporting is enabled.');
return trimmed.replace(/\/+$/, '');
}
function requireConfigValue(value: string | undefined | null, label: string): string {
const trimmed = (value ?? '').trim();
if (!trimmed) throw new Error(`${label} is required when reporting is enabled.`);
return trimmed;
}
function requireNonBlank(value: string | undefined | null, label: string): string {
const trimmed = (value ?? '').trim();
if (!trimmed) throw new Error(`${label} is required.`);
return trimmed;
}
function takeTag(xml: string, tag: string): string | null {
if (!xml) return null;
const match = xml.match(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'i'));
return match?.[1]?.trim() || null;
}
function takeTags(xml: string, tag: string): Array<string> {
if (!xml) return [];
return [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'gi'))]
.map((match) => match[1]?.trim() ?? '')
.filter(Boolean);
}
function parseResponseCode(value: string | null): number | null {
if (!value) return null;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? parsed : null;
}
function parseResponseMeta(res: Response, body: string): NcmecResponseMeta {
return {
responseCode: parseResponseCode(takeTag(body, 'responseCode')),
responseDescription: takeTag(body, 'responseDescription'),
requestId: res.headers.get('Request-ID'),
};
}
interface NcmecReportedUser {
id: bigint | null;
screenName: string | null;
displayName: string | null;
espService: string | null;
permanentlyDisabledAt: Date | null;
person: NcmecReportedUserPerson | null;
ipCaptureEvents: ReadonlyArray<NcmecIpCaptureEvent>;
}
interface NcmecReportedUserPerson {
emails: ReadonlyArray<NcmecContactEmail>;
phones: ReadonlyArray<NcmecContactPhone>;
dateOfBirth: string | null;
}
interface NcmecContactEmail {
address: string;
type?: 'Home' | 'Work' | 'Business' | 'Recovery';
verified?: boolean | null;
}
interface NcmecContactPhone {
number: string;
type?: 'Mobile' | 'Home' | 'Business' | 'Work' | 'Fax' | 'Internet' | 'Recovery';
verified?: boolean | null;
}
interface NcmecIpCaptureEvent {
ipAddress: string;
eventName?: 'Login' | 'Registration' | 'Purchase' | 'Upload' | 'Other' | 'Unknown';
dateTime?: Date | null;
possibleProxy?: boolean | null;
}
interface NcmecReportXmlInput {
attachmentUrl: string;
reportedAt: Date;
reporterFullName: string;
reporterEmail: string;
reportedUser: NcmecReportedUser;
priorNcmecReportIds: ReadonlyArray<string>;
additionalInfo: string | null;
}
interface NcmecFileDetailsXmlInput {
reportId: string;
fileId: string;
filename: string;
uploadedToEspTimestamp: Date | null;
fileViewedByEsp?: boolean;
ipCaptureEvent: NcmecIpCaptureEvent | null;
additionalInfo: string | null;
}
export function buildNcmecReportXml(input: NcmecReportXmlInput): string {
const schemaLocation = `${schemaRoot()}/xsd`;
const {firstName, lastName} = splitName(input.reporterFullName);
const reporterEmail = requireNonBlank(input.reporterEmail, 'NCMEC reporter email');
const screenName = input.reportedUser.screenName
? ` <screenName>${escapeXml(input.reportedUser.screenName)}</screenName>\n`
: '';
const displayName = input.reportedUser.displayName
? ` <displayName>${escapeXml(input.reportedUser.displayName)}</displayName>\n`
: '';
const reportedUserPerson = renderReportedUserPerson(input.reportedUser.person ?? null);
const espIdentifier =
input.reportedUser.id !== null
? ` <espIdentifier>${escapeXml(input.reportedUser.id.toString())}</espIdentifier>\n`
: '';
const espService = input.reportedUser.espService
? ` <espService>${escapeXml(input.reportedUser.espService)}</espService>\n`
: '';
const ipCaptureEvents = (input.reportedUser.ipCaptureEvents ?? [])
.map((event) => renderIpCaptureEvent(event, ' '))
.join('');
const priorReports = input.priorNcmecReportIds
.map((id) => ` <priorCTReports>${escapeXml(id)}</priorCTReports>\n`)
.join('');
const disabled = input.reportedUser.permanentlyDisabledAt
? ` <accountPermanentlyDisabled disabledDate="${escapeXml(input.reportedUser.permanentlyDisabledAt.toISOString())}" userNotified="false">true</accountPermanentlyDisabled>\n`
: '';
const allEmailsReported =
input.reportedUser.person && input.reportedUser.person.emails.length > 0
? ' <allEmailsReported>true</allEmailsReported>\n'
: '';
const reportAdditionalInfo = input.additionalInfo
? ` <additionalInfo>${escapeXml(input.additionalInfo)}</additionalInfo>\n`
: '';
return `<?xml version="1.0" encoding="UTF-8"?>
<report xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="${escapeXml(schemaLocation)}">
<incidentSummary>
<incidentType>Child Pornography (possession, manufacture, and distribution)</incidentType>
<platform>Fluxer</platform>
<incidentDateTime>${escapeXml(input.reportedAt.toISOString())}</incidentDateTime>
</incidentSummary>
<internetDetails>
<webPageIncident>
<url>${escapeXml(input.attachmentUrl)}</url>
</webPageIncident>
</internetDetails>
<reporter>
<reportingPerson>
<firstName>${escapeXml(firstName)}</firstName>
<lastName>${escapeXml(lastName)}</lastName>
<email>${escapeXml(reporterEmail)}</email>
</reportingPerson>
</reporter>
<personOrUserReported>
${reportedUserPerson}${espIdentifier}${espService}${screenName}${displayName}${ipCaptureEvents}${priorReports}${disabled}${allEmailsReported} </personOrUserReported>
${reportAdditionalInfo}</report>`;
}
export function buildNcmecFileDetailsXml(input: NcmecFileDetailsXmlInput): string {
const schemaLocation = `${schemaRoot()}/xsd`;
const uploadedToEspTimestamp = input.uploadedToEspTimestamp
? ` <uploadedToEspTimestamp>${escapeXml(input.uploadedToEspTimestamp.toISOString())}</uploadedToEspTimestamp>\n`
: '';
const fileViewedByEsp = input.fileViewedByEsp === true ? ' <fileViewedByEsp>true</fileViewedByEsp>\n' : '';
const ipCaptureEvent = input.ipCaptureEvent ? renderIpCaptureEvent(input.ipCaptureEvent, ' ') : '';
const additionalInfo = input.additionalInfo
? ` <additionalInfo>${escapeXml(input.additionalInfo)}</additionalInfo>\n`
: '';
return `<?xml version="1.0" encoding="UTF-8"?>
<fileDetails xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="${escapeXml(schemaLocation)}">
<reportId>${escapeXml(input.reportId)}</reportId>
<fileId>${escapeXml(input.fileId)}</fileId>
<originalFileName>${escapeXml(input.filename)}</originalFileName>
${uploadedToEspTimestamp}${fileViewedByEsp}${ipCaptureEvent}${additionalInfo}</fileDetails>`;
}
function schemaRoot(): string {
const trimmed = (Config.ncmec.baseUrl ?? '').trim().replace(/\/+$/, '');
return trimmed || 'https://report.cybertip.org/ispws';
}
function splitName(fullName: string): {
firstName: string;
lastName: string;
} {
const parts = fullName.trim().split(/\s+/).filter(Boolean);
if (parts.length === 0) return {firstName: 'Unknown', lastName: 'Reporter'};
if (parts.length === 1) return {firstName: parts[0]!, lastName: 'Reporter'};
return {firstName: parts[0]!, lastName: parts.slice(1).join(' ')};
}
function renderReportedUserPerson(input: NcmecReportedUserPerson | null): string {
if (!input) return '';
const phones = input.phones.map((phone) => renderPhone(phone)).join('');
const emails = input.emails.map((email) => renderEmail(email)).join('');
const dateOfBirth = input.dateOfBirth
? ` <dateOfBirth>${escapeXml(input.dateOfBirth)}</dateOfBirth>\n`
: '';
if (!phones && !emails && !dateOfBirth) {
return '';
}
return ` <personOrUserReportedPerson>\n${phones}${emails}${dateOfBirth} </personOrUserReportedPerson>\n`;
}
function renderPhone(phone: NcmecContactPhone): string {
const attrs: Array<string> = [];
if (phone.type) {
attrs.push(`type="${escapeXml(phone.type)}"`);
}
if (typeof phone.verified === 'boolean') {
attrs.push(`verified="${phone.verified ? 'true' : 'false'}"`);
}
const attrText = attrs.length > 0 ? ` ${attrs.join(' ')}` : '';
return ` <phone${attrText}>${escapeXml(phone.number)}</phone>\n`;
}
function renderEmail(email: NcmecContactEmail): string {
const attrs: Array<string> = [];
if (email.type) {
attrs.push(`type="${escapeXml(email.type)}"`);
}
if (typeof email.verified === 'boolean') {
attrs.push(`verified="${email.verified ? 'true' : 'false'}"`);
}
const attrText = attrs.length > 0 ? ` ${attrs.join(' ')}` : '';
return ` <email${attrText}>${escapeXml(email.address)}</email>\n`;
}
function renderIpCaptureEvent(event: NcmecIpCaptureEvent, indent: string): string {
const eventName = event.eventName ? `${indent} <eventName>${escapeXml(event.eventName)}</eventName>\n` : '';
const dateTime = event.dateTime
? `${indent} <dateTime>${escapeXml(event.dateTime.toISOString())}</dateTime>\n`
: '';
const possibleProxy =
typeof event.possibleProxy === 'boolean'
? `${indent} <possibleProxy>${event.possibleProxy ? 'true' : 'false'}</possibleProxy>\n`
: '';
return `${indent}<ipCaptureEvent>\n${indent} <ipAddress>${escapeXml(event.ipAddress)}</ipAddress>\n${eventName}${dateTime}${possibleProxy}${indent}</ipCaptureEvent>\n`;
}
function escapeXml(value: string): string {
return value
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
.replaceAll('"', '&quot;')
.replaceAll("'", '&apos;');
}
@@ -1,35 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {AttachmentID, UserID} from '@app/api/BrandedTypes';
import {fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import type {NcmecAttachmentSubmissionRow, NcmecUserWorkflowRow} from '@app/api/database/types/CsamTypes';
import {NcmecAttachmentSubmissions, NcmecUserWorkflows} from '@app/api/Tables';
const GET_ATTACHMENT_SUBMISSION_QUERY = NcmecAttachmentSubmissions.select({
where: NcmecAttachmentSubmissions.where.eq('attachment_id'),
limit: 1,
});
const GET_USER_WORKFLOW_QUERY = NcmecUserWorkflows.select({
where: NcmecUserWorkflows.where.eq('user_id'),
limit: 1,
});
export class NcmecRepository {
async getAttachmentSubmission(attachmentId: AttachmentID): Promise<NcmecAttachmentSubmissionRow | null> {
return await fetchOne<NcmecAttachmentSubmissionRow>(
GET_ATTACHMENT_SUBMISSION_QUERY.bind({attachment_id: BigInt(attachmentId)}),
);
}
async upsertAttachmentSubmission(row: NcmecAttachmentSubmissionRow): Promise<void> {
await upsertOne(NcmecAttachmentSubmissions.insert(row));
}
async getUserWorkflow(userId: UserID): Promise<NcmecUserWorkflowRow | null> {
return await fetchOne<NcmecUserWorkflowRow>(GET_USER_WORKFLOW_QUERY.bind({user_id: BigInt(userId)}));
}
async upsertUserWorkflow(row: NcmecUserWorkflowRow): Promise<void> {
await upsertOne(NcmecUserWorkflows.insert(row));
}
}
File diff suppressed because it is too large. Load diff
@@ -1,202 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
buildNcmecFileDetailsXml,
buildNcmecReportXml,
createNcmecApiConfig,
NcmecReporter,
} from '@app/api/csam/NcmecReporter';
import {INVALID_REPORT_XML, MALFORMED_REPORT_XML} from '@app/api/test/fixtures/ncmec/NcmecXmlFixtures';
import {fakeNcmecServer} from '@app/api/test/Setup';
import {describe, expect, test} from 'vitest';
function reporter(): NcmecReporter {
return new NcmecReporter({config: createNcmecApiConfig(), fetch});
}
function validReportXml(overrides: {priorNcmecReportIds?: Array<string>} = {}): string {
return buildNcmecReportXml({
attachmentUrl: 'https://cdn.example.com/x.png',
reportedAt: new Date('2026-04-15T12:00:00Z'),
reporterFullName: 'Lilith Example',
reporterEmail: '[email protected]',
reportedUser: {
id: 42n,
screenName: 'evil-user',
displayName: 'Evil User',
espService: 'Fluxer',
permanentlyDisabledAt: null,
person: {
emails: [{address: '[email protected]', type: 'Home', verified: true}],
phones: [{number: '+15551234567', type: 'Mobile'}],
dateOfBirth: '1999-02-03',
},
ipCaptureEvents: [
{
ipAddress: '203.0.113.10',
eventName: 'Upload',
dateTime: new Date('2026-04-15T11:55:00Z'),
},
],
},
priorNcmecReportIds: overrides.priorNcmecReportIds ?? [],
additionalInfo: 'Test report.',
});
}
describe('NcmecReporter', () => {
test('submitReport returns a numeric report ID', async () => {
const reportId = await reporter().submitReport(validReportXml());
expect(reportId).toMatch(/^\d+$/);
expect(fakeNcmecServer.getReport(reportId)).not.toBeNull();
});
test('uploadEvidence returns the server-computed md5', async () => {
const r = reporter();
const reportId = await r.submitReport(validReportXml());
const upload = await r.uploadEvidence(reportId, new Uint8Array([1, 2, 3]), 'evidence.png');
expect(upload.fileId).toMatch(/^file-\d+$/);
expect(upload.md5).toMatch(/^[0-9a-f]{32}$/);
});
test('submitFileDetails marks the file as having details submitted', async () => {
const r = reporter();
const reportId = await r.submitReport(validReportXml());
const {fileId} = await r.uploadEvidence(reportId, new Uint8Array([4, 5, 6]), 'evidence.png');
await r.submitFileDetails(
buildNcmecFileDetailsXml({
reportId,
fileId,
filename: 'evidence.png',
uploadedToEspTimestamp: new Date('2026-04-15T11:55:00Z'),
fileViewedByEsp: true,
ipCaptureEvent: {
ipAddress: '203.0.113.10',
eventName: 'Upload',
dateTime: new Date('2026-04-15T11:55:00Z'),
},
additionalInfo: 'context',
}),
);
const record = fakeNcmecServer.getReport(reportId)!;
expect(record.files.find((f) => f.fileId === fileId)?.submittedDetails).toBe(true);
});
test('submitFileDetails throws when file details fail documented validation', async () => {
await expect(
reporter().submitFileDetails(`<?xml version="1.0" encoding="UTF-8"?>
<fileDetails xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://report.cybertip.org/ispws/xsd">
<originalFileName>missing-ids.jpg</originalFileName>
</fileDetails>`),
).rejects.toThrow('responseCode 4100');
});
test('finish returns every uploaded file id', async () => {
const r = reporter();
const reportId = await r.submitReport(validReportXml());
const a = await r.uploadEvidence(reportId, new Uint8Array([7]), 'a.png');
const b = await r.uploadEvidence(reportId, new Uint8Array([8]), 'b.png');
const result = await r.finish(reportId);
expect(result.reportId).toBe(reportId);
expect(result.fileIds).toEqual([a.fileId, b.fileId]);
});
test('retract marks the report as retracted', async () => {
const r = reporter();
const reportId = await r.submitReport(validReportXml());
await r.retract(reportId);
expect(fakeNcmecServer.getReport(reportId)!.retracted).toBe(true);
});
test('submitReport throws when the server rejects the XML', async () => {
await expect(reporter().submitReport(INVALID_REPORT_XML)).rejects.toThrow('responseCode 4100');
});
test('submitReport surfaces the documented malformed-xml response code', async () => {
await expect(reporter().submitReport(MALFORMED_REPORT_XML)).rejects.toThrow('responseCode 4110');
});
test('submitReport throws when the server returns 401', async () => {
fakeNcmecServer.setFailure('unauthorized');
await expect(reporter().submitReport(validReportXml())).rejects.toThrow(/http 401/);
fakeNcmecServer.setFailure('none');
});
test('buildNcmecReportXml emits personOrUserReported with priorCTReports and espIdentifier', () => {
const xml = validReportXml({priorNcmecReportIds: ['NCMEC-100', 'NCMEC-200']});
expect(xml).toContain('<personOrUserReported>');
expect(xml).toContain('<personOrUserReportedPerson>');
expect(xml).toContain('<espIdentifier>42</espIdentifier>');
expect(xml).toContain('<espService>Fluxer</espService>');
expect(xml).toContain('<screenName>evil-user</screenName>');
expect(xml).toContain('<displayName>Evil User</displayName>');
expect(xml).toContain('<email type="Home" verified="true">[email protected]</email>');
expect(xml).toContain('<phone type="Mobile">+15551234567</phone>');
expect(xml).toContain('<dateOfBirth>1999-02-03</dateOfBirth>');
expect(xml).toContain('<ipAddress>203.0.113.10</ipAddress>');
expect(xml).toContain('<eventName>Upload</eventName>');
expect(xml).toContain('<priorCTReports>NCMEC-100</priorCTReports>');
expect(xml).toContain('<priorCTReports>NCMEC-200</priorCTReports>');
expect(xml).toContain('<allEmailsReported>true</allEmailsReported>');
expect(xml).toContain('<incidentType>Child Pornography (possession, manufacture, and distribution)</incidentType>');
expect(xml).toContain('<platform>Fluxer</platform>');
});
test('buildNcmecReportXml emits incident additionalInfo at the report root', () => {
const xml = validReportXml();
expect(xml).toContain('<additionalInfo>Test report.</additionalInfo>');
expect(xml).toMatch(/<\/personOrUserReported>\s*<additionalInfo>Test report\.<\/additionalInfo>\s*<\/report>/);
});
test('buildNcmecReportXml emits accountPermanentlyDisabled when the user was banned', () => {
const xml = buildNcmecReportXml({
attachmentUrl: 'https://cdn.example.com/x.png',
reportedAt: new Date('2026-04-15T12:00:00Z'),
reporterFullName: 'Lilith Example',
reporterEmail: '[email protected]',
reportedUser: {
id: 42n,
screenName: null,
displayName: null,
espService: null,
permanentlyDisabledAt: new Date('2026-04-14T08:00:00Z'),
person: null,
ipCaptureEvents: [],
},
priorNcmecReportIds: [],
additionalInfo: null,
});
expect(xml).toContain('<accountPermanentlyDisabled disabledDate="2026-04-14T08:00:00.000Z"');
});
test('buildNcmecFileDetailsXml emits upload metadata for the uploaded file', () => {
const xml = buildNcmecFileDetailsXml({
reportId: 'NCMEC-100',
fileId: 'file-100',
filename: 'evidence.png',
uploadedToEspTimestamp: new Date('2026-04-15T11:55:00Z'),
fileViewedByEsp: true,
ipCaptureEvent: {
ipAddress: '203.0.113.10',
eventName: 'Upload',
dateTime: new Date('2026-04-15T11:55:00Z'),
},
additionalInfo: 'Uploaded from Fluxer',
});
expect(xml).toContain('<uploadedToEspTimestamp>2026-04-15T11:55:00.000Z</uploadedToEspTimestamp>');
expect(xml).toContain('<fileViewedByEsp>true</fileViewedByEsp>');
expect(xml).toContain('<ipCaptureEvent>');
expect(xml).toContain('<ipAddress>203.0.113.10</ipAddress>');
expect(xml).toContain('<additionalInfo>Uploaded from Fluxer</additionalInfo>');
});
test('buildNcmecReportXml requires a reporter email address', () => {
expect(() =>
buildNcmecReportXml({
attachmentUrl: 'https://cdn.example.com/x.png',
reportedAt: new Date('2026-04-15T12:00:00Z'),
reporterFullName: 'Lilith Example',
reporterEmail: '' as string,
reportedUser: {
id: 42n,
screenName: null,
displayName: null,
espService: null,
permanentlyDisabledAt: null,
person: null,
ipCaptureEvents: [],
},
priorNcmecReportIds: [],
additionalInfo: null,
}),
).toThrow('NCMEC reporter email');
});
});
@@ -1,75 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export interface NcmecAttachmentSubmissionRow {
attachment_id: bigint;
user_id: bigint | null;
channel_id: bigint;
message_id: bigint;
filename: string;
source_report_id: bigint | null;
status: string;
ncmec_report_id: string | null;
ncmec_file_id: string | null;
submitted_at: Date | null;
submitted_by_admin_id: bigint | null;
reporter_full_name: string | null;
failure_reason: string | null;
archive_completed_at: Date | null;
content_deleted_at: Date | null;
created_at: Date;
updated_at: Date;
}
export const NCMEC_ATTACHMENT_SUBMISSION_COLUMNS = [
'attachment_id',
'user_id',
'channel_id',
'message_id',
'filename',
'source_report_id',
'status',
'ncmec_report_id',
'ncmec_file_id',
'submitted_at',
'submitted_by_admin_id',
'reporter_full_name',
'failure_reason',
'archive_completed_at',
'content_deleted_at',
'created_at',
'updated_at',
] as const satisfies ReadonlyArray<keyof NcmecAttachmentSubmissionRow>;
export interface NcmecUserWorkflowRow {
user_id: bigint;
deleted_at: Date | null;
deleted_by_admin_id: bigint | null;
deletion_private_reason: string | null;
deletion_ncmec_report_id: string | null;
archive_id: bigint | null;
archive_requested_at: Date | null;
archive_requested_by_admin_id: bigint | null;
archive_audit_log_reason: string | null;
archive_completed_at: Date | null;
deletion_job_queued_at: Date | null;
previous_report_ids: Set<string> | null;
created_at: Date;
updated_at: Date;
}
export const NCMEC_USER_WORKFLOW_COLUMNS = [
'user_id',
'deleted_at',
'deleted_by_admin_id',
'deletion_private_reason',
'deletion_ncmec_report_id',
'archive_id',
'archive_requested_at',
'archive_requested_by_admin_id',
'archive_audit_log_reason',
'archive_completed_at',
'deletion_job_queued_at',
'previous_report_ids',
'created_at',
'updated_at',
] as const satisfies ReadonlyArray<keyof NcmecUserWorkflowRow>;
@@ -37,7 +37,6 @@ const EXPECTED_LOADS: Readonly<Record<string, readonly [number, Coverage]>> = {
'channel/services/message/UserMessageDeletionService.ts': [2, 'admin_or_system'],
'channel/services/thread/ThreadCreationService.ts': [4, 'gated'],
'channel/services/thread/ThreadDeletionService.ts': [1, 'admin_or_system'],
'csam/NcmecSubmissionService.ts': [1, 'admin_or_system'],
'guild/services/channel/ChannelOperationsService.ts': [2, 'gated'],
'guild/services/GuildDiscoveryService.ts': [1, 'gated'],
'guild/services/data/GuildOperationsService.ts': [2, 'gated'],
@@ -92,7 +92,6 @@ import {
getKVBulkMessageDeletionQueue,
getKVThreadAutoArchiveQueue,
getLimitConfigService,
getNcmecSubmissionService,
getOAuth2TokenRepository,
getPasswordChangeRepository,
getPremiumStateReconciliationQueueService,
@@ -428,10 +427,6 @@ class RequestServices implements RequestScopedServices {
return getLimitConfigService();
}
get ncmecSubmissionService() {
return getNcmecSubmissionService();
}
get oauth2TokenRepository() {
return getOAuth2TokenRepository();
}
@@ -5,16 +5,12 @@ import {AdminApiKeyRepository} from '@app/api/admin/repositories/AdminApiKeyRepo
import {AdminArchiveRepository} from '@app/api/admin/repositories/AdminArchiveRepository';
import {AdminApiKeyService} from '@app/api/admin/services/AdminApiKeyService';
import {AdminArchiveService} from '@app/api/admin/services/AdminArchiveService';
import {AdminAuditService} from '@app/api/admin/services/AdminAuditService';
import {Config} from '@app/api/Config';
import {ChannelRepository} from '@app/api/channel/ChannelRepository';
import {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository';
import {StreamPreviewService} from '@app/api/channel/services/StreamPreviewService';
import type {APIConfig} from '@app/api/config/APIConfig';
import {ConnectionRepository} from '@app/api/connection/ConnectionRepository';
import {createNcmecApiConfig, NcmecReporter} from '@app/api/csam/NcmecReporter';
import {NcmecRepository} from '@app/api/csam/NcmecRepository';
import {NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import {DonationRepository} from '@app/api/donation/DonationRepository';
import {createEmailProvider} from '@app/api/email/EmailProviderFactory';
import {FavoriteMemeRepository} from '@app/api/favorite_meme/FavoriteMemeRepository';
@@ -321,40 +317,7 @@ export function getKVAccountDeletionQueue(): KVAccountDeletionQueueService {
}
export const getThemeService = singleton(() => new ThemeService(getStorageService()));
const getNcmecReporter = singleton(() => new NcmecReporter({config: createNcmecApiConfig(), fetch}));
const getNcmecRepository = singleton(() => new NcmecRepository());
export const getAttachmentUploadTraceRepository = singleton(() => new AttachmentUploadTraceRepository());
export const getNcmecSubmissionService = singleton(
() =>
new NcmecSubmissionService({
reportRepository: getReportRepository(),
ncmecApi: getNcmecReporter(),
ncmecRepository: getNcmecRepository(),
attachmentUploadTraceRepository: getAttachmentUploadTraceRepository(),
storageService: getStorageService(),
channelRepository: getChannelRepository(),
userRepository: getUserRepository(),
guildRepository: getGuildRepository(),
gatewayService: getGatewayService(),
userCacheService: createUserCacheService(),
adminArchiveService: new AdminArchiveService(
getAdminArchiveRepository(),
getUserRepository(),
getGuildRepository(),
getStorageService(),
getSnowflakeService(),
getWorkerService(),
),
adminAuditService: new AdminAuditService(getAdminRepository(), getSnowflakeService(), {
userRepository: getUserRepository(),
guildRepository: getGuildRepository(),
channelRepository: getChannelRepository(),
}),
purgeQueue: getPurgeQueue(),
workerService: getWorkerService(),
deletionQueue: getKVAccountDeletionQueue(),
}),
);
let _virusScanInitPromise: Promise<void> | null = null;
@@ -48,7 +48,6 @@ const REQUEST_SERVICE_VARIABLES: ReadonlyArray<keyof HonoEnv['Variables']> = [
'limitConfigService',
'mediaService',
'messageRequestService',
'ncmecSubmissionService',
'oauth2ApplicationsRequestService',
'oauth2RequestService',
'oauth2Service',
-4
View File
@@ -39734,8 +39734,6 @@
"MAX_THREAD_MEMBERS",
"MAX_WEBHOOKS_PER_CHANNEL",
"MAX_WEBHOOKS_PER_GUILD",
"NCMEC_ALREADY_SUBMITTED",
"NCMEC_SUBMISSION_FAILED",
"MEDIA_METADATA_ERROR",
"METHOD_NOT_ALLOWED",
"MISSING_ACCESS",
@@ -40031,8 +40029,6 @@
"This thread has reached its member limit of {count}.",
"You've reached the maximum of {count, plural, one {# webhook} other {# webhooks}} per channel.",
"You've reached the maximum of {count, plural, one {# webhook} other {# webhooks}} per community.",
"This content has already been submitted to NCMEC.",
"We couldn't submit the report to NCMEC. Please try again later.",
"Media metadata error.",
"Method not allowed.",
"You don't have access to this resource or feature.",
@@ -11,7 +11,7 @@ import {
} from '@app/api/auth/tests/AuthTestUtils';
import {createGuildID, createReportID, createUserID} from '@app/api/BrandedTypes';
import {authorizeBot, createTestBotAccount} from '@app/api/bot/tests/BotTestUtils';
import {Config} from '@app/api/Config';
import {Config, getConfig} from '@app/api/Config';
import {loadFixture} from '@app/api/channel/tests/AttachmentTestUtils';
import {
acceptInvite,
@@ -27,7 +27,7 @@ import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils';
import {GuildMemberRepository} from '@app/api/guild/repositories/GuildMemberRepository';
import {resetActivityEventsForTests, startActivityEvents} from '@app/api/infrastructure/activity/ActivityEvents';
import type {ActivityPublisher} from '@app/api/infrastructure/activity/ActivitySpool';
import {getRateLimitService} from '@app/api/middleware/ServiceSingletons';
import {getInstanceConfigRepository, getRateLimitService} from '@app/api/middleware/ServiceSingletons';
import {getReportFlowVariant, type ReportFlowStepInput} from '@app/api/report/flows/ReportFlowRegistry';
import {ReportRepository} from '@app/api/report/ReportRepository';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
@@ -46,6 +46,7 @@ import {
executeWebhook,
executeWebhookWithAttachments,
} from '@app/api/webhook/tests/WebhookTestUtils';
import {AccountIdentityModes, TagStyles} from '@fluxer/constants/src/AccountIdentityConstants';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
@@ -336,14 +337,7 @@ async function expectStoredAuthor(reportId: string, world: World, kind: AuthorKi
}
async function createAdmin(harness: ApiTestHarness): Promise<TestAccount> {
return setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.REPORT_VIEW,
AdminACLs.CSAM_SUBMIT_NCMEC,
AdminACLs.MESSAGE_DELETE,
AdminACLs.USER_DELETE,
AdminACLs.ARCHIVE_TRIGGER_USER,
]);
return setUserACLs(harness, await createTestAccount(harness), [AdminACLs.AUTHENTICATE, AdminACLs.REPORT_VIEW]);
}
describe('Reports of webhook and bot messages', () => {
@@ -577,6 +571,33 @@ describe('Reports of webhook and bot messages', () => {
});
});
test('the webhook creator tag follows the tag style of the instance', async () => {
const config = getConfig();
const originalSelfHosted = config.instance.selfHosted;
config.instance.selfHosted = true;
try {
await getInstanceConfigRepository().setAccountIdentityMode(AccountIdentityModes.EMAIL, 'setup', TagStyles.NONE);
const world = await setupWorld(harness);
const report = await submitFlow(harness, world.reporter.token, world.channelId, world.messages.webhook)
.expect(HTTP_STATUS.OK)
.execute();
const admin = await createAdmin(harness);
const detail = await createBuilder<AdminReport>(harness, admin.token)
.get(`/admin/reports/${report.report_id}`)
.expect(HTTP_STATUS.OK)
.execute();
expect(detail).toMatchObject({
reported_webhook_creator_id: world.owner.userId,
reported_webhook_creator_tag: world.creator.username,
reported_webhook_creator_username: world.creator.username,
reported_webhook_creator_discriminator: '0000',
});
} finally {
config.instance.selfHosted = originalSelfHosted;
getInstanceConfigRepository().clearCacheForTesting();
}
});
test('the admin API shows webhook fields and search filters by webhook', async () => {
const world = await setupWorld(harness);
const webhookReport = await submitFlow(harness, world.reporter.token, world.channelId, world.messages.webhook)
@@ -624,7 +645,6 @@ describe('Reports of webhook and bot messages', () => {
author_id: world.webhook.id,
author_username: WEBHOOK_USERNAME,
webhook_id: world.webhook.id,
user_prior_ncmec_report_ids: [],
});
const neighbor = detail.message_context!.find((entry) => entry.id === world.messages.user)!;
expect(neighbor).toMatchObject({author_id: world.author.userId, webhook_id: null});
@@ -751,7 +771,7 @@ describe('Reports of webhook and bot messages', () => {
});
});
test('webhook attachments are kept as evidence and can go to NCMEC through the report', async () => {
test('webhook attachments are kept as evidence after the message is deleted', async () => {
const world = await setupWorld(harness);
const {json: message} = await executeWebhookWithAttachments(harness, {
webhookId: world.webhook.id,
@@ -770,25 +790,15 @@ describe('Reports of webhook and bot messages', () => {
const key = `attachments/${world.channelId}/${attachment.id}/evidence.png`;
expect(harness.storageService.hasObject(Config.s3.buckets.reports, key)).toBe(true);
await deleteWebhookMessageByToken(harness, world.webhook.id, world.webhook.token, message!.id);
expect(harness.storageService.hasObject(Config.s3.buckets.reports, key)).toBe(true);
const admin = await createAdmin(harness);
await createBuilder(harness, admin.token)
.post('/admin/messages/ncmec-reports')
.body({
channel_id: world.channelId,
message_id: message!.id,
attachment_id: attachment.id,
filename: 'evidence.png',
reporter_full_name: 'Webhook Reporter',
confirmed_viewed: true,
source_report_id: result.report_id,
})
.expect(HTTP_STATUS.OK)
.execute();
const detail = await createBuilder<AdminReport>(harness, admin.token)
.get(`/admin/reports/${result.report_id}`)
.expect(HTTP_STATUS.OK)
.execute();
const evidence = detail.message_context!.find((entry) => entry.id === message!.id)!;
expect(evidence.attachments[0]?.ncmec_status).not.toBe('not_submitted');
expect(evidence.attachments.map((entry) => ({id: entry.id, filename: entry.filename}))).toEqual([
{id: attachment.id, filename: 'evidence.png'},
]);
});
});
-11
View File
@@ -23,7 +23,6 @@ import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
import {MockSnowflakeService} from '@app/api/test/mocks/MockSnowflakeService';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
import {RepositoryBackedMessageResponseDataService} from '@app/api/test/mocks/RepositoryBackedMessageResponseDataService';
import {fakeNcmecServer} from '@app/api/test/msw/handlers/NcmecHandlers';
import {server} from '@app/api/test/msw/server';
import {mapUserToPartialResponse} from '@app/api/user/UserMappers';
import {loadConfig} from '@fluxer/config/src/ConfigLoader';
@@ -184,13 +183,6 @@ const testApiConfig: APIConfig = {
sandboxUserIds: [],
sandboxEntitlesAll: false,
},
ncmec: {
...apiConfig.ncmec,
enabled: true,
baseUrl: fakeNcmecServer.baseUrl,
username: 'usr123',
password: 'pswd123',
},
};
testApiConfig.dev.relaxRegistrationRateLimits = true;
testApiConfig.dev.disableRateLimits = true;
@@ -211,8 +203,6 @@ setInjectedMessageResponseDataService(new RepositoryBackedMessageResponseDataSer
enableSearchTaskTracking();
export {fakeNcmecServer};
beforeAll(async () => {
server.listen({
onUnhandledRequest: 'error',
@@ -224,7 +214,6 @@ afterEach(async () => {
resetCassandraQueryExecutorForTesting();
getInstanceConfigRepository().clearCacheForTesting();
server.resetHandlers();
fakeNcmecServer.reset();
});
afterAll(async () => {
@@ -1,28 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export const INVALID_REPORT_XML = `<?xml version="1.0" encoding="UTF-8"?>
<report xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://report.cybertip.org/ispws/xsd">
<reporter>
<reportingPerson>
<firstName>John</firstName>
<lastName>Smith</lastName>
<email>jsmith@example.com</email>
</reportingPerson>
</reporter>
</report>`;
export const MALFORMED_REPORT_XML = `<?xml version="1.0" encoding="UTF-8"?>
<report xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://report.cybertip.org/ispws/xsd">
<incidentSummary>
<incidentType>Child Pornography (possession, manufacture, and distribution)</incidentType>
<incidentDateTime>2012-10-15T08:00:00-07:00</incidentDateTime>
</incidentSummary>
<reporter>
<reportingPerson>
<firstName>John</firstName>
<lastName>Smith</lastName>
<email>jsmith@example.com</email>
</reportingPerson>
</reporter>
</report`;
@@ -1,284 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import crypto from 'node:crypto';
import {XMLParser, XMLValidator} from 'fast-xml-parser';
import {HttpResponse, http, type RequestHandler} from 'msw';
type FakeNcmecFailure = 'none' | 'submit' | 'upload' | 'fileinfo' | 'finish' | 'retract' | 'unauthorized';
interface FakeNcmecFileRecord {
fileId: string;
filename: string;
size: number;
md5: string;
submittedDetails: boolean;
detailsXml: string | null;
}
interface FakeNcmecReportRecord {
reportId: string;
reportXml: string;
files: Array<FakeNcmecFileRecord>;
finished: boolean;
retracted: boolean;
authUsername: string;
}
const BASE_URL = 'https://ncmec.test/ispws';
const USERNAME = 'usr123';
const PASSWORD = 'pswd123';
const xmlParser = new XMLParser({ignoreAttributes: false, parseTagValue: false, trimValues: true});
class MswNcmecServer {
readonly baseUrl = BASE_URL;
private reportCounter = 1000000;
private fileCounter = 1;
private requestCounter = 1;
private readonly reports = new Map<string, FakeNcmecReportRecord>();
private failureMode: FakeNcmecFailure = 'none';
reset(): void {
this.reports.clear();
this.reportCounter = 1000000;
this.fileCounter = 1;
this.requestCounter = 1;
this.failureMode = 'none';
}
setFailure(mode: FakeNcmecFailure): void {
this.failureMode = mode;
}
getReports(): Array<FakeNcmecReportRecord> {
return [...this.reports.values()];
}
getReport(reportId: string): FakeNcmecReportRecord | null {
return this.reports.get(reportId) ?? null;
}
createHandlers(): Array<RequestHandler> {
return [
http.get(`${BASE_URL}/status`, ({request}) => this.handleStatus(request)),
http.get(`${BASE_URL}/xsd`, () => this.text(200, '<schema></schema>', 'text/plain; charset=utf-8')),
http.post(`${BASE_URL}/submit`, ({request}) => this.handleSubmit(request)),
http.post(`${BASE_URL}/upload`, ({request}) => this.handleUpload(request)),
http.post(`${BASE_URL}/fileinfo`, ({request}) => this.handleFileInfo(request)),
http.post(`${BASE_URL}/finish`, ({request}) => this.handleFinish(request)),
http.post(`${BASE_URL}/retract`, ({request}) => this.handleRetract(request)),
];
}
private handleStatus(request: Request): HttpResponse<string> {
const unauthorized = this.maybeUnauthorized(request);
if (unauthorized) return unauthorized;
return this.xml(
200,
ok(`<responseDescription>Remote User : ${USERNAME}, Remote Ip : 127.0.0.1</responseDescription>`),
);
}
private async handleSubmit(request: Request): Promise<HttpResponse<string>> {
const failure = this.effectiveFailure(request);
const unauthorized = this.maybeUnauthorized(request, failure);
if (unauthorized) return unauthorized;
if (failure === 'submit') return this.xml(200, errorXml(1100, 'Save failed'));
if (!isXmlContentType(request)) return this.xml(200, errorXml(4000, 'Invalid request'));
const body = await request.text();
if (XMLValidator.validate(body) !== true) return this.xml(200, errorXml(4110, 'Malformed XML submittal'));
const report = parseReportDocument(body);
if (!hasRequiredReportFields(report)) return this.xml(200, errorXml(4100, 'Validation failed'));
const reportId = String(this.reportCounter++);
this.reports.set(reportId, {
reportId,
reportXml: body,
files: [],
finished: false,
retracted: false,
authUsername: USERNAME,
});
return this.xml(200, ok(`<reportId>${reportId}</reportId>`));
}
private async handleUpload(request: Request): Promise<HttpResponse<string>> {
const failure = this.effectiveFailure(request);
const unauthorized = this.maybeUnauthorized(request, failure);
if (unauthorized) return unauthorized;
if (failure === 'upload') return this.xml(200, errorXml(1111, 'File upload failed'));
const form = await request.formData();
const reportId = String(form.get('id') ?? '');
const file = form.get('file');
const record = this.reports.get(reportId);
if (!reportId || !record) return this.xml(200, errorXml(5001, 'Report does not exist'));
if (record.retracted) return this.xml(200, errorXml(5101, 'Report already retracted'));
if (record.finished) return this.xml(200, errorXml(5102, 'Report already finished'));
if (!(file instanceof Blob)) return this.xml(200, errorXml(4200, 'Malformed file submittal'));
const buffer = Buffer.from(await file.arrayBuffer());
const md5 = crypto.createHash('md5').update(buffer).digest('hex');
const fileId = `file-${this.fileCounter++}`;
const fileRecord: FakeNcmecFileRecord = {
fileId,
filename: (file as File).name || 'upload.bin',
size: buffer.byteLength,
md5,
submittedDetails: false,
detailsXml: null,
};
record.files.push(fileRecord);
return this.xml(200, ok(`<reportId>${reportId}</reportId><fileId>${fileId}</fileId><hash>${md5}</hash>`));
}
private async handleFileInfo(request: Request): Promise<HttpResponse<string>> {
const failure = this.effectiveFailure(request);
const unauthorized = this.maybeUnauthorized(request, failure);
if (unauthorized) return unauthorized;
if (failure === 'fileinfo') return this.xml(200, errorXml(1300, 'Update failed'));
if (!isXmlContentType(request)) return this.xml(200, errorXml(4000, 'Invalid request'));
const body = await request.text();
if (XMLValidator.validate(body) !== true) return this.xml(200, errorXml(4110, 'Malformed XML submittal'));
const fileDetails = parseFileDetailsDocument(body);
if (!hasFileDetailsFields(fileDetails)) return this.xml(200, errorXml(4100, 'Validation failed'));
const reportId = nonBlankString(fileDetails?.reportId);
const fileId = nonBlankString(fileDetails?.fileId);
const record = reportId ? this.reports.get(reportId) : null;
if (!record) return this.xml(200, errorXml(5001, 'Report does not exist'));
const file = record.files.find((candidate) => candidate.fileId === fileId);
if (!file) return this.xml(200, errorXml(5002, 'File does not exist'));
file.submittedDetails = true;
file.detailsXml = body;
return this.xml(200, ok(`<reportId>${reportId}</reportId>`));
}
private async handleFinish(request: Request): Promise<HttpResponse<string>> {
const failure = this.effectiveFailure(request);
const unauthorized = this.maybeUnauthorized(request, failure);
if (unauthorized) return unauthorized;
if (failure === 'finish') return this.xml(200, errorXml(1300, 'Update failed'));
const form = await request.formData();
const reportId = String(form.get('id') ?? '');
const record = this.reports.get(reportId);
if (!record) return this.xml(200, errorXml(5001, 'Report does not exist'));
record.finished = true;
const files = record.files.map((file) => ` <fileId>${file.fileId}</fileId>`).join('\n');
return this.xml(
200,
`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<reportDoneResponse>
<responseCode>0</responseCode>
<reportId>${reportId}</reportId>
<files>
${files}
</files>
</reportDoneResponse>`,
);
}
private async handleRetract(request: Request): Promise<HttpResponse<string>> {
const failure = this.effectiveFailure(request);
const unauthorized = this.maybeUnauthorized(request, failure);
if (unauthorized) return unauthorized;
if (failure === 'retract') return this.xml(200, errorXml(1300, 'Update failed'));
const form = await request.formData();
const reportId = String(form.get('id') ?? '');
const record = this.reports.get(reportId);
if (!record) return this.xml(200, errorXml(5001, 'Report does not exist'));
record.retracted = true;
return this.xml(200, ok(`<reportId>${reportId}</reportId>`));
}
private effectiveFailure(request: Request): FakeNcmecFailure {
return ((request.headers.get('x-test-fail') ?? '') || this.failureMode) as FakeNcmecFailure;
}
private maybeUnauthorized(request: Request, failure = this.failureMode): HttpResponse<string> | null {
if (failure === 'unauthorized') return this.text(401, '');
const auth = request.headers.get('authorization') ?? '';
if (!auth.toLowerCase().startsWith('basic ')) return this.text(401, '');
const decoded = Buffer.from(auth.slice(6), 'base64').toString('utf-8');
return decoded === `${USERNAME}:${PASSWORD}` ? null : this.text(401, '');
}
private xml(status: number, body: string): HttpResponse<string> {
return this.text(status, body, 'application/xml; charset=utf-8');
}
private text(status: number, body: string, contentType = 'text/plain; charset=utf-8'): HttpResponse<string> {
return new HttpResponse(body, {
status,
headers: {
'content-type': contentType,
'Request-ID': `req-${this.requestCounter++}`,
},
});
}
}
function ok(extra: string): string {
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<reportResponse>
<responseCode>0</responseCode>
<responseDescription>Success</responseDescription>
${extra}
</reportResponse>`;
}
function errorXml(code: number, description: string): string {
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<reportResponse>
<responseCode>${code}</responseCode>
<responseDescription>${description}</responseDescription>
</reportResponse>`;
}
function isXmlContentType(request: Request): boolean {
return (request.headers.get('content-type') ?? '').toLowerCase().includes('text/xml');
}
function parseReportDocument(xml: string): Record<string, unknown> | null {
const parsed = xmlParser.parse(xml) as Record<string, unknown>;
return objectField(parsed, 'report');
}
function parseFileDetailsDocument(xml: string): Record<string, unknown> | null {
const parsed = xmlParser.parse(xml) as Record<string, unknown>;
return objectField(parsed, 'fileDetails');
}
function hasRequiredReportFields(report: Record<string, unknown> | null): boolean {
const incidentSummary = objectField(report, 'incidentSummary');
const reporter = objectField(report, 'reporter');
const reportingPerson = objectField(reporter, 'reportingPerson');
return Boolean(
nonBlankString(incidentSummary?.incidentType) &&
nonBlankString(incidentSummary?.incidentDateTime) &&
nonBlankString(reportingPerson?.email),
);
}
function hasFileDetailsFields(fileDetails: Record<string, unknown> | null): boolean {
return Boolean(nonBlankString(fileDetails?.reportId) && nonBlankString(fileDetails?.fileId));
}
function objectField(value: unknown, key: string): Record<string, unknown> | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
const field = (value as Record<string, unknown>)[key];
if (!field || typeof field !== 'object' || Array.isArray(field)) return null;
return field as Record<string, unknown>;
}
function nonBlankString(value: unknown): string | null {
if (typeof value === 'string') {
const trimmed = value.trim();
return trimmed || null;
}
if (typeof value === 'number' || typeof value === 'bigint') {
return String(value);
}
return null;
}
export const fakeNcmecServer = new MswNcmecServer();
export function createNcmecHandlers(): Array<RequestHandler> {
return fakeNcmecServer.createHandlers();
}
-2
View File
@@ -1,13 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createNcmecHandlers} from '@app/api/test/msw/handlers/NcmecHandlers';
import {createOnionooDetailsHandler} from '@app/api/test/msw/handlers/OnionooHandlers';
import {createOpenNsfwHandlers} from '@app/api/test/msw/handlers/OpenNsfwHandlers';
import {createPwnedPasswordsRangeHandler} from '@app/api/test/msw/handlers/PwnedPasswordsHandlers';
import {setupServer} from 'msw/node';
export const server = setupServer(
...createNcmecHandlers(),
...createOpenNsfwHandlers(),
createOnionooDetailsHandler(),
createPwnedPasswordsRangeHandler(),
-2
View File
@@ -18,7 +18,6 @@ import type {StreamService} from '@app/api/channel/services/StreamService';
import type {ThreadService} from '@app/api/channel/services/thread/ThreadService';
import type {ConnectionRequestService} from '@app/api/connection/ConnectionRequestService';
import type {ConnectionService} from '@app/api/connection/ConnectionService';
import type {NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import type {DonationService} from '@app/api/donation/DonationService';
import type {FavoriteMemeRequestService} from '@app/api/favorite_meme/FavoriteMemeRequestService';
import type {FavoriteMemeService} from '@app/api/favorite_meme/FavoriteMemeService';
@@ -160,7 +159,6 @@ export interface HonoEnv {
reportService: ReportService;
reportRequestService: ReportRequestService;
contactChangeLogService: UserContactChangeLogService;
ncmecSubmissionService: NcmecSubmissionService;
requestCache: RequestCache;
rpcService: RpcService;
searchService: SearchService;
@@ -20,7 +20,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
oEmbedBytes: 256 * 1024,
pwnedPasswordsBytes: 1024 * 1024,
externalTemplateBytes: 512 * 1024,
ncmecResponseBytes: 64 * 1024,
fileBlocklistBytes: 25 * 1024 * 1024,
urlBlocklistBytes: 25 * 1024 * 1024,
} as const;
@@ -9,7 +9,6 @@ import type {ChannelRepository} from '@app/api/channel/ChannelRepository';
import type {ChannelService} from '@app/api/channel/services/ChannelService';
import type {ConnectionRepository} from '@app/api/connection/ConnectionRepository';
import {ConnectionService} from '@app/api/connection/ConnectionService';
import type {NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import {DonationRepository} from '@app/api/donation/DonationRepository';
import type {IDonationRepository} from '@app/api/donation/IDonationRepository';
import type {FavoriteMemeRepository} from '@app/api/favorite_meme/FavoriteMemeRepository';
@@ -75,7 +74,6 @@ import {
getKVActivityTracker,
getKVBulkMessageDeletionQueue,
getLimitConfigService,
getNcmecSubmissionService,
getOAuth2TokenRepository,
getPremiumStateReconciliationQueueService,
getPurgeQueue,
@@ -167,7 +165,6 @@ export interface WorkerDependencies {
channelService: ChannelService;
guildAuditLogService: GuildAuditLogService;
contactChangeLogService: UserContactChangeLogService;
ncmecSubmissionService: NcmecSubmissionService;
donationRepository: IDonationRepository;
guildService: GuildService;
billingRepository: BillingRepository;
@@ -207,7 +204,6 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS
const connectionService = new ConnectionService(connectionRepository, gatewayService);
const mediaService = getMediaService();
const discriminatorService = getDiscriminatorService();
const ncmecSubmissionService = getNcmecSubmissionService();
const avatarService = getAvatarService();
const entityAssetService = getEntityAssetService();
await ensureVirusScanInitialized();
@@ -329,7 +325,6 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS
storeEntitlementService,
guildAuditLogService,
contactChangeLogService,
ncmecSubmissionService,
get stripe() {
return getStripeClient();
},
@@ -44,7 +44,6 @@ const LANE_CONFIG = {
'bulkDeleteUserMessagesScoped',
'deleteUserMessagesInGuildByTime',
'messageShred',
'finalizeNcmecAttachmentReport',
'harvestGuildData',
'harvestUserData',
'batchGuildAuditLogMessageDeletes',
@@ -61,7 +60,7 @@ const LANE_CONFIG = {
'bulkDeleteMessagesForUsers',
'drainActivitySpool',
] as const,
retiredTasks: ['sendScheduledMessage'],
retiredTasks: ['sendScheduledMessage', 'finalizeNcmecAttachmentReport'],
concurrency: 8,
maxAckPending: 50,
ackWaitMs: 60000,
@@ -20,7 +20,6 @@ import expireAttachments from '@app/api/worker/tasks/ExpireAttachments';
import expireReportSnapshots from '@app/api/worker/tasks/ExpireReportSnapshots';
import expireStaleJobs from '@app/api/worker/tasks/ExpireStaleJobs';
import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds';
import finalizeNcmecAttachmentReport from '@app/api/worker/tasks/FinalizeNcmecAttachmentReport';
import flushUserActivityBuffer from '@app/api/worker/tasks/FlushUserActivityBuffer';
import handleMentionChunk from '@app/api/worker/tasks/HandleMentionChunk';
import handleMentions from '@app/api/worker/tasks/HandleMentions';
@@ -93,7 +92,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
expireReportSnapshots,
expireStaleJobs,
extractEmbeds,
finalizeNcmecAttachmentReport,
handleMentions,
handleMentionChunk,
harvestGuildData,
@@ -1,26 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAttachmentID} from '@app/api/BrandedTypes';
import {Logger} from '@app/api/Logger';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {z} from 'zod';
const PayloadSchema = z.object({
attachmentId: z.string(),
requeueCount: z.number().int().min(0).optional(),
});
const finalizeNcmecAttachmentReport: WorkerTaskHandler = async (payload, helpers) => {
const validated = PayloadSchema.parse(payload);
helpers.logger.debug({payload: validated}, 'Processing finalizeNcmecAttachmentReport task');
const attachmentId = createAttachmentID(BigInt(validated.attachmentId));
try {
const deps = getWorkerDependencies();
await deps.ncmecSubmissionService.finalizeAttachmentReport(attachmentId, validated.requeueCount ?? 0);
} catch (error) {
Logger.error({error, attachmentId: attachmentId.toString()}, 'Failed to finalize NCMEC attachment report');
throw error;
}
};
export default finalizeNcmecAttachmentReport;
@@ -70,7 +70,7 @@ describe('WorkerLaneConfig', () => {
expect(retired).toEqual({
realtime: [],
unfurl: [],
lifecycle: ['sendScheduledMessage'],
lifecycle: ['sendScheduledMessage', 'finalizeNcmecAttachmentReport'],
batch: [],
crosspost: [],
});