fix(media): stream large files and honour size settings (#3387)

packages/schema/src/domains/channel/ChannelRequestSchemas.ts

@@ -3,7 +3,6 @@
 import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
 import {CONTENT_WARNING_TEXT_MAX_LENGTH} from '@fluxer/constants/src/GuildConstants';
 import {
-	AVATAR_MAX_SIZE,
 	CHANNEL_RATE_LIMIT_PER_USER_MAX,
 	CHANNEL_RATE_LIMIT_PER_USER_MIN,
 	CHANNEL_TOPIC_MAX_LENGTH,
@@ -33,7 +32,7 @@ import {
 import {ThreadAutoArchiveDurationSchema} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas';
 import {ReadStateResponse} from '@fluxer/schema/src/domains/gateway/GatewaySchemas';
 import {ChannelOverwriteTypeSchema, GeneralChannelNameType} from '@fluxer/schema/src/primitives/ChannelValidators';
-import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators';
+import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators';
 import {ContentWarningLevelSchema} from '@fluxer/schema/src/primitives/GuildValidators';
 import {QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators';
 import {
@@ -134,9 +133,7 @@ const ChannelUpdateCommon = ChannelCommonBase.extend({
 			'Legacy: setting true maps to nsfw_override=true; setting false maps to nsfw_override=null (inherit). Prefer nsfw_override.',
 		),
 	...ChannelContentWarningFields,
-	icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('Base64-encoded icon image for group DM channels'),
+	icon: createBase64ImageType().nullish().describe('Base64-encoded icon image for group DM channels'),
 	owner_id: SnowflakeType.nullish().describe('ID of the new owner for group DM channels'),
 	nicks: ChannelNicknameOverrides.optional().describe('Custom nicknames for users in this channel'),
 	rtc_region: createStringType(RTC_REGION_ID_MIN_LENGTH, RTC_REGION_ID_MAX_LENGTH)
@@ -257,9 +254,7 @@ const ChannelUpdateLinkRequest = ChannelUpdateCommon.extend({
 const ChannelUpdateGroupDmRequest = z.object({
 	type: createNamedLiteral(ChannelTypes.GROUP_DM, 'GROUP_DM', 'Channel type (group DM)'),
 	name: GeneralChannelNameType.nullish().describe('The name of the group DM'),
-	icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('Base64-encoded icon image for the group DM'),
+	icon: createBase64ImageType().nullish().describe('Base64-encoded icon image for the group DM'),
 	owner_id: SnowflakeType.nullish().describe('ID of the new owner of the group DM'),
 	nicks: ChannelNicknameOverrides.nullish().describe('Custom nicknames for users in this group DM'),
 	nsfw: z.boolean().nullish().describe('Whether the group DM is marked for mature content (owner only)'),

packages/schema/src/domains/guild/GuildRequestSchemas.ts

@@ -8,19 +8,16 @@ import {
 	SystemChannelFlagsDescriptions,
 } from '@fluxer/constants/src/GuildConstants';
 import {
-	AVATAR_MAX_SIZE,
-	EMOJI_MAX_SIZE,
 	MAX_GUILD_ROLES,
 	MAX_GUILD_STICKER_TAGS,
 	MAX_TEMP_BAN_DURATION_SECONDS,
 	MIN_TEMP_BAN_DURATION_SECONDS,
-	STICKER_MAX_SIZE,
 } from '@fluxer/constants/src/LimitConstants';
 import {SudoVerificationSchema} from '@fluxer/schema/src/domains/auth/AuthSchemas';
 import {GuildFeatureSchema} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
 import {TemplateSerializedGuild} from '@fluxer/schema/src/domains/guild/GuildTemplateSchemas';
 import {VanityURLCodeType} from '@fluxer/schema/src/primitives/ChannelValidators';
-import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators';
+import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators';
 import {
 	ContentWarningLevelSchema,
 	DefaultMessageNotificationsSchema,
@@ -52,9 +49,7 @@ function coerceBlankStringToNull(value: unknown): unknown {
 
 export const GuildCreateRequest = z.object({
 	name: createStringType(1, 100).describe('The name of the guild (1-100 characters)'),
-	icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('Base64-encoded image data for the guild icon'),
+	icon: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild icon'),
 	empty_features: z.boolean().optional().describe('Whether to create the guild without default features'),
 	template: TemplateSerializedGuild.optional().describe('Serialised template data to use for guild structure'),
 });
@@ -64,9 +59,7 @@ export type GuildCreateRequest = z.infer<typeof GuildCreateRequest>;
 export const GuildUpdateRequest = z
 	.object({
 		name: createStringType(1, 100).describe('The name of the guild (1-100 characters)'),
-		icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('Base64-encoded image data for the guild icon'),
+		icon: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild icon'),
 		system_channel_id: SnowflakeType.nullish().describe('The ID of the channel where system messages are sent'),
 		system_channel_flags: createBitflagInt32Type(
 			SystemChannelFlags,
@@ -108,13 +101,9 @@ export const GuildUpdateRequest = z
 			GuildExplicitContentFilterSchema,
 			'Level of content filtering for explicit media',
 		),
-		banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('Base64-encoded image data for the guild banner'),
-		splash: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('Base64-encoded image data for the guild splash screen'),
-		embed_splash: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
+		banner: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild banner'),
+		splash: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild splash screen'),
+		embed_splash: createBase64ImageType()
 			.nullish()
 			.describe('Base64-encoded image data for the embedded invite splash'),
 		splash_card_alignment: SplashCardAlignmentSchema.optional().describe(
@@ -148,12 +137,8 @@ export const GuildMemberUpdateRequest = z.object({
 		.optional()
 		.transform((ids) => (ids ? new Set(ids) : undefined))
 		.describe(`Array of role IDs to assign to the member (max ${MAX_GUILD_ROLES})`),
-	avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('Base64-encoded image data for the member guild avatar'),
-	banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('Base64-encoded image data for the member guild banner'),
+	avatar: createBase64ImageType().nullish().describe('Base64-encoded image data for the member guild avatar'),
+	banner: createBase64ImageType().nullish().describe('Base64-encoded image data for the member guild banner'),
 	bio: createStringType(1, 320).nullish().describe('The member guild profile bio (1-320 characters)'),
 	pronouns: createStringType(1, 40).nullish().describe('The member guild profile pronouns (1-40 characters)'),
 	accent_color: ColorType.nullish().describe('The accent color for the member guild profile as an integer'),
@@ -208,9 +193,7 @@ export const GuildEmojiCreateRequest = z.object({
 	name: createStringType(2, 32)
 		.refine((value) => /^[a-zA-Z0-9_]+$/.test(value), 'Emoji name can only contain letters, numbers, and underscores')
 		.describe('The name of the emoji (2-32 characters, alphanumeric and underscores only)'),
-	image: createBase64StringType(1, base64LengthForBytes(EMOJI_MAX_SIZE)).describe(
-		'Base64-encoded image data for the emoji',
-	),
+	image: createBase64ImageType().describe('Base64-encoded image data for the emoji'),
 });
 
 export type GuildEmojiCreateRequest = z.infer<typeof GuildEmojiCreateRequest>;
@@ -247,9 +230,7 @@ export const GuildStickerCreateRequest = z.object({
 		.optional()
 		.default([])
 		.describe(`Array of autocomplete/suggestion tags (max ${MAX_GUILD_STICKER_TAGS} tags, each 1-30 characters)`),
-	image: createBase64StringType(1, base64LengthForBytes(STICKER_MAX_SIZE)).describe(
-		'Base64-encoded image data for the sticker',
-	),
+	image: createBase64ImageType().describe('Base64-encoded image data for the sticker'),
 });
 
 export type GuildStickerCreateRequest = z.infer<typeof GuildStickerCreateRequest>;

packages/schema/src/domains/oauth/OAuthSchemas.ts

@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {ApplicationFlags, BotFlags, BotFlagsDescriptions} from '@fluxer/constants/src/BotConstants';
-import {AVATAR_MAX_SIZE, MAX_APPLICATION_REDIRECT_URIS} from '@fluxer/constants/src/LimitConstants';
+import {MAX_APPLICATION_REDIRECT_URIS} from '@fluxer/constants/src/LimitConstants';
 import {PublicUserFlags, PublicUserFlagsDescriptions} from '@fluxer/constants/src/UserConstants';
 import {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas';
-import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators';
+import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators';
 import {
 	createBitflagInt32Type,
 	createNamedObject,
@@ -414,12 +414,8 @@ export type ApplicationUpdateRequest = z.infer<typeof ApplicationUpdateRequest>;
 export const BotProfileUpdateRequest = z.object({
 	username: UsernameType.optional().describe('The username of the bot'),
 	discriminator: DiscriminatorType.optional().describe('The discriminator of the bot'),
-	avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('The avatar image as base64'),
-	banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('The banner image as base64'),
+	avatar: createBase64ImageType().nullish().describe('The avatar image as base64'),
+	banner: createBase64ImageType().nullish().describe('The banner image as base64'),
 	bio: createStringType(0, 1024).nullish().describe('The bio or description of the bot'),
 	bot_flags: createBitflagInt32Type(BotFlags, BotFlagsDescriptions, 'The bot user flags', 'BotFlags').optional(),
 });

packages/schema/src/domains/user/UserRequestSchemas.ts

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {AVATAR_MAX_SIZE, MAX_GROUP_DM_OTHER_RECIPIENTS, MAX_GUILDS_PREMIUM} from '@fluxer/constants/src/LimitConstants';
+import {MAX_GROUP_DM_OTHER_RECIPIENTS, MAX_GUILDS_PREMIUM} from '@fluxer/constants/src/LimitConstants';
 import {StatusTypes} from '@fluxer/constants/src/StatusConstants';
 import {
 	DEFAULT_GUILD_FOLDER_ICON,
@@ -23,7 +23,7 @@ import {
 	SYNCED_PREFERENCES_MAX_ENCODED_LENGTH,
 } from '@fluxer/schema/src/domains/user/SyncedPreferencesCodec';
 import {isValidSingleUnicodeEmoji} from '@fluxer/schema/src/primitives/EmojiValidators';
-import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators';
+import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators';
 import {LocaleSchema} from '@fluxer/schema/src/primitives/LocaleSchema';
 import {createQueryIntegerType, DateTimeType, QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators';
 import {
@@ -67,12 +67,8 @@ export const UserUpdateRequest = z
 		email: EmailType.describe('The email address for the account'),
 		new_password: PasswordType.describe('The new password to set'),
 		password: PasswordType.describe('The current password for verification'),
-		avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('Base64-encoded avatar image'),
-		banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('Base64-encoded profile banner image'),
+		avatar: createBase64ImageType().nullish().describe('Base64-encoded avatar image'),
+		banner: createBase64ImageType().nullish().describe('Base64-encoded profile banner image'),
 		bio: createStringType(1, 320).nullish().describe('User biography text (max 320 characters)'),
 		pronouns: createStringType(1, 40).nullish().describe('User pronouns (max 40 characters)'),
 		accent_color: ColorType.nullish().describe('Profile accent color as integer'),

packages/schema/src/domains/webhook/WebhookRequestSchemas.ts

@@ -6,7 +6,7 @@ import {
 	MessageFlags,
 	MessageFlagsDescriptions,
 } from '@fluxer/constants/src/ChannelConstants';
-import {AVATAR_MAX_SIZE, MAX_MESSAGE_LENGTH_PREMIUM} from '@fluxer/constants/src/LimitConstants';
+import {MAX_MESSAGE_LENGTH_PREMIUM} from '@fluxer/constants/src/LimitConstants';
 import {
 	MAX_APPLIED_TAGS_PER_THREAD,
 	THREAD_NAME_MAX_LENGTH,
@@ -19,7 +19,7 @@ import {
 	RichEmbedRequest,
 } from '@fluxer/schema/src/domains/message/MessageRequestSchemas';
 import {AllowedMentionsRequest, MessageReferenceRequest} from '@fluxer/schema/src/domains/message/SharedMessageSchemas';
-import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators';
+import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators';
 import {QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators';
 import {
 	coerceNumberFromString,
@@ -37,9 +37,7 @@ import {z} from 'zod';
 
 export const WebhookCreateRequest = z.object({
 	name: WebhookNameType.describe('The name of the webhook'),
-	avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-		.nullish()
-		.describe('The avatar image as a base64-encoded data URI'),
+	avatar: createBase64ImageType().nullish().describe('The avatar image as a base64-encoded data URI'),
 });
 
 export type WebhookCreateRequest = z.infer<typeof WebhookCreateRequest>;
@@ -47,9 +45,7 @@ export type WebhookCreateRequest = z.infer<typeof WebhookCreateRequest>;
 export const WebhookUpdateRequest = z
 	.object({
 		name: WebhookNameType.describe('The new name of the webhook'),
-		avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('The new avatar image as a base64-encoded data URI'),
+		avatar: createBase64ImageType().nullish().describe('The new avatar image as a base64-encoded data URI'),
 		channel_id: SnowflakeType.describe('The ID of the channel to move the webhook to'),
 	})
 	.partial();
@@ -59,9 +55,7 @@ export type WebhookUpdateRequest = z.infer<typeof WebhookUpdateRequest>;
 export const WebhookTokenUpdateRequest = z
 	.strictObject({
 		name: WebhookNameType.describe('The new name of the webhook'),
-		avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE))
-			.nullish()
-			.describe('The new avatar image as a base64-encoded data URI'),
+		avatar: createBase64ImageType().nullish().describe('The new avatar image as a base64-encoded data URI'),
 	})
 	.partial();
 

packages/schema/src/primitives/FileValidators.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {BASE64_IMAGE_MAX_SIZE} from '@fluxer/constants/src/LimitConstants';
 import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
 import {
 	normalizeString,
@@ -106,3 +107,7 @@ export function createBase64StringType(minLength = 1, maxLength = 256) {
 		'Base64ImageType',
 	);
 }
+
+export function createBase64ImageType() {
+	return createBase64StringType(1, base64LengthForBytes(BASE64_IMAGE_MAX_SIZE));
+}

packages/schema/src/primitives/tests/FileValidators.test.ts

@@ -1,9 +1,15 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {AVATAR_MAX_SIZE, EMOJI_MAX_SIZE, STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants';
+import {
+	AVATAR_MAX_SIZE,
+	BASE64_IMAGE_MAX_SIZE,
+	EMOJI_MAX_SIZE,
+	STICKER_MAX_SIZE,
+} from '@fluxer/constants/src/LimitConstants';
 import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
 import {
 	base64LengthForBytes,
+	createBase64ImageType,
 	createBase64StringType,
 	FilenameType,
 	isValidBase64,
@@ -120,6 +126,7 @@ describe('base64LengthForBytes', () => {
 		[4, 8],
 		[EMOJI_MAX_SIZE, 699052],
 		[AVATAR_MAX_SIZE, 13981016],
+		[BASE64_IMAGE_MAX_SIZE, 139810136],
 	])('encodes a ceiling of %i bytes in %i characters', (bytes, expected) => {
 		expect(base64LengthForBytes(bytes)).toBe(expected);
 	});
@@ -138,3 +145,13 @@ describe('base64LengthForBytes', () => {
 		},
 	);
 });
+
+describe('createBase64ImageType', () => {
+	it.each([EMOJI_MAX_SIZE + 1, STICKER_MAX_SIZE * 2, AVATAR_MAX_SIZE + 1])(
+		'accepts %i decoded bytes so configured size limits above the stock values can apply',
+		(bytes) => {
+			const encoded = Buffer.alloc(bytes, 1).toString('base64');
+			expect(createBase64ImageType().parse(encoded)).toBe(encoded);
+		},
+	);
+});