fix(media): stream large files and honour size settings (#3387)

fluxer_api/src/api/infrastructure/AvatarService.ts

@@ -95,13 +95,13 @@ export class AvatarService {
 			return null;
 		}
 		const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image;
-		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const maxAvatarSize = this.resolveSizeLimit('avatar_max_size', AVATAR_MAX_SIZE);
-		if (imageBuffer.length > maxAvatarSize) {
+		if (Buffer.byteLength(base64Data, 'base64') > maxAvatarSize) {
 			throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, {
 				maxSize: maxAvatarSize,
 			});
 		}
+		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const kind = this.prefixToKind(prefix);
 		const metadata = this.requireAllowedMetadata({
 			metadata: await this.mediaService.getMetadata({
@@ -143,13 +143,13 @@ export class AvatarService {
 	}> {
 		const {errorPath, base64Image, guildFeatures} = params;
 		const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image;
-		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const maxEmojiSize = this.resolveSizeLimit('emoji_max_size', EMOJI_MAX_SIZE, guildFeatures);
-		if (imageBuffer.length > maxEmojiSize) {
+		if (Buffer.byteLength(base64Data, 'base64') > maxEmojiSize) {
 			throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, {
 				maxSize: maxEmojiSize,
 			});
 		}
+		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const metadata = this.requireAllowedMetadata({
 			metadata: await this.mediaService.getMetadata({
 				type: 'base64',
@@ -244,13 +244,13 @@ export class AvatarService {
 	}> {
 		const {errorPath, base64Image, guildFeatures} = params;
 		const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image;
-		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const maxStickerSize = this.resolveSizeLimit('sticker_max_size', STICKER_MAX_SIZE, guildFeatures);
-		if (imageBuffer.length > maxStickerSize) {
+		if (Buffer.byteLength(base64Data, 'base64') > maxStickerSize) {
 			throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, {
 				maxSize: maxStickerSize,
 			});
 		}
+		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const metadata = this.requireAllowedMetadata({
 			metadata: await this.mediaService.getMetadata({
 				type: 'base64',

fluxer_api/src/api/infrastructure/EntityAssetService.ts

@@ -291,13 +291,13 @@ export class EntityAssetService {
 		animated: boolean;
 	}> {
 		const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image;
-		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const maxAvatarSize = this.resolveAvatarSizeLimit();
-		if (imageBuffer.length > maxAvatarSize) {
+		if (Buffer.byteLength(base64Data, 'base64') > maxAvatarSize) {
 			throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, {
 				maxSize: maxAvatarSize,
 			});
 		}
+		const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64'));
 		const kind = this.mapAssetTypeToAssetKind(assetType);
 		const metadata = this.requireAllowedMetadata({
 			metadata: await this.mediaService.getMetadata({

fluxer_api/src/api/infrastructure/IMediaService.ts

@@ -42,6 +42,10 @@ export type MediaProxyMetadataRequest =
 	| MediaProxyMetadataBase64Request
 	| MediaProxyMetadataS3Request;
 
+export interface MediaProxyMetadataOptions {
+	sizeBytes?: number;
+}
+
 export interface MediaProxyMetadataResponse {
 	format: string;
 	content_type: string;
@@ -83,7 +87,10 @@ export interface MediaProxyFrameResponse {
 }
 
 export abstract class IMediaService {
-	abstract getMetadata(request: MediaProxyMetadataRequest): Promise<MediaProxyMetadataResponse | null>;
+	abstract getMetadata(
+		request: MediaProxyMetadataRequest,
+		options?: MediaProxyMetadataOptions,
+	): Promise<MediaProxyMetadataResponse | null>;
 
 	abstract getExternalMediaProxyURL(url: string): string;
 

fluxer_api/src/api/infrastructure/MediaService.ts

@@ -7,6 +7,7 @@ import {
 	MEDIA_PROXY_METADATA_REQUEST_VERSION,
 	type MediaProxyFrameRequest,
 	type MediaProxyFrameResponse,
+	type MediaProxyMetadataOptions,
 	type MediaProxyMetadataRequest,
 	type MediaProxyMetadataResponse,
 	type MediaProxyNsfwMode,
@@ -36,6 +37,17 @@ const MEDIA_PROXY_FRAMES_MAX_BYTES = 512 * 1024;
 const MEDIA_PROXY_SNIFF_MAX_BYTES = 1024;
 const MEDIA_PROXY_REQUEST_TIMEOUT_MS = ms('30 seconds');
 const MEDIA_PROXY_SNIFF_TIMEOUT_MS = ms('2 seconds');
+const MEDIA_PROXY_BUFFERED_INPUT_MAX_BYTES = 500 * 1024 * 1024;
+const MEDIA_PROXY_SPOOLED_INPUT_BYTES_PER_SECOND = 16 * 1024 * 1024;
+
+export function metadataRequestTimeoutMs(sizeBytes: number | undefined): number {
+	if (sizeBytes === undefined || sizeBytes <= MEDIA_PROXY_BUFFERED_INPUT_MAX_BYTES) {
+		return MEDIA_PROXY_REQUEST_TIMEOUT_MS;
+	}
+	return (
+		MEDIA_PROXY_REQUEST_TIMEOUT_MS + Math.ceil(sizeBytes / MEDIA_PROXY_SPOOLED_INPUT_BYTES_PER_SECOND) * ms('1 second')
+	);
+}
 
 function isMediaProxySniffResponse(value: unknown): value is MediaProxySniffResponse {
 	return isJsonRecord(value) && (value.content_type === null || typeof value.content_type === 'string');
@@ -105,9 +117,16 @@ export class MediaService extends IMediaService {
 		this.proxyURL = new URL(Config.endpoints.media);
 	}
 
-	async getMetadata(request: MediaProxyMetadataRequest): Promise<MediaProxyMetadataResponse | null> {
+	async getMetadata(
+		request: MediaProxyMetadataRequest,
+		options: MediaProxyMetadataOptions = {},
+	): Promise<MediaProxyMetadataResponse | null> {
 		const nsfwMode = this.getNsfwMode(request);
-		const response = await this.makeRequest('/_metadata', this.toMetadataWireRequest(request, nsfwMode));
+		const response = await this.makeRequest(
+			'/_metadata',
+			this.toMetadataWireRequest(request, nsfwMode),
+			metadataRequestTimeoutMs(options.sizeBytes),
+		);
 		if (!response) {
 			return null;
 		}

fluxer_api/src/api/infrastructure/MediaServiceMetadataTimeout.test.ts

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {metadataRequestTimeoutMs} from '@app/api/infrastructure/MediaService';
+import {describe, expect, it} from 'vitest';
+
+const MIB = 1024 * 1024;
+
+describe('metadataRequestTimeoutMs', () => {
+	it.each([[undefined], [0], [100 * MIB], [500 * MIB]])(
+		'keeps the 30 second window for an input of %s bytes that media-proxy buffers',
+		(sizeBytes) => {
+			expect(metadataRequestTimeoutMs(sizeBytes)).toBe(30_000);
+		},
+	);
+
+	it.each([
+		[500 * MIB + 1, 62_000],
+		[700 * MIB, 74_000],
+		[2 * 1024 * MIB, 158_000],
+		[8 * 1024 * MIB, 542_000],
+	])('widens the window for an input of %s bytes that media-proxy copies to disk first', (sizeBytes, timeoutMs) => {
+		expect(metadataRequestTimeoutMs(sizeBytes)).toBe(timeoutMs);
+	});
+});

fluxer_api/src/api/infrastructure/StorageObjectHelpers.ts

@@ -228,15 +228,59 @@ async function stripMetadataWithExiftool(data: Uint8Array, extension: string): P
 	}
 }
 
-async function stripVideoMetadataInPlace(inputPath: string, outputPath: string): Promise<void> {
-	await execFilePromise('ffmpeg', [
+function ffmpegMuxerForContentType(contentType: string): string | null {
+	if (contentType.includes('3gpp')) return '3gp';
+	if (contentType.includes('mp4')) return 'mp4';
+	if (contentType.includes('webm')) return 'webm';
+	if (contentType.includes('quicktime')) return 'mov';
+	if (contentType.includes('x-matroska')) return 'matroska';
+	if (contentType.includes('x-msvideo')) return 'avi';
+	if (contentType.includes('x-flv')) return 'flv';
+	if (contentType.includes('mp2t')) return 'mpegts';
+	if (contentType.includes('mpeg')) return contentType.startsWith('audio/') ? 'mp3' : 'mpeg';
+	if (contentType.includes('x-ms-wmv')) return 'asf';
+	if (contentType.includes('wav')) return 'wav';
+	if (contentType.includes('flac')) return 'flac';
+	if (contentType.includes('aac')) return 'adts';
+	if (contentType.includes('aiff')) return 'aiff';
+	if (contentType.includes('ogg')) return 'ogg';
+	return null;
+}
+
+const FFMPEG_FAST_START_MUXERS = new Set(['mp4', 'mov', '3gp']);
+const FFMPEG_DEMUXER_FOR_MUXER: Readonly<Record<string, string>> = {
+	'3gp': 'mp4',
+	mov: 'mp4',
+	webm: 'matroska',
+	adts: 'aac',
+};
+const FFMPEG_STRIP_TIMEOUT_MS = 10 * 60 * 1000;
+const FFMPEG_MAX_BUFFER_BYTES = 8 * 1024 * 1024;
+const FFPROBE_TIMEOUT_MS = 30 * 1000;
+const FFPROBE_MAX_BUFFER_BYTES = 64 * 1024;
+
+function requireFfmpegMuxer(contentType: string): string {
+	const muxer = ffmpegMuxerForContentType(contentType);
+	if (!muxer) {
+		throw new Error(`No ffmpeg output format for content type ${contentType}`);
+	}
+	return muxer;
+}
+
+export function ffmpegMetadataStripArgs(inputPath: string, outputPath: string, contentType: string): Array<string> {
+	const muxer = requireFfmpegMuxer(contentType);
+	return [
 		'-hide_banner',
 		'-loglevel',
-		'warning',
+		'error',
 		'-i',
 		inputPath,
 		'-map',
-		'0',
+		'0:v?',
+		'-map',
+		'0:a?',
+		'-map',
+		'0:s?',
 		'-map_metadata',
 		'-1',
 		'-map_metadata:s',
@@ -247,9 +291,29 @@ async function stripVideoMetadataInPlace(inputPath: string, outputPath: string):
 		'-1',
 		'-c',
 		'copy',
+		...(FFMPEG_FAST_START_MUXERS.has(muxer) ? ['-movflags', '+faststart'] : []),
+		'-f',
+		muxer,
 		'-y',
 		outputPath,
-	]);
+	];
+}
+
+export function ffmpegContainerMatchesContentType(formatNames: string, contentType: string): boolean {
+	const muxer = requireFfmpegMuxer(contentType);
+	return formatNames
+		.trim()
+		.split(',')
+		.includes(FFMPEG_DEMUXER_FOR_MUXER[muxer] ?? muxer);
+}
+
+async function probeFfmpegContainer(inputPath: string): Promise<string> {
+	const {stdout} = await execFilePromise(
+		'ffprobe',
+		['-v', 'error', '-show_entries', 'format=format_name', '-of', 'default=nw=1:nk=1', inputPath],
+		{timeout: FFPROBE_TIMEOUT_MS, maxBuffer: FFPROBE_MAX_BUFFER_BYTES},
+	);
+	return stdout;
 }
 
 async function cleanupTempFiles(paths: ReadonlyArray<string>): Promise<
@@ -392,8 +456,16 @@ async function stripVideoFileToFile(
 	outputPath: string,
 	contentType: string,
 ): Promise<{contentType: string; width?: number; height?: number}> {
-	await stripVideoMetadataInPlace(inputPath, outputPath);
-	return {contentType: normalizeContentType(contentType)};
+	const normalizedContentType = normalizeContentType(contentType);
+	const container = await probeFfmpegContainer(inputPath);
+	if (!ffmpegContainerMatchesContentType(container, normalizedContentType)) {
+		throw new Error(`Container ${container.trim()} does not match content type ${normalizedContentType}`);
+	}
+	await execFilePromise('ffmpeg', ffmpegMetadataStripArgs(inputPath, outputPath, normalizedContentType), {
+		timeout: FFMPEG_STRIP_TIMEOUT_MS,
+		maxBuffer: FFMPEG_MAX_BUFFER_BYTES,
+	});
+	return {contentType: normalizedContentType};
 }
 
 export async function processMediaFile(

fluxer_api/src/api/infrastructure/StorageObjectHelpersFfmpegFormat.test.ts

@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import {
+	ffmpegContainerMatchesContentType,
+	ffmpegMetadataStripArgs,
+	processMediaFile,
+} from '@app/api/infrastructure/StorageObjectHelpers';
+import {temporaryFile} from 'tempy';
+import {afterEach, describe, expect, it} from 'vitest';
+
+const INPUT_PATH = '/tmp/0123456789abcdef';
+const OUTPUT_PATH = '/tmp/fedcba9876543210';
+
+function outputFormat(contentType: string): string | undefined {
+	const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, contentType);
+	const flag = args.indexOf('-f');
+	return flag === -1 ? undefined : args[flag + 1];
+}
+
+describe('ffmpegMetadataStripArgs', () => {
+	it.each([
+		['video/mp4', 'mp4'],
+		['video/quicktime', 'mov'],
+		['video/3gpp', '3gp'],
+		['video/webm', 'webm'],
+		['video/x-matroska', 'matroska'],
+		['video/x-msvideo', 'avi'],
+		['video/x-flv', 'flv'],
+		['video/mp2t', 'mpegts'],
+		['video/mpeg', 'mpeg'],
+		['video/x-ms-wmv', 'asf'],
+		['video/ogg', 'ogg'],
+		['audio/mpeg', 'mp3'],
+		['audio/mp4', 'mp4'],
+		['audio/ogg', 'ogg'],
+		['audio/webm', 'webm'],
+		['audio/wav', 'wav'],
+		['audio/flac', 'flac'],
+		['audio/aac', 'adts'],
+		['audio/aiff', 'aiff'],
+	])('names the %s output format because the output path has no extension', (contentType, muxer) => {
+		expect(outputFormat(contentType)).toBe(muxer);
+	});
+
+	it('writes to the output path last and reads the input path', () => {
+		const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/mp4');
+		expect(args[args.indexOf('-i') + 1]).toBe(INPUT_PATH);
+		expect(args.at(-1)).toBe(OUTPUT_PATH);
+	});
+
+	it.each([
+		['video/mp4', true],
+		['video/quicktime', true],
+		['video/webm', false],
+		['audio/mpeg', false],
+	])('moves the %s index to the front of the file: %s', (contentType, fastStart) => {
+		const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, contentType);
+		expect(args.includes('+faststart')).toBe(fastStart);
+	});
+
+	it('refuses a container it has no output format for', () => {
+		expect(() => ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/x-unknown-container')).toThrow(
+			/output format/,
+		);
+	});
+
+	it('keeps video, audio and subtitle streams and leaves data streams behind', () => {
+		const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/quicktime');
+		const maps = args.flatMap((arg, index) => (arg === '-map' ? [args[index + 1]] : []));
+		expect(maps).toEqual(['0:v?', '0:a?', '0:s?']);
+	});
+});
+
+describe('ffmpegContainerMatchesContentType', () => {
+	it.each([
+		['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/mp4'],
+		['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/quicktime'],
+		['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/3gpp'],
+		['mov,mp4,m4a,3gp,3g2,mj2\n', 'audio/mp4'],
+		['matroska,webm\n', 'video/webm'],
+		['matroska,webm\n', 'video/x-matroska'],
+		['matroska,webm\n', 'audio/webm'],
+		['avi\n', 'video/x-msvideo'],
+		['flv\n', 'video/x-flv'],
+		['mpegts\n', 'video/mp2t'],
+		['mpeg\n', 'video/mpeg'],
+		['asf\n', 'video/x-ms-wmv'],
+		['ogg\n', 'video/ogg'],
+		['mp3\n', 'audio/mpeg'],
+		['ogg\n', 'audio/ogg'],
+		['wav\n', 'audio/wav'],
+		['flac\n', 'audio/flac'],
+		['aac\n', 'audio/aac'],
+		['aiff\n', 'audio/aiff'],
+	])('accepts the %s container for %s', (formatNames, contentType) => {
+		expect(ffmpegContainerMatchesContentType(formatNames, contentType)).toBe(true);
+	});
+
+	it.each([
+		['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/x-matroska'],
+		['matroska,webm\n', 'video/mp4'],
+		['mp3\n', 'video/mpeg'],
+		['mpeg\n', 'audio/mpeg'],
+		['mpegts\n', 'video/mpeg'],
+		['wav\n', 'audio/flac'],
+	])('refuses the %s container for %s', (formatNames, contentType) => {
+		expect(ffmpegContainerMatchesContentType(formatNames, contentType)).toBe(false);
+	});
+});
+
+const hasFfmpeg = ['ffmpeg', 'ffprobe'].every((tool) => spawnSync(tool, ['-version']).status === 0);
+
+describe.skipIf(!hasFfmpeg)('processMediaFile with ffmpeg', () => {
+	const paths: Array<string> = [];
+
+	function tempPath(): string {
+		const path = temporaryFile();
+		paths.push(path);
+		return path;
+	}
+
+	function generate(args: ReadonlyArray<string>): string {
+		const path = tempPath();
+		execFileSync('ffmpeg', [
+			'-loglevel',
+			'error',
+			'-f',
+			'lavfi',
+			'-i',
+			'testsrc=s=64x48:r=5',
+			'-f',
+			'lavfi',
+			'-i',
+			'sine=f=440',
+			'-t',
+			'1',
+			'-c:v',
+			'mpeg4',
+			'-c:a',
+			'aac',
+			'-metadata',
+			'title=private-title',
+			...args,
+			'-y',
+			path,
+		]);
+		return path;
+	}
+
+	function probe(path: string): {
+		format: {format_name: string; tags?: Record<string, string>};
+		streams: Array<{codec_type: string}>;
+	} {
+		return JSON.parse(
+			execFileSync('ffprobe', ['-v', 'error', '-show_format', '-show_streams', '-of', 'json', path]).toString(),
+		);
+	}
+
+	afterEach(() => {
+		for (const path of paths.splice(0)) {
+			fs.rmSync(path, {force: true});
+		}
+	});
+
+	it('rewrites a QuickTime file without its tags and data stream at paths that have no extension', async () => {
+		const input = generate(['-timecode', '01:00:00:00', '-f', 'mov']);
+		expect(probe(input).format.tags?.title).toBe('private-title');
+		expect(probe(input).streams.map((stream) => stream.codec_type)).toContain('data');
+		const output = tempPath();
+
+		const processed = await processMediaFile(input, output, 'video/quicktime');
+
+		const stripped = probe(output);
+		expect(processed?.contentType).toBe('video/quicktime');
+		expect(processed?.contentLength).toBe(fs.statSync(output).size);
+		expect(stripped.format.format_name).toContain('mov');
+		expect(stripped.format.tags?.title).toBeUndefined();
+		expect(stripped.streams.map((stream) => stream.codec_type).sort()).toEqual(['audio', 'video']);
+	});
+
+	it('refuses a file whose container is not the one its content type names', async () => {
+		const input = generate(['-f', 'matroska']);
+
+		await expect(processMediaFile(input, tempPath(), 'video/mp4')).rejects.toThrow(/does not match content type/);
+	});
+});