fix(voice): request a real camera frame rate at capture (#2418)

fluxer_app/src/features/voice/components/modals/CameraPreviewModal.tsx

@@ -24,6 +24,7 @@ import {useMaybeVoiceRoom} from '@app/features/voice/components/VoiceRoomContext
 import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade';
 import {VOICE_CAMERA_USER_LIMIT_REACHED_DESCRIPTOR} from '@app/features/voice/engine/media_engine_facade/shared';
 import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
+import {getCameraCaptureDimensions} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets';
 import {useCameraUserCapBlocked} from '@app/features/voice/hooks/useCameraUserCapBlocked';
 import VoiceSettings, {
 	BLUR_BACKGROUND_ID,
@@ -90,12 +91,6 @@ interface CameraPreviewModalProps {
 	isCameraEnabled?: boolean;
 }
 
-interface VideoResolutionPreset {
-	width: number;
-	height: number;
-	frameRate: number;
-}
-
 const TARGET_ASPECT_RATIO = 16 / 9;
 const ASPECT_RATIO_TOLERANCE = 0.1;
 const RESOLUTION_WAIT_TIMEOUT = 2000;
@@ -108,11 +103,6 @@ const VIDEO_READY_STATE_HAS_CURRENT_DATA = 2;
 const RESOLUTION_FIX_TRIGGER_DELAY = 800;
 const RESOLUTION_FIX_SETTLE_DELAY = 1200;
 const RESOLUTION_FIX_SWITCH_BACK_DELAY = 500;
-const CAMERA_RESOLUTION_PRESETS: Record<'low' | 'medium' | 'high', VideoResolutionPreset> = {
-	low: {width: 640, height: 360, frameRate: 24},
-	medium: {width: 1280, height: 720, frameRate: 30},
-	high: {width: 1920, height: 1080, frameRate: 30},
-};
 
 interface CameraPreviewConfig {
 	videoDeviceId: string;
@@ -280,15 +270,15 @@ async function setupPreviewTrackAndProcessor(args: CameraPreviewTrackSetupArgs):
 		await args.processorRef.current.destroy();
 		args.processorRef.current = null;
 	}
-	const resolutionPreset = CAMERA_RESOLUTION_PRESETS[args.cameraResolution];
+	const captureDimensions = getCameraCaptureDimensions(args.cameraResolution);
 	const track = await createLocalVideoTrack({
 		deviceId:
 			args.effectiveVideoDeviceId && args.effectiveVideoDeviceId !== 'default'
 				? args.effectiveVideoDeviceId
 				: undefined,
 		resolution: {
-			width: resolutionPreset.width,
-			height: resolutionPreset.height,
+			width: captureDimensions.width,
+			height: captureDimensions.height,
 			frameRate: args.videoFrameRate,
 			aspectRatio: TARGET_ASPECT_RATIO,
 		},

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets.test.ts

@@ -3,16 +3,12 @@
 import type {CameraResolution} from '@app/features/voice/state/VoiceSettings';
 import {VideoPresets} from 'livekit-client';
 import {describe, expect, it} from 'vitest';
-import {
-	getCameraCaptureDimensions,
-	getCameraVideoPreset,
-	pickCameraSubscriptionQuality,
-} from './VoiceEngineV2AppCameraResolutionPresets';
+import {getCameraCaptureDimensions, pickCameraSubscriptionQuality} from './VoiceEngineV2AppCameraResolutionPresets';
 
 const EXPECTED_PRESET_TABLE = [
 	{resolution: 'high', width: 1920, height: 1080, frameRate: 30, videoPreset: VideoPresets.h1080},
 	{resolution: 'medium', width: 1280, height: 720, frameRate: 30, videoPreset: VideoPresets.h720},
-	{resolution: 'low', width: 640, height: 360, frameRate: 24, videoPreset: VideoPresets.h360},
+	{resolution: 'low', width: 640, height: 360, frameRate: 20, videoPreset: VideoPresets.h360},
 ] as const;
 
 describe('getCameraCaptureDimensions', () => {
@@ -28,29 +24,17 @@ describe('getCameraCaptureDimensions', () => {
 
 	it('falls back to the low entry for an unknown resolution key', () => {
 		const dimensions = getCameraCaptureDimensions('ultra' as CameraResolution);
-		expect(dimensions).toEqual({width: 640, height: 360, frameRate: 24});
-	});
-});
-
-describe('getCameraVideoPreset', () => {
-	for (const row of EXPECTED_PRESET_TABLE) {
-		it(`maps ${row.resolution} to the ${row.videoPreset.height}p LiveKit preset`, () => {
-			expect(getCameraVideoPreset(row.resolution)).toBe(row.videoPreset);
-		});
-	}
-
-	it('falls back to the 360p preset for an unknown resolution key', () => {
-		expect(getCameraVideoPreset('4k' as CameraResolution)).toBe(VideoPresets.h360);
+		expect(dimensions).toEqual({width: 640, height: 360, frameRate: 20});
 	});
 });
 
 describe('camera preset table consistency', () => {
 	for (const row of EXPECTED_PRESET_TABLE) {
-		it(`keeps capture dimensions and video preset in agreement for ${row.resolution}`, () => {
+		it(`keeps capture dimensions and the LiveKit encode preset in agreement for ${row.resolution}`, () => {
 			const capture = getCameraCaptureDimensions(row.resolution);
-			const preset = getCameraVideoPreset(row.resolution);
-			expect(preset.width).toBe(capture.width);
-			expect(preset.height).toBe(capture.height);
+			expect(row.videoPreset.width).toBe(capture.width);
+			expect(row.videoPreset.height).toBe(capture.height);
+			expect(row.videoPreset.encoding.maxFramerate).toBe(capture.frameRate);
 		});
 	}
 
@@ -62,6 +46,16 @@ describe('camera preset table consistency', () => {
 	});
 });
 
+describe('camera capture constraint shape', () => {
+	for (const row of EXPECTED_PRESET_TABLE) {
+		it(`hands getUserMedia a plain video resolution for ${row.resolution}`, () => {
+			const capture = getCameraCaptureDimensions(row.resolution);
+			expect(Object.keys(capture).sort()).toEqual(['frameRate', 'height', 'width']);
+			expect(capture.frameRate).toBeGreaterThan(0);
+		});
+	}
+});
+
 describe('pickCameraSubscriptionQuality', () => {
 	it('requests the lowest layer for a hidden or zero-sized tile', () => {
 		expect(pickCameraSubscriptionQuality(0, 0)).toBe('low');

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets.ts

@@ -14,12 +14,17 @@ export interface CameraCaptureDimensions {
 const CAMERA_RESOLUTION_PRESET_TABLE = {
 	high: {capture: {width: 1920, height: 1080, frameRate: 30}, videoPreset: VideoPresets.h1080},
 	medium: {capture: {width: 1280, height: 720, frameRate: 30}, videoPreset: VideoPresets.h720},
-	low: {capture: {width: 640, height: 360, frameRate: 24}, videoPreset: VideoPresets.h360},
+	low: {capture: {width: 640, height: 360, frameRate: 20}, videoPreset: VideoPresets.h360},
 } as const;
 
 for (const [resolution, entry] of Object.entries(CAMERA_RESOLUTION_PRESET_TABLE)) {
 	assert.equal(entry.videoPreset.width, entry.capture.width, `camera preset width must agree for ${resolution}`);
 	assert.equal(entry.videoPreset.height, entry.capture.height, `camera preset height must agree for ${resolution}`);
+	assert.equal(
+		entry.videoPreset.encoding.maxFramerate,
+		entry.capture.frameRate,
+		`camera preset frame rate must agree for ${resolution}`,
+	);
 }
 
 function resolvePresetEntry(resolution: CameraResolution) {
@@ -35,13 +40,6 @@ export function getCameraCaptureDimensions(resolution: CameraResolution): Camera
 	return dimensions;
 }
 
-export function getCameraVideoPreset(resolution: CameraResolution) {
-	const preset = resolvePresetEntry(resolution).videoPreset;
-	assert.ok(preset.width > 0, 'camera video preset width must be positive');
-	assert.ok(preset.height > 0, 'camera video preset height must be positive');
-	return preset;
-}
-
 const CAMERA_SUBSCRIPTION_QUALITY_LAYERS = [
 	{quality: 'low', layer: VideoPresets.h180},
 	{quality: 'medium', layer: VideoPresets.h360},

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx

@@ -66,7 +66,7 @@ import {
 	isMutedOrDeafened,
 	isPermissionDeniedError,
 } from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
-import {getCameraVideoPreset} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets';
+import {getCameraCaptureDimensions} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets';
 import {
 	runCameraTransition,
 	type VoiceEngineV2AppCameraTransitionOutcome,
@@ -1141,8 +1141,8 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 				await clearCameraVideoProcessor(cameraTrack);
 			}
 		}
-		const videoResolution = getCameraVideoPreset(VoiceSettings.getCameraResolution());
-		await participant.setCameraEnabled(enabled, {resolution: videoResolution, ...restOptions});
+		const captureDimensions = getCameraCaptureDimensions(VoiceSettings.getCameraResolution());
+		await participant.setCameraEnabled(enabled, {resolution: captureDimensions, ...restOptions});
 		await this.enforceCameraPublicationCap(participant, enabled ? 'after camera enable' : 'after camera disable');
 		if (enabled) {
 			await this.applyBackgroundToCamera(participant);