mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
fix(voice): keep published codecs inside the opt-in policy (#2422)
This commit is contained in:
1 parent
2426a5769d
commit
902dd60ff9
51 files changed
+1032
-313
No files matched your search
@@ -7,6 +7,7 @@ import {
|
||||
type CodecPreference,
|
||||
type CodecSupportInfo,
|
||||
getCodecCapabilityReport,
|
||||
isVideoCodecAllowedForPublish,
|
||||
resolveEffectiveScreenShareEncoderMode,
|
||||
type ScreenShareEncoderMode,
|
||||
selectNativeScreenCaptureScreenShareCodec,
|
||||
@@ -184,12 +185,7 @@ export function getScreenShareCodecPreferenceOrder(
|
||||
encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder();
|
||||
const order =
|
||||
preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder;
|
||||
const filtered = order.filter(
|
||||
(codec) =>
|
||||
(codec !== 'av1' || VoiceSettings.getScreenShareAv1OptIn()) &&
|
||||
(codec !== 'h265' || VoiceSettings.getScreenShareHevcOptIn()),
|
||||
);
|
||||
return filtered;
|
||||
return order.filter((codec) => isVideoCodecAllowedForPublish(codec));
|
||||
}
|
||||
|
||||
function getHardwareFirstScreenShareCodecPreferenceOrder(): ReadonlyArray<VideoCodec> {
|
||||
@@ -209,14 +205,15 @@ function getLocalEncodeCapabilities(): Record<VideoCodec, boolean> {
|
||||
(codec) => report[codec].supported && report[codec].hardwareAccelerated === 'hardware',
|
||||
);
|
||||
const advertise = (codec: VideoCodec): boolean =>
|
||||
(pinned === codec && report[codec].supported) ||
|
||||
shouldAdvertiseVideoEncode(codec, report[codec], encoderMode, hardwareEncodeAvailable, report);
|
||||
isVideoCodecAllowedForPublish(codec) &&
|
||||
((pinned === codec && report[codec].supported) ||
|
||||
shouldAdvertiseVideoEncode(codec, report[codec], encoderMode, hardwareEncodeAvailable, report));
|
||||
return {
|
||||
av1: advertise('av1'),
|
||||
h265: advertise('h265'),
|
||||
h264: advertise('h264'),
|
||||
vp9: advertise('vp9'),
|
||||
vp8: report.vp8.supported,
|
||||
vp8: report.vp8.supported && isVideoCodecAllowedForPublish('vp8'),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +124,23 @@ describe('VoiceScreenShareStateMachine', () => {
|
||||
expect(snapshot.context.publicationReplaceInFlight).toBe(false);
|
||||
});
|
||||
|
||||
it('lets a pending start or restore raise and drop the replace lock around a codec correction', () => {
|
||||
let snapshot = createVoiceScreenShareSnapshot();
|
||||
snapshot = transitionVoiceScreenShareSnapshot(snapshot, {type: 'share.publicationReplace.set', inFlight: true});
|
||||
expect(snapshot.context.publicationReplaceInFlight).toBe(false);
|
||||
snapshot = transitionVoiceScreenShareSnapshot(snapshot, {type: 'share.start', sourceType: 'display'});
|
||||
snapshot = transitionVoiceScreenShareSnapshot(snapshot, {type: 'share.publicationReplace.set', inFlight: true});
|
||||
expect(getVoiceScreenShareStateValue(snapshot)).toBe('pending');
|
||||
expect(snapshot.context.pendingOperation).toBe('starting');
|
||||
expect(snapshot.context.publicationReplaceInFlight).toBe(true);
|
||||
snapshot = transitionVoiceScreenShareSnapshot(snapshot, {type: 'share.publicationReplace.set', inFlight: false});
|
||||
expect(snapshot.context.publicationReplaceInFlight).toBe(false);
|
||||
snapshot = transitionVoiceScreenShareSnapshot(snapshot, {type: 'share.publicationReplace.set', inFlight: true});
|
||||
snapshot = resolveActive(snapshot);
|
||||
expect(getVoiceScreenShareStateValue(snapshot)).toBe('active');
|
||||
expect(snapshot.context.publicationReplaceInFlight).toBe(false);
|
||||
});
|
||||
|
||||
it('tracks codec readiness through the xstate context while starting', () => {
|
||||
let snapshot = createVoiceScreenShareSnapshot();
|
||||
expect(snapshot.context.codecReadiness).toBe('idle');
|
||||
|
||||
@@ -59,6 +59,7 @@ export type VoiceScreenShareEvent =
|
||||
| {type: 'share.codecReadiness.timeout'}
|
||||
| {type: 'share.codecReadiness.reset'}
|
||||
| {type: 'share.streamingPriority.set'; active: boolean}
|
||||
| {type: 'share.publicationReplace.set'; inFlight: boolean}
|
||||
| {type: 'share.queuedStop.clear'}
|
||||
| {
|
||||
type: 'share.localWatcher.sync';
|
||||
@@ -261,6 +262,11 @@ export const voiceScreenShareStateMachine = setup({
|
||||
setStreamingPriority: assign(({context, event}) =>
|
||||
event.type === 'share.streamingPriority.set' ? {...context, streamingPriorityHeld: event.active} : context,
|
||||
),
|
||||
setPublicationReplaceInFlight: assign(({context, event}) =>
|
||||
event.type === 'share.publicationReplace.set' && context.pendingOperation
|
||||
? {...context, publicationReplaceInFlight: event.inFlight}
|
||||
: context,
|
||||
),
|
||||
clearQueuedStop: assign(({context}) =>
|
||||
context.queuedStopRequest ? {...context, queuedStopRequest: null} : context,
|
||||
),
|
||||
@@ -306,6 +312,7 @@ export const voiceScreenShareStateMachine = setup({
|
||||
'share.encoderVerification.scheduled': {target: 'routing', actions: 'scheduleEncoderVerification'},
|
||||
'share.encoderVerification.cleared': {target: 'routing', actions: 'clearEncoderVerification'},
|
||||
'share.streamingPriority.set': {target: 'routing', actions: 'setStreamingPriority'},
|
||||
'share.publicationReplace.set': {target: 'routing', actions: 'setPublicationReplaceInFlight'},
|
||||
'share.queuedStop.clear': {target: 'routing', actions: 'clearQueuedStop'},
|
||||
'share.localWatcher.sync': {target: 'routing', actions: 'syncLocalWatcher'},
|
||||
'share.clearWatchCommands': {actions: 'clearWatchCommands'},
|
||||
@@ -326,6 +333,7 @@ export const voiceScreenShareStateMachine = setup({
|
||||
'share.encoderVerification.scheduled': {target: 'routing', actions: 'scheduleEncoderVerification'},
|
||||
'share.encoderVerification.cleared': {target: 'routing', actions: 'clearEncoderVerification'},
|
||||
'share.streamingPriority.set': {target: 'routing', actions: 'setStreamingPriority'},
|
||||
'share.publicationReplace.set': {target: 'routing', actions: 'setPublicationReplaceInFlight'},
|
||||
'share.queuedStop.clear': {target: 'routing', actions: 'clearQueuedStop'},
|
||||
'share.localWatcher.sync': {target: 'routing', actions: 'syncLocalWatcher'},
|
||||
'share.clearWatchCommands': {actions: 'clearWatchCommands'},
|
||||
@@ -346,6 +354,7 @@ export const voiceScreenShareStateMachine = setup({
|
||||
'share.encoderVerification.scheduled': {actions: 'scheduleEncoderVerification'},
|
||||
'share.encoderVerification.cleared': {actions: 'clearEncoderVerification'},
|
||||
'share.streamingPriority.set': {actions: 'setStreamingPriority'},
|
||||
'share.publicationReplace.set': {actions: 'setPublicationReplaceInFlight'},
|
||||
'share.queuedStop.clear': {actions: 'clearQueuedStop'},
|
||||
'share.localWatcher.sync': {actions: 'syncLocalWatcher'},
|
||||
'share.clearWatchCommands': {actions: 'clearWatchCommands'},
|
||||
|
||||
@@ -38,6 +38,10 @@ import {
|
||||
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
|
||||
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
|
||||
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
|
||||
import {
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
getRoomVideoPublishDefaults,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {
|
||||
getVideoDecoderExclusionsSync,
|
||||
@@ -140,9 +144,12 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
|
||||
return true;
|
||||
}
|
||||
|
||||
const ROOM_PUBLISH_DEFAULTS: RoomOptions['publishDefaults'] = {
|
||||
screenShareEncoding: {maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS, maxFramerate: 30, priority: 'high'},
|
||||
};
|
||||
function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
|
||||
return {
|
||||
screenShareEncoding: {maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS, maxFramerate: 30, priority: 'high'},
|
||||
...getRoomVideoPublishDefaults(),
|
||||
};
|
||||
}
|
||||
|
||||
function createRoomOptions(
|
||||
e2eeKey: string | null,
|
||||
@@ -156,7 +163,7 @@ function createRoomOptions(
|
||||
adaptiveStream: false,
|
||||
dynacast: true,
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: ROOM_PUBLISH_DEFAULTS,
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions,
|
||||
};
|
||||
let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
|
||||
@@ -681,7 +688,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
adaptiveStream: false,
|
||||
dynacast: true,
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: ROOM_PUBLISH_DEFAULTS,
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
|
||||
};
|
||||
if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {
|
||||
@@ -804,6 +811,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
Array.from(oldParticipant.trackPublications.values()),
|
||||
);
|
||||
const errors: Array<{source: string; error: unknown}> = [];
|
||||
let codecPolicyFailure: Error | null = null;
|
||||
for (const publication of publications) {
|
||||
const track = publication.track as LocalTrack | undefined;
|
||||
if (!isReadyToRepublishTrack(track)) {
|
||||
@@ -823,7 +831,16 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
source: publication.source,
|
||||
name: publication.trackName,
|
||||
};
|
||||
await newParticipant.publishTrack(track.mediaStreamTrack, publishOptions);
|
||||
const republished = await newParticipant.publishTrack(track.mediaStreamTrack, publishOptions);
|
||||
const violation = publishOptions.videoCodec
|
||||
? findVideoPublishCodecPolicyViolation(publishOptions.videoCodec, republished.options?.videoCodec)
|
||||
: null;
|
||||
if (violation) {
|
||||
codecPolicyFailure = new Error(
|
||||
`Region hot-swap: ${publication.source} negotiated ${violation.negotiated} after requesting ${violation.requested}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push({source: publication.source ?? 'unknown', error});
|
||||
logger.warn('Region hot-swap: failed to republish track', {
|
||||
@@ -832,6 +849,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
});
|
||||
}
|
||||
}
|
||||
if (codecPolicyFailure) throw codecPolicyFailure;
|
||||
const screenShareFailure = errors.find(
|
||||
(error) => error.source === Track.Source.ScreenShare || error.source === Track.Source.ScreenShareAudio,
|
||||
);
|
||||
|
||||
@@ -98,6 +98,10 @@ import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
|
||||
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {buildMicrophonePublishOptions} from '@app/features/voice/utils/AudioPublishOptions';
|
||||
import {
|
||||
buildCameraPublishOptions,
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {applyBackgroundProcessor, clearCameraVideoProcessor} from '@app/features/voice/utils/VideoBackgroundProcessor';
|
||||
import {
|
||||
removeVoiceInputProcessor,
|
||||
@@ -126,11 +130,13 @@ import type {
|
||||
LocalVideoTrack,
|
||||
Room,
|
||||
TrackPublishOptions,
|
||||
VideoCaptureOptions,
|
||||
} from 'livekit-client';
|
||||
import {Track} from 'livekit-client';
|
||||
|
||||
const logger = new Logger('VoiceEngineV2AppMediaExecutionAdapter');
|
||||
const LOCAL_SPEAKING_ANALYSER_INTERVAL_MS = 50;
|
||||
const CAMERA_PUBLISH_CODEC_CORRECTION_MAX = 1;
|
||||
export const REPUBLISH_MICROPHONE_GUARD_MS = 150;
|
||||
type VoiceMuteReason = VoiceEngineV2AppVoiceMuteReason;
|
||||
|
||||
@@ -1141,8 +1147,17 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
|
||||
await clearCameraVideoProcessor(cameraTrack);
|
||||
}
|
||||
}
|
||||
const captureDimensions = getCameraCaptureDimensions(VoiceSettings.getCameraResolution());
|
||||
await participant.setCameraEnabled(enabled, {resolution: captureDimensions, ...restOptions});
|
||||
const captureOptions: VideoCaptureOptions = {
|
||||
resolution: getCameraCaptureDimensions(VoiceSettings.getCameraResolution()),
|
||||
...restOptions,
|
||||
};
|
||||
if (enabled) {
|
||||
const publishOptions = buildCameraPublishOptions();
|
||||
await participant.setCameraEnabled(true, captureOptions, publishOptions);
|
||||
await this.enforceCameraPublishCodecPolicy(participant, publishOptions);
|
||||
} else {
|
||||
await participant.setCameraEnabled(false, captureOptions);
|
||||
}
|
||||
await this.enforceCameraPublicationCap(participant, enabled ? 'after camera enable' : 'after camera disable');
|
||||
if (enabled) {
|
||||
await this.applyBackgroundToCamera(participant);
|
||||
@@ -1156,6 +1171,31 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceCameraPublishCodecPolicy(
|
||||
participant: Room['localParticipant'],
|
||||
initialPublishOptions: TrackPublishOptions,
|
||||
): Promise<void> {
|
||||
let publishOptions = initialPublishOptions;
|
||||
for (let corrections = 0; ; corrections++) {
|
||||
const publication = getLocalCameraPublications(participant)[0];
|
||||
const track = publication?.videoTrack as LocalVideoTrack | undefined;
|
||||
const requested = publishOptions.videoCodec;
|
||||
if (!publication || !track || !requested) return;
|
||||
const violation = findVideoPublishCodecPolicyViolation(requested, publication.options?.videoCodec ?? track.codec);
|
||||
if (!violation) return;
|
||||
logger.warn('Camera published a codec outside the publish policy', {...violation, corrections});
|
||||
if (corrections >= CAMERA_PUBLISH_CODEC_CORRECTION_MAX || !violation.alternative) {
|
||||
await participant.setCameraEnabled(false);
|
||||
throw new Error(
|
||||
`camera negotiated ${violation.negotiated} after requesting ${violation.requested}; no allowed codec could be published`,
|
||||
);
|
||||
}
|
||||
await participant.unpublishTrack(track, false);
|
||||
publishOptions = buildCameraPublishOptions(violation.alternative);
|
||||
await participant.publishTrack(track, {...publishOptions, source: Track.Source.Camera});
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceCameraPublicationCap(participant: Room['localParticipant'], reason: string): Promise<void> {
|
||||
const result = await enforceLocalMediaPublicationCap(participant, 'camera');
|
||||
for (const failure of result.failedPublications) {
|
||||
|
||||
+81
-18
@@ -65,8 +65,9 @@ import {
|
||||
SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS,
|
||||
} from '@app/features/voice/utils/AudioPublishOptions';
|
||||
import {
|
||||
markScreenShareCodecEncodeRuntimeFailure,
|
||||
resolveScreenShareEncoderVerificationAction,
|
||||
type ScreenShareContentSource,
|
||||
type ScreenShareEncoderVerificationAction,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {disarmVirtmic} from '@app/features/voice/utils/LinuxScreenShareAudio';
|
||||
import {
|
||||
@@ -107,6 +108,13 @@ const SCREEN_SHARE_SOURCE_STOPPED_DESCRIPTOR = msg({
|
||||
'Body of a modal shown when a browser screen share track ends outside the app, for example from the browser sharing controls.',
|
||||
context: 'screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_CODEC_POLICY_FAILED_DESCRIPTOR = msg({
|
||||
message:
|
||||
'Your screen share could not be published with a compatible video codec, so it was stopped. Try sharing again.',
|
||||
comment:
|
||||
'Body of a modal shown when the video codec the browser negotiated for an active screen share is outside the codecs the app allows and no allowed codec could be published, which forces the share to stop.',
|
||||
context: 'screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_ENCODER_FAILED_DESCRIPTOR = msg({
|
||||
message: 'Your video encoder stopped encoding frames, so your screen share was stopped. Try sharing again.',
|
||||
comment:
|
||||
@@ -127,6 +135,7 @@ export interface ScreenShareReconnectSnapshot {
|
||||
}
|
||||
|
||||
const selectScreenShareSetEnabledOptions = selectVoiceEngineV2AppScreenShareSetEnabledOptions;
|
||||
const SCREEN_SHARE_VERIFIED_CODEC_CORRECTION_MAX = 1;
|
||||
|
||||
class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
private readonly lifecycle: VoiceScreenShareLifecycleStore;
|
||||
@@ -134,6 +143,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
private activeScreenShareEndListener: (() => void) | null = null;
|
||||
private endedScreenShareStopInFlight: Promise<void> | null = null;
|
||||
encoderVerificationTimer: NodeJS.Timeout | null = null;
|
||||
private readonly verifiedCodecCorrectionsByTrack = new WeakMap<MediaStreamTrack, number>();
|
||||
sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null;
|
||||
|
||||
readonly liveKitFlows: VoiceEngineV2AppScreenShareLiveKitFlows;
|
||||
@@ -420,7 +430,24 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
const recovered = await this.liveKitFlows.republishActiveShareWithCodec(room, track, codec);
|
||||
if (recovered) return;
|
||||
}
|
||||
this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec);
|
||||
this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec, 'stalled');
|
||||
}
|
||||
|
||||
private async correctVerifiedScreenShareCodec(
|
||||
room: Room | null,
|
||||
participant: LocalParticipant,
|
||||
track: LocalVideoTrack,
|
||||
action: Extract<ScreenShareEncoderVerificationAction, {kind: 'correct-negotiated'}>,
|
||||
): Promise<void> {
|
||||
if (!this.isScreenShareTrackPublishedInternal(participant, track)) return;
|
||||
const mediaStreamTrack = track.mediaStreamTrack;
|
||||
const corrections = this.verifiedCodecCorrectionsByTrack.get(mediaStreamTrack) ?? 0;
|
||||
if (corrections < SCREEN_SHARE_VERIFIED_CODEC_CORRECTION_MAX && action.alternative) {
|
||||
this.verifiedCodecCorrectionsByTrack.set(mediaStreamTrack, corrections + 1);
|
||||
const recovered = await this.liveKitFlows.republishActiveShareWithCodec(room, track, action.alternative);
|
||||
if (recovered) return;
|
||||
}
|
||||
this.stopScreenShareAfterEncoderFailure(room, participant, track, action.requested, 'codec-policy');
|
||||
}
|
||||
|
||||
private stopScreenShareAfterEncoderFailure(
|
||||
@@ -428,11 +455,22 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
participant: LocalParticipant,
|
||||
track: LocalVideoTrack,
|
||||
codec: VideoCodec,
|
||||
cause: 'stalled' | 'codec-policy',
|
||||
): void {
|
||||
logger.error('Screen share encoder produced no frames and no other codec took over; disabling screen share', {
|
||||
codec,
|
||||
});
|
||||
this.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_ENCODER_FAILED_DESCRIPTOR));
|
||||
if (cause === 'stalled') {
|
||||
logger.error('Screen share encoder produced no frames and no other codec took over; disabling screen share', {
|
||||
codec,
|
||||
});
|
||||
} else {
|
||||
logger.error('Screen share kept publishing a codec outside the publish policy; disabling screen share', {
|
||||
codec,
|
||||
});
|
||||
}
|
||||
this.showScreenShareEndedModalInternal(
|
||||
i18n._(
|
||||
cause === 'stalled' ? SCREEN_SHARE_ENCODER_FAILED_DESCRIPTOR : SCREEN_SHARE_CODEC_POLICY_FAILED_DESCRIPTOR,
|
||||
),
|
||||
);
|
||||
if (this.endedScreenShareStopInFlight) return;
|
||||
if (!this.isScreenShareTrackPublishedInternal(participant, track)) return;
|
||||
this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'});
|
||||
@@ -468,19 +506,44 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
() => sender.getStats(),
|
||||
codec,
|
||||
(failure) => {
|
||||
const failureReason =
|
||||
failure.reason === 'codec-mismatch' ? 'screen-share-codec-mismatch' : 'screen-share-encode-stalled';
|
||||
markScreenShareCodecEncodeRuntimeFailure(failure.codec, failureReason);
|
||||
logger.warn('Screen share encoder verification failed', {codec: failure.codec, failureReason});
|
||||
if (failure.reason !== 'stalled') return;
|
||||
void this.recoverActiveScreenShareAfterEncoderFailure(room, participant, track, failure.codec).catch(
|
||||
(error) => {
|
||||
logger.warn('Failed to recover screen share after encoder verification failure', {
|
||||
error,
|
||||
codec: failure.codec,
|
||||
const action = resolveScreenShareEncoderVerificationAction(failure);
|
||||
switch (action.kind) {
|
||||
case 'ignore-repeated-stall':
|
||||
return;
|
||||
case 'recover-stalled':
|
||||
logger.warn('Screen share encoder verification failed', {
|
||||
codec: action.codec,
|
||||
failureReason: 'screen-share-encode-stalled',
|
||||
});
|
||||
},
|
||||
);
|
||||
void this.recoverActiveScreenShareAfterEncoderFailure(room, participant, track, action.codec).catch(
|
||||
(error) => {
|
||||
logger.warn('Failed to recover screen share after encoder verification failure', {
|
||||
error,
|
||||
codec: action.codec,
|
||||
});
|
||||
},
|
||||
);
|
||||
return;
|
||||
case 'accept-negotiated':
|
||||
logger.info('Screen share publisher negotiated a different codec inside the publish policy', {
|
||||
requested: action.requested,
|
||||
negotiated: action.negotiated,
|
||||
});
|
||||
return;
|
||||
case 'correct-negotiated':
|
||||
logger.warn('Screen share is sending a codec outside the publish policy', {
|
||||
requested: action.requested,
|
||||
negotiated: action.negotiated,
|
||||
alternative: action.alternative,
|
||||
});
|
||||
void this.correctVerifiedScreenShareCodec(room, participant, track, action).catch((error) => {
|
||||
logger.warn('Failed to correct screen share codec after encoder verification', {
|
||||
error,
|
||||
requested: action.requested,
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
},
|
||||
);
|
||||
this.trackPlumbing.bindKeyFrameRequests(room, participant, track);
|
||||
|
||||
+95
-12
@@ -53,7 +53,11 @@ import {
|
||||
prepareHighFidelityScreenShareAudioTrack,
|
||||
SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS,
|
||||
} from '@app/features/voice/utils/AudioPublishOptions';
|
||||
import type {ScreenShareContentSource} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
type ScreenShareContentSource,
|
||||
type VideoPublishCodecPolicyViolation,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {commitNativeAudioBridgeReplacement} from '@app/features/voice/utils/NativeAudioCaptureBridge';
|
||||
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
|
||||
import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
|
||||
@@ -88,6 +92,24 @@ function isUserCancelledOrPermissionDeniedError(error: unknown): boolean {
|
||||
}
|
||||
|
||||
const COMMITTED_PUBLICATION_INVARIANT_ATTEMPTS = 2;
|
||||
const SCREEN_SHARE_PUBLISH_CODEC_CORRECTION_MAX = 1;
|
||||
|
||||
export class ScreenSharePublishCodecPolicyError extends Error {
|
||||
readonly violation: VideoPublishCodecPolicyViolation;
|
||||
|
||||
constructor(violation: VideoPublishCodecPolicyViolation) {
|
||||
super(
|
||||
`screen share negotiated ${violation.negotiated} after requesting ${violation.requested}; no allowed codec could be published`,
|
||||
);
|
||||
this.name = 'ScreenSharePublishCodecPolicyError';
|
||||
this.violation = violation;
|
||||
}
|
||||
}
|
||||
|
||||
interface EnforcedScreenSharePublish {
|
||||
effectivePublishOptions: TrackPublishOptions | undefined;
|
||||
track: LocalVideoTrack | undefined;
|
||||
}
|
||||
|
||||
interface ScreenShareReplacementSnapshot {
|
||||
videoTrack: MediaStreamTrack;
|
||||
@@ -332,8 +354,11 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
);
|
||||
if (enabled) this.adapter.setStreamingPriorityInternal(true);
|
||||
try {
|
||||
const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(enabled, publishOptions);
|
||||
await participant.setScreenShareEnabled(enabled, restOptions, effectivePublishOptions);
|
||||
const requestedPublishOptions = await this.adapter.getEffectivePublishOptionsInternal(enabled, publishOptions);
|
||||
await participant.setScreenShareEnabled(enabled, restOptions, requestedPublishOptions);
|
||||
const effectivePublishOptions = enabled
|
||||
? (await this.enforcePublishCodecPolicy(participant, requestedPublishOptions)).effectivePublishOptions
|
||||
: requestedPublishOptions;
|
||||
await this.finalizeSetEnabledSuccess(
|
||||
room,
|
||||
participant,
|
||||
@@ -386,23 +411,73 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
participant: LocalParticipant,
|
||||
videoTrack: LocalVideoTrack,
|
||||
audioTrack: LocalAudioTrack | undefined,
|
||||
effectivePublishOptions: TrackPublishOptions | undefined,
|
||||
requestedPublishOptions: TrackPublishOptions | undefined,
|
||||
publishedTracks: Array<LocalAudioTrack | LocalVideoTrack>,
|
||||
): Promise<void> {
|
||||
): Promise<TrackPublishOptions | undefined> {
|
||||
assert.ok(participant);
|
||||
assert.ok(videoTrack);
|
||||
await participant.publishTrack(videoTrack, {
|
||||
...effectivePublishOptions,
|
||||
...requestedPublishOptions,
|
||||
source: Track.Source.ScreenShare,
|
||||
stream: VoiceTrackSource.ScreenShare,
|
||||
});
|
||||
publishedTracks.push(videoTrack);
|
||||
const enforced = await this.enforcePublishCodecPolicy(participant, requestedPublishOptions);
|
||||
if (enforced.track && enforced.track !== videoTrack) {
|
||||
publishedTracks.splice(publishedTracks.indexOf(videoTrack), 1, enforced.track);
|
||||
}
|
||||
if (audioTrack) {
|
||||
prepareHighFidelityScreenShareAudioTrack(audioTrack.mediaStreamTrack);
|
||||
await participant.publishTrack(audioTrack, SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS);
|
||||
publishedTracks.push(audioTrack);
|
||||
}
|
||||
await enforceLocalMediaPublicationCap(participant, VoiceTrackSource.ScreenShare);
|
||||
return enforced.effectivePublishOptions;
|
||||
}
|
||||
|
||||
private async enforcePublishCodecPolicy(
|
||||
participant: LocalParticipant,
|
||||
requestedPublishOptions: TrackPublishOptions | undefined,
|
||||
): Promise<EnforcedScreenSharePublish> {
|
||||
let effectivePublishOptions = requestedPublishOptions;
|
||||
for (let corrections = 0; ; corrections++) {
|
||||
const publication = getLocalScreenShareVideoPublications(participant)[0];
|
||||
const track = (publication?.videoTrack ?? publication?.track) as LocalVideoTrack | undefined;
|
||||
const requested = effectivePublishOptions?.videoCodec;
|
||||
if (!publication || !track || !requested) return {effectivePublishOptions, track};
|
||||
const violation = findVideoPublishCodecPolicyViolation(requested, publication.options?.videoCodec ?? track.codec);
|
||||
if (!violation) return {effectivePublishOptions, track};
|
||||
logger.warn('Screen share published a codec outside the publish policy', {...violation, corrections});
|
||||
const mediaStreamTrack = track.mediaStreamTrack;
|
||||
const replaceAlreadyInFlight = this.adapter.isScreenSharePublicationReplaceInFlight();
|
||||
this.adapter.transitionScreenShareLifecycleInternal({type: 'share.publicationReplace.set', inFlight: true});
|
||||
const alternative = corrections < SCREEN_SHARE_PUBLISH_CODEC_CORRECTION_MAX ? violation.alternative : null;
|
||||
try {
|
||||
await participant.unpublishTrack(track, alternative === null);
|
||||
if (alternative === null) {
|
||||
throw new ScreenSharePublishCodecPolicyError(violation);
|
||||
}
|
||||
const nextPublishOptions: TrackPublishOptions = {...effectivePublishOptions, videoCodec: alternative};
|
||||
delete nextPublishOptions.backupCodec;
|
||||
delete nextPublishOptions.backupCodecPolicy;
|
||||
delete nextPublishOptions.scalabilityMode;
|
||||
delete nextPublishOptions.simulcast;
|
||||
effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, nextPublishOptions);
|
||||
await participant.publishTrack(mediaStreamTrack, {
|
||||
...effectivePublishOptions,
|
||||
source: Track.Source.ScreenShare,
|
||||
stream: VoiceTrackSource.ScreenShare,
|
||||
...(publication.trackName ? {name: publication.trackName} : {}),
|
||||
});
|
||||
} finally {
|
||||
if (!replaceAlreadyInFlight) {
|
||||
this.adapter.transitionScreenShareLifecycleInternal({
|
||||
type: 'share.publicationReplace.set',
|
||||
inFlight: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async finalizeDeviceShareSuccess(
|
||||
@@ -519,9 +594,15 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
const createdTracks: Array<LocalAudioTrack | LocalVideoTrack> = [];
|
||||
const publishedTracks: Array<LocalAudioTrack | LocalVideoTrack> = [];
|
||||
try {
|
||||
const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
|
||||
const requestedPublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
|
||||
const {videoTrack, audioTrack} = await this.createDeviceTracksForShare(options, createdTracks);
|
||||
await this.publishDeviceTracks(participant, videoTrack, audioTrack, effectivePublishOptions, publishedTracks);
|
||||
const effectivePublishOptions = await this.publishDeviceTracks(
|
||||
participant,
|
||||
videoTrack,
|
||||
audioTrack,
|
||||
requestedPublishOptions,
|
||||
publishedTracks,
|
||||
);
|
||||
await this.finalizeDeviceShareSuccess(
|
||||
room,
|
||||
participant,
|
||||
@@ -735,14 +816,15 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
let videoPublished = false;
|
||||
try {
|
||||
await participant.unpublishTrack(screenShareTrack, false);
|
||||
const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, nextPublishOptions);
|
||||
const requestedPublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, nextPublishOptions);
|
||||
await participant.publishTrack(mediaStreamTrack, {
|
||||
...effectivePublishOptions,
|
||||
...requestedPublishOptions,
|
||||
source: Track.Source.ScreenShare,
|
||||
stream: VoiceTrackSource.ScreenShare,
|
||||
...(publication.trackName ? {name: publication.trackName} : {}),
|
||||
});
|
||||
videoPublished = true;
|
||||
const {effectivePublishOptions} = await this.enforcePublishCodecPolicy(participant, requestedPublishOptions);
|
||||
await runScreenShareActivationRitual({
|
||||
adapter: this.adapter,
|
||||
room,
|
||||
@@ -1603,13 +1685,14 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
if (getLocalScreenSharePublications(participant).length > 0) {
|
||||
await this.adapter.cleanupLingeringScreenShareTracks(participant);
|
||||
}
|
||||
const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
|
||||
const requestedPublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
|
||||
await participant.publishTrack(snapshot.videoTrack, {
|
||||
...effectivePublishOptions,
|
||||
...requestedPublishOptions,
|
||||
source: Track.Source.ScreenShare,
|
||||
stream: VoiceTrackSource.ScreenShare,
|
||||
});
|
||||
videoPublished = true;
|
||||
const {effectivePublishOptions} = await this.enforcePublishCodecPolicy(participant, requestedPublishOptions);
|
||||
const audioPublished = await this.restoreReconnectAudio(participant, snapshot);
|
||||
await this.finalizeRestoreReconnectSuccess(room, participant, snapshot, effectivePublishOptions, audioPublished);
|
||||
return true;
|
||||
|
||||
@@ -9,6 +9,8 @@ import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
adjustScreenShareEncodingForCodec,
|
||||
getCodecCapabilityReport,
|
||||
resolveVideoPublishCodecPolicy,
|
||||
type VideoPublishCodecPolicy,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {getGpuEncoderReportSync, loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
|
||||
@@ -244,6 +246,15 @@ function getConfiguredBackupCodecForPrimary(primaryCodec: VideoCodec):
|
||||
return {codec: 'h264'};
|
||||
}
|
||||
|
||||
function resolveBackupCodecWithinPolicy(
|
||||
configured: TrackPublishOptions['backupCodec'] | undefined,
|
||||
policy: VideoPublishCodecPolicy,
|
||||
): TrackPublishOptions['backupCodec'] {
|
||||
if (configured === undefined || configured === true) return policy.backupCodec;
|
||||
if (configured === false) return false;
|
||||
return configured.codec !== policy.primary && policy.allowed.includes(configured.codec) ? configured : false;
|
||||
}
|
||||
|
||||
function getEffectiveScreenShareScalabilityMode(
|
||||
codec: VideoCodec,
|
||||
publishOptions?: TrackPublishOptions,
|
||||
@@ -292,8 +303,12 @@ export async function getEffectivePublishOptions(
|
||||
return publishOptions;
|
||||
}
|
||||
await waitForGpuEncoderReportForPublish(options);
|
||||
const preferredVideoCodec = publishOptions?.videoCodec ?? getPreferredScreenShareCodec();
|
||||
const backupCodec = publishOptions?.backupCodec ?? getConfiguredBackupCodecForPrimary(preferredVideoCodec);
|
||||
const policy = resolveVideoPublishCodecPolicy(publishOptions?.videoCodec ?? getPreferredScreenShareCodec());
|
||||
const preferredVideoCodec = policy.primary;
|
||||
const backupCodec = resolveBackupCodecWithinPolicy(
|
||||
publishOptions?.backupCodec ?? getConfiguredBackupCodecForPrimary(preferredVideoCodec),
|
||||
policy,
|
||||
);
|
||||
const backupCodecPolicy =
|
||||
publishOptions?.backupCodecPolicy ?? (backupCodec ? BackupCodecPolicy.SIMULCAST : undefined);
|
||||
const scalabilityMode = getEffectiveScreenShareScalabilityMode(preferredVideoCodec, publishOptions, {
|
||||
@@ -306,7 +321,7 @@ export async function getEffectivePublishOptions(
|
||||
degradationPreference: SCREEN_SHARE_DEGRADATION_PREFERENCE,
|
||||
simulcast: isSvcScreenShareCodec(preferredVideoCodec) ? false : (publishOptions?.simulcast ?? false),
|
||||
scalabilityMode,
|
||||
...(backupCodec !== undefined ? {backupCodec} : {}),
|
||||
backupCodec,
|
||||
...(backupCodecPolicy !== undefined ? {backupCodecPolicy} : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,12 +8,15 @@ let av1OptIn = false;
|
||||
let hevcOptIn = false;
|
||||
let desktop = true;
|
||||
let gpuReport: HardwareEncodeReport | null = null;
|
||||
let cameraPreference = 'auto';
|
||||
let openH264Status: {enabled: boolean; downloaded: boolean} | null = null;
|
||||
let codecCapabilityInfo: MockInstance<(...args: Array<unknown>) => void>;
|
||||
|
||||
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
|
||||
default: {
|
||||
getScreenShareAv1OptIn: () => av1OptIn,
|
||||
getScreenShareHevcOptIn: () => hevcOptIn,
|
||||
getPreferredVideoCodec: () => cameraPreference,
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -39,29 +42,34 @@ vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
|
||||
getOpenH264StatusSync: () => null,
|
||||
getOpenH264StatusSync: () => openH264Status,
|
||||
resetOpenH264Status: () => undefined,
|
||||
}));
|
||||
|
||||
const ALL_SENDER_CODECS = ['video/VP8', 'video/VP9', 'video/H264', 'video/H265', 'video/AV1'];
|
||||
let senderCodecs = ALL_SENDER_CODECS;
|
||||
|
||||
Object.defineProperty(globalThis, 'RTCRtpSender', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: {
|
||||
getCapabilities: () => ({
|
||||
codecs: [
|
||||
{mimeType: 'video/VP8'},
|
||||
{mimeType: 'video/VP9'},
|
||||
{mimeType: 'video/H264'},
|
||||
{mimeType: 'video/H265'},
|
||||
{mimeType: 'video/AV1'},
|
||||
],
|
||||
}),
|
||||
getCapabilities: () => ({codecs: senderCodecs.map((mimeType) => ({mimeType}))}),
|
||||
},
|
||||
});
|
||||
|
||||
const {getCodecCapabilityReport, resetCachedCodecCapabilities, selectAutomaticScreenShareCodec} = await import(
|
||||
'./CodecCapabilityDetector'
|
||||
);
|
||||
const {
|
||||
buildCameraPublishOptions,
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
getAllowedVideoPublishCodecs,
|
||||
getCodecCapabilityReport,
|
||||
getRoomVideoPublishDefaults,
|
||||
isVideoCodecAllowedForPublish,
|
||||
markScreenShareCodecEncodeRuntimeFailure,
|
||||
resetCachedCodecCapabilities,
|
||||
resolveScreenShareEncoderVerificationAction,
|
||||
resolveVideoPublishCodecPolicy,
|
||||
selectAutomaticScreenShareCodec,
|
||||
} = await import('./CodecCapabilityDetector');
|
||||
|
||||
const ALL_SOFTWARE: HardwareEncodeReport = {
|
||||
av1: 'software',
|
||||
@@ -158,3 +166,125 @@ describe('screen-share AV1/HEVC opt-in gate', () => {
|
||||
expect(getCodecCapabilityReport().h265.supported).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('video publish codec policy', () => {
|
||||
beforeEach(() => {
|
||||
av1OptIn = false;
|
||||
hevcOptIn = false;
|
||||
desktop = true;
|
||||
gpuReport = null;
|
||||
cameraPreference = 'auto';
|
||||
openH264Status = null;
|
||||
senderCodecs = ALL_SENDER_CODECS;
|
||||
resetCachedCodecCapabilities();
|
||||
});
|
||||
|
||||
it('keeps AV1 and HEVC out of the allowed set while the opt-ins are off', () => {
|
||||
expect(getAllowedVideoPublishCodecs()).toEqual(['vp8', 'h264', 'vp9']);
|
||||
expect(isVideoCodecAllowedForPublish('av1')).toBe(false);
|
||||
expect(isVideoCodecAllowedForPublish('h265')).toBe(false);
|
||||
});
|
||||
|
||||
it('admits AV1 and HEVC only once their opt-ins are on', () => {
|
||||
av1OptIn = true;
|
||||
expect(getAllowedVideoPublishCodecs()).toEqual(['vp8', 'h264', 'vp9', 'av1']);
|
||||
hevcOptIn = true;
|
||||
expect(getAllowedVideoPublishCodecs()).toEqual(['vp8', 'h264', 'vp9', 'av1', 'h265']);
|
||||
expect(resolveVideoPublishCodecPolicy('av1').primary).toBe('av1');
|
||||
expect(resolveVideoPublishCodecPolicy('h265').primary).toBe('h265');
|
||||
});
|
||||
|
||||
it('substitutes an opted-out request with the first compatible codec and never AV1 or HEVC', () => {
|
||||
expect(resolveVideoPublishCodecPolicy('av1')).toMatchObject({primary: 'h264', backupCodec: false});
|
||||
expect(resolveVideoPublishCodecPolicy('h265')).toMatchObject({primary: 'h264', backupCodec: false});
|
||||
expect(resolveVideoPublishCodecPolicy('vp9')).toMatchObject({primary: 'vp9', backupCodec: {codec: 'h264'}});
|
||||
expect(resolveVideoPublishCodecPolicy('h264')).toMatchObject({primary: 'h264', backupCodec: false});
|
||||
});
|
||||
|
||||
it('drops H.264 from the allowed set when the sender cannot encode it even though OpenH264 claims it is ready', () => {
|
||||
senderCodecs = ALL_SENDER_CODECS.filter((mimeType) => mimeType !== 'video/H264');
|
||||
openH264Status = {enabled: true, downloaded: true};
|
||||
expect(getCodecCapabilityReport().h264.supported).toBe(true);
|
||||
expect(isVideoCodecAllowedForPublish('h264')).toBe(false);
|
||||
expect(resolveVideoPublishCodecPolicy('h264')).toMatchObject({primary: 'vp9', backupCodec: false});
|
||||
expect(getRoomVideoPublishDefaults()).toEqual({videoCodec: 'vp9', backupCodec: false});
|
||||
});
|
||||
|
||||
it('drops a codec from the allowed set once it failed to encode at runtime', () => {
|
||||
expect(markScreenShareCodecEncodeRuntimeFailure('vp9', 'test')).toBe(true);
|
||||
expect(getAllowedVideoPublishCodecs()).toEqual(['vp8', 'h264']);
|
||||
expect(resolveVideoPublishCodecPolicy('vp9').primary).toBe('h264');
|
||||
});
|
||||
|
||||
it('hands the camera an explicit allowed codec for every preference while the opt-ins are off', () => {
|
||||
expect(buildCameraPublishOptions()).toEqual({
|
||||
videoCodec: 'vp9',
|
||||
backupCodec: {codec: 'h264'},
|
||||
backupCodecPolicy: 1,
|
||||
});
|
||||
for (const preference of ['auto', 'vp8', 'vp9', 'h264', 'av1', 'h265']) {
|
||||
cameraPreference = preference;
|
||||
const options = buildCameraPublishOptions();
|
||||
expect(options.videoCodec).toBeDefined();
|
||||
expect(getAllowedVideoPublishCodecs()).toContain(options.videoCodec);
|
||||
expect(options.videoCodec).not.toBe('av1');
|
||||
expect(options.videoCodec).not.toBe('h265');
|
||||
}
|
||||
cameraPreference = 'av1';
|
||||
expect(buildCameraPublishOptions().videoCodec).toBe('h264');
|
||||
av1OptIn = true;
|
||||
expect(buildCameraPublishOptions().videoCodec).toBe('av1');
|
||||
});
|
||||
|
||||
it('derives the room publish defaults from the same policy', () => {
|
||||
expect(getRoomVideoPublishDefaults()).toEqual({videoCodec: 'vp9', backupCodec: {codec: 'h264'}});
|
||||
cameraPreference = 'av1';
|
||||
expect(getRoomVideoPublishDefaults().videoCodec).toBe('h264');
|
||||
});
|
||||
|
||||
it('flags a negotiated codec outside the policy with a bounded alternative', () => {
|
||||
expect(findVideoPublishCodecPolicyViolation('h264', 'av1')).toEqual({
|
||||
requested: 'h264',
|
||||
negotiated: 'av1',
|
||||
alternative: 'vp9',
|
||||
});
|
||||
expect(findVideoPublishCodecPolicyViolation('h264', 'h264')).toBeNull();
|
||||
expect(findVideoPublishCodecPolicyViolation('h264', 'vp9')).toBeNull();
|
||||
expect(findVideoPublishCodecPolicyViolation('h264', undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('no longer blacklists the requested codec when the publisher negotiated a different one', () => {
|
||||
expect(
|
||||
resolveScreenShareEncoderVerificationAction({reason: 'codec-mismatch', codec: 'h264', activeCodecs: ['av1']}),
|
||||
).toEqual({kind: 'correct-negotiated', requested: 'h264', negotiated: ['av1'], alternative: 'vp9'});
|
||||
expect(isVideoCodecAllowedForPublish('h264')).toBe(true);
|
||||
expect(getCodecCapabilityReport().h264.supported).toBe(true);
|
||||
expect(
|
||||
resolveScreenShareEncoderVerificationAction({reason: 'codec-mismatch', codec: 'h264', activeCodecs: ['vp9']}),
|
||||
).toEqual({kind: 'accept-negotiated', requested: 'h264', negotiated: ['vp9']});
|
||||
expect(isVideoCodecAllowedForPublish('h264')).toBe(true);
|
||||
});
|
||||
|
||||
it('still blacklists a codec whose encoder stalled, and only warns about it once', () => {
|
||||
expect(resolveScreenShareEncoderVerificationAction({reason: 'stalled', codec: 'h264'})).toEqual({
|
||||
kind: 'recover-stalled',
|
||||
codec: 'h264',
|
||||
});
|
||||
expect(isVideoCodecAllowedForPublish('h264')).toBe(false);
|
||||
expect(getCodecCapabilityReport().h264).toMatchObject({supported: false, reason: 'runtime-failed'});
|
||||
expect(resolveScreenShareEncoderVerificationAction({reason: 'stalled', codec: 'h264'})).toEqual({
|
||||
kind: 'ignore-repeated-stall',
|
||||
codec: 'h264',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps recovering from a vp8 stall because vp8 is never blacklisted', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
expect(resolveScreenShareEncoderVerificationAction({reason: 'stalled', codec: 'vp8'})).toEqual({
|
||||
kind: 'recover-stalled',
|
||||
codec: 'vp8',
|
||||
});
|
||||
expect(isVideoCodecAllowedForPublish('vp8')).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -17,10 +17,13 @@ import {
|
||||
resetNativeHardwareEncoderCapabilities,
|
||||
} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
|
||||
import {getOpenH264StatusSync, resetOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
|
||||
import type {VideoCodec, VideoEncoding} from 'livekit-client';
|
||||
import type {TrackPublishDefaults, TrackPublishOptions} from 'livekit-client';
|
||||
import {BackupCodecPolicy, supportsVideoCodec, type VideoCodec, type VideoEncoding} from 'livekit-client';
|
||||
|
||||
const logger = new Logger('CodecCapabilityDetector');
|
||||
export const LIVEKIT_SUPPORTED_CODECS: ReadonlyArray<VideoCodec> = ['vp8', 'h264', 'vp9', 'av1', 'h265'];
|
||||
const PUBLISH_CODEC_FALLBACK_ORDER: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8', 'av1', 'h265'];
|
||||
const LAST_RESORT_PUBLISH_CODEC: VideoCodec = 'vp8';
|
||||
|
||||
export interface CodecCapabilities {
|
||||
vp8: boolean;
|
||||
@@ -478,6 +481,122 @@ export function markScreenShareCodecEncodeRuntimeFailure(codec: VideoCodec, reas
|
||||
return true;
|
||||
}
|
||||
|
||||
export type VideoPublishCodecDenial = 'sender-cannot-encode' | 'policy' | 'runtime-failed';
|
||||
|
||||
export interface VideoPublishCodecPolicy {
|
||||
allowed: ReadonlyArray<VideoCodec>;
|
||||
requested: VideoCodec;
|
||||
primary: VideoCodec;
|
||||
backupCodec: false | {codec: 'h264'};
|
||||
}
|
||||
|
||||
export interface VideoPublishCodecPolicyViolation {
|
||||
requested: VideoCodec;
|
||||
negotiated: VideoCodec;
|
||||
alternative: VideoCodec | null;
|
||||
}
|
||||
|
||||
export type ScreenShareEncoderVerificationAction =
|
||||
| {kind: 'recover-stalled'; codec: VideoCodec}
|
||||
| {kind: 'ignore-repeated-stall'; codec: VideoCodec}
|
||||
| {kind: 'accept-negotiated'; requested: VideoCodec; negotiated: ReadonlyArray<VideoCodec>}
|
||||
| {
|
||||
kind: 'correct-negotiated';
|
||||
requested: VideoCodec;
|
||||
negotiated: ReadonlyArray<VideoCodec>;
|
||||
alternative: VideoCodec | null;
|
||||
};
|
||||
|
||||
export function getVideoPublishCodecDenial(codec: VideoCodec): VideoPublishCodecDenial | null {
|
||||
if (!supportsVideoCodec(codec)) return 'sender-cannot-encode';
|
||||
if (getScreenShareCodecPolicyUnsupported(codec, buildScreenShareCodecPolicyContext())) return 'policy';
|
||||
if (runtimeEncodeFailureCodecs.has(codec)) return 'runtime-failed';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isVideoCodecAllowedForPublish(codec: VideoCodec): boolean {
|
||||
return getVideoPublishCodecDenial(codec) === null;
|
||||
}
|
||||
|
||||
export function getAllowedVideoPublishCodecs(): ReadonlyArray<VideoCodec> {
|
||||
return LIVEKIT_SUPPORTED_CODECS.filter(isVideoCodecAllowedForPublish);
|
||||
}
|
||||
|
||||
export function selectVideoPublishCodecAlternative(requested: VideoCodec): VideoCodec | null {
|
||||
return (
|
||||
PUBLISH_CODEC_FALLBACK_ORDER.find((codec) => codec !== requested && isVideoCodecAllowedForPublish(codec)) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveVideoPublishCodecPolicy(requested: VideoCodec): VideoPublishCodecPolicy {
|
||||
const allowed = getAllowedVideoPublishCodecs();
|
||||
const primary = allowed.includes(requested)
|
||||
? requested
|
||||
: (PUBLISH_CODEC_FALLBACK_ORDER.find((codec) => allowed.includes(codec)) ?? LAST_RESORT_PUBLISH_CODEC);
|
||||
if (primary !== requested) {
|
||||
logger.warn('Requested publish codec is outside the publish policy; substituting', {
|
||||
requested,
|
||||
primary,
|
||||
denial: getVideoPublishCodecDenial(requested),
|
||||
allowed,
|
||||
});
|
||||
}
|
||||
const backupCodec =
|
||||
primary !== 'h264' && primary !== 'vp8' && allowed.includes('h264') ? {codec: 'h264' as const} : false;
|
||||
return {allowed, requested, primary, backupCodec};
|
||||
}
|
||||
|
||||
export function getCameraPublishCodecPolicy(): VideoPublishCodecPolicy {
|
||||
return resolveVideoPublishCodecPolicy(selectOptimalCameraCodec(VoiceSettings.getPreferredVideoCodec()));
|
||||
}
|
||||
|
||||
export function buildCameraPublishOptions(codec?: VideoCodec): TrackPublishOptions {
|
||||
const policy = codec ? resolveVideoPublishCodecPolicy(codec) : getCameraPublishCodecPolicy();
|
||||
return {
|
||||
videoCodec: policy.primary,
|
||||
backupCodec: policy.backupCodec,
|
||||
...(policy.backupCodec ? {backupCodecPolicy: BackupCodecPolicy.SIMULCAST} : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function getRoomVideoPublishDefaults(): Pick<TrackPublishDefaults, 'videoCodec' | 'backupCodec'> {
|
||||
const policy = getCameraPublishCodecPolicy();
|
||||
return {
|
||||
videoCodec: policy.primary,
|
||||
backupCodec: policy.allowed.includes('h264') ? {codec: 'h264'} : false,
|
||||
};
|
||||
}
|
||||
|
||||
export function findVideoPublishCodecPolicyViolation(
|
||||
requested: VideoCodec,
|
||||
negotiated: VideoCodec | undefined,
|
||||
): VideoPublishCodecPolicyViolation | null {
|
||||
if (!negotiated || isVideoCodecAllowedForPublish(negotiated)) return null;
|
||||
return {requested, negotiated, alternative: selectVideoPublishCodecAlternative(requested)};
|
||||
}
|
||||
|
||||
export function resolveScreenShareEncoderVerificationAction(
|
||||
failure:
|
||||
| {reason: 'stalled'; codec: VideoCodec}
|
||||
| {reason: 'codec-mismatch'; codec: VideoCodec; activeCodecs: ReadonlyArray<VideoCodec>},
|
||||
): ScreenShareEncoderVerificationAction {
|
||||
if (failure.reason === 'stalled') {
|
||||
if (runtimeEncodeFailureCodecs.has(failure.codec)) return {kind: 'ignore-repeated-stall', codec: failure.codec};
|
||||
markScreenShareCodecEncodeRuntimeFailure(failure.codec, 'screen-share-encode-stalled');
|
||||
return {kind: 'recover-stalled', codec: failure.codec};
|
||||
}
|
||||
const disallowed = failure.activeCodecs.filter((codec) => !isVideoCodecAllowedForPublish(codec));
|
||||
if (disallowed.length === 0) {
|
||||
return {kind: 'accept-negotiated', requested: failure.codec, negotiated: failure.activeCodecs};
|
||||
}
|
||||
return {
|
||||
kind: 'correct-negotiated',
|
||||
requested: failure.codec,
|
||||
negotiated: disallowed,
|
||||
alternative: selectVideoPublishCodecAlternative(failure.codec),
|
||||
};
|
||||
}
|
||||
|
||||
export function selectNativeScreenCaptureScreenShareCodec(preference: CodecPreference = 'auto'): VideoCodec {
|
||||
return selectOptimalScreenShareCodec(preference);
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@ import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/v
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
type CodecCapabilityReport,
|
||||
getCameraPublishCodecPolicy,
|
||||
getCodecCapabilityReport,
|
||||
getLiveKitSupportedCodecs,
|
||||
selectOptimalCameraCodec,
|
||||
selectOptimalScreenShareCodec,
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
@@ -225,7 +225,7 @@ async function collectCodecMetadata(): Promise<Record<string, unknown>> {
|
||||
report = null;
|
||||
}
|
||||
try {
|
||||
resolvedCameraCodec = selectOptimalCameraCodec(cameraPreference);
|
||||
resolvedCameraCodec = getCameraPublishCodecPolicy().primary;
|
||||
} catch {
|
||||
resolvedCameraCodec = null;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user