feat(voice): drive screen share quality from viewer demand (#2421)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-03 04:49:29 +02:00
1 parent 66517c925b
commit 2426a5769d
5 files changed
+316 -40

No files matched your search

@@ -66,6 +66,7 @@ import {
useEffectiveTrackRef,
useIntersection,
useScreenShareAudioPublication,
useScreenShareViewerDemand,
useScreensharePreviewUploader,
useScreenshareWatchSubscription,
useTileContextMenuActive,
@@ -459,6 +460,12 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
onVideoSubscriptionError: reportVideoSubscriptionError,
getGraphSnapshot: getVoiceMediaGraphSnapshotForTile,
});
useScreenShareViewerDemand({
enabled: isInteractiveScreenShareTile && !isOwnScreenShare && isWatching && hasSubscribedScreenShareVideo,
publication,
videoRef,
onError: reportVideoSubscriptionError,
});
useEffect(() => {
if (!isScreenShare || isOwnScreenShare || isFocusedPlaceholderTile) return;
if (!isWatching) return;
@@ -3,6 +3,7 @@
import {Endpoints} from '@app/features/app/constants/Endpoints';
import {http} from '@app/features/platform/transport/RestTransport';
import {HttpError} from '@app/features/platform/types/EndpointError';
import {observeResize} from '@app/features/platform/utils/SharedResizeObserver';
import ContextMenu from '@app/features/ui/state/ContextMenu';
import PrivacyPreferences from '@app/features/user/state/PrivacyPreferences';
import {parseStreamKey} from '@app/features/voice/components/StreamKeys';
@@ -21,6 +22,7 @@ import {
type StreamPreviewUploadUrlCacheEntryLike,
} from '@app/features/voice/components/voice_participant_tile/StreamPreviewUploadPolicy';
import {logger} from '@app/features/voice/components/voice_participant_tile/shared';
import {resolveDevicePixelRatio} from '@app/features/voice/DevicePixelRatio';
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
import {useStoreVersion} from '@app/features/voice/engine/Store';
@@ -37,7 +39,13 @@ import {
import VoiceEngineV2AppSubscriptionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter';
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import {
applyScreenShareViewerDemand,
isWithinScreenShareViewerDemandDeadBand,
measureScreenShareViewerDemand,
releaseScreenShareViewerDemand,
SCREEN_SHARE_VIEWER_DEMAND_DEBOUNCE_MS,
type ScreenSharePublicationOperation,
type ScreenShareViewerDemandDimensions,
syncScreenSharePublication,
syncWatchedScreenSharePublications,
} from '@app/features/voice/utils/ScreenShareSubscriptionPolicy';
@@ -360,6 +368,66 @@ export function useScreenShareAudioPublication(
return {publication, hasTrack};
}
let nextScreenShareViewerDemandKey = 0;
export function useScreenShareViewerDemand({
enabled,
publication,
videoRef,
onError,
}: {
enabled: boolean;
publication: RemoteTrackPublication | null | undefined;
videoRef: React.RefObject<HTMLVideoElement | null>;
onError?: (operation: ScreenSharePublicationOperation, label: string, error: unknown) => void;
}): void {
const viewerKeyRef = useRef<string | null>(null);
if (viewerKeyRef.current === null) {
nextScreenShareViewerDemandKey += 1;
viewerKeyRef.current = `screen-share-viewer-${nextScreenShareViewerDemandKey}`;
}
const viewerKey = viewerKeyRef.current;
const trackSid = publication?.trackSid ?? null;
const onErrorRef = useRef(onError);
onErrorRef.current = onError;
useEffect(() => {
if (!enabled || !publication || !trackSid) return;
const element = videoRef.current;
if (!element) return;
const label = 'screen share video publication';
const handleError = (operation: ScreenSharePublicationOperation, errorLabel: string, error: unknown) => {
logger.error(`${operation} failed for ${errorLabel}`, error);
onErrorRef.current?.(operation, errorLabel, error);
};
let applied: ScreenShareViewerDemandDimensions | null = null;
let timeoutId: ReturnType<typeof setTimeout> | null = null;
const apply = () => {
timeoutId = null;
const target = videoRef.current;
const next = measureScreenShareViewerDemand(
target,
resolveDevicePixelRatio(target?.ownerDocument.defaultView ?? null),
);
if (!next || isWithinScreenShareViewerDemandDeadBand(applied, next)) return;
applied = next;
applyScreenShareViewerDemand({publication, label, viewerKey, dimensions: next, onError: handleError});
};
const schedule = () => {
if (timeoutId !== null) return;
timeoutId = setTimeout(apply, SCREEN_SHARE_VIEWER_DEMAND_DEBOUNCE_MS);
};
schedule();
const stopObserving = typeof ResizeObserver === 'undefined' ? null : observeResize(element, schedule);
return () => {
if (timeoutId !== null) {
clearTimeout(timeoutId);
}
stopObserving?.();
releaseScreenShareViewerDemand({publication, label, trackSid, viewerKey, onError: handleError});
};
}, [enabled, publication, trackSid, videoRef, viewerKey]);
}
export function useScreenshareWatchSubscription(opts: {
isScreenShare: boolean;
trackRef: TrackReferenceOrPlaceholder;
@@ -18,6 +18,10 @@ import {
type ScreenSharePublicationOperation,
ScreenShareWatchErrorCode,
} from '@app/features/voice/state/ScreenShareWatchFailures';
import {
clearScreenShareViewerDemand,
resolveScreenSharePublicationVideoRequest,
} from '@app/features/voice/utils/ScreenShareSubscriptionPolicy';
import type {RemoteParticipant, RemoteTrackPublication, Room} from 'livekit-client';
import {VideoQuality} from 'livekit-client';
@@ -30,6 +34,13 @@ const qualityMap: Record<VoiceMediaGraphVideoQuality, VideoQuality> = {
high: VideoQuality.HIGH,
};
const graphQualityMap = new Map<VideoQuality, VoiceMediaGraphVideoQuality>(
Object.entries(qualityMap).map(([graphQuality, videoQuality]): [VideoQuality, VoiceMediaGraphVideoQuality] => [
videoQuality,
graphQuality as VoiceMediaGraphVideoQuality,
]),
);
export class ScreenShareSubscriptionManager extends Store {
private room: Room | null = null;
private observers = new Map<string, IntersectionObserver>();
@@ -44,6 +55,7 @@ export class ScreenShareSubscriptionManager extends Store {
};
setRoom(room: Room | null): void {
clearScreenShareViewerDemand();
this.update(() => {
this.room = room;
});
@@ -292,10 +304,17 @@ export class ScreenShareSubscriptionManager extends Store {
participantIdentity: string,
publication: RemoteTrackPublication,
quality: VoiceMediaGraphVideoQuality,
): boolean {
return this.runPublicationOperation(participantIdentity, publication, 'setVideoQuality', () => {
publication.setVideoQuality(qualityMap[quality]);
): VoiceMediaGraphVideoQuality | null {
const request = resolveScreenSharePublicationVideoRequest(publication, qualityMap[quality]);
const applied = this.runPublicationOperation(participantIdentity, publication, request.operation, () => {
if (request.operation === 'setVideoDimensions') {
publication.setVideoDimensions(request.dimensions);
return;
}
publication.setVideoQuality(request.quality);
});
if (!applied) return null;
return graphQualityMap.get(request.quality) ?? null;
}
private createObserver(participantIdentity: string, element: HTMLElement): IntersectionObserver {
@@ -400,15 +419,16 @@ export class ScreenShareSubscriptionManager extends Store {
publication: RemoteTrackPublication,
enabled: boolean,
quality: VoiceMediaGraphVideoQuality,
): boolean {
): VoiceMediaGraphVideoQuality | null {
const subscribedApplied = this.runPublicationOperation(participantIdentity, publication, 'setSubscribed', () => {
publication.setSubscribed(true);
});
const enabledApplied = this.runPublicationOperation(participantIdentity, publication, 'setEnabled', () => {
publication.setEnabled(enabled);
});
const qualityApplied = this.applyQuality(participantIdentity, publication, quality);
return subscribedApplied && enabledApplied && qualityApplied;
const appliedQuality = this.applyQuality(participantIdentity, publication, quality);
if (!subscribedApplied || !enabledApplied) return null;
return appliedQuality;
}
private subscribePublication(
@@ -421,9 +441,12 @@ export class ScreenShareSubscriptionManager extends Store {
return;
}
this.withActiveScreenSharePublications(participantIdentity, (publications, participant) => {
let appliedQuality: VoiceMediaGraphVideoQuality | null = null;
let applied = true;
for (const publication of publications) {
applied = this.applySubscribeOperations(participantIdentity, publication, enabled, quality) && applied;
const publicationQuality = this.applySubscribeOperations(participantIdentity, publication, enabled, quality);
if (publicationQuality === null) applied = false;
else appliedQuality = publicationQuality;
}
for (const publication of ScreenSharePublicationMigration.getScreenSharePublicationsToDisable(participant)) {
this.runPublicationOperation(participantIdentity, publication, 'setSubscribed', () => {
@@ -433,7 +456,7 @@ export class ScreenShareSubscriptionManager extends Store {
if (!applied) return;
this.reportActualChanged(participantIdentity, {
enabled,
quality,
quality: appliedQuality,
trackSid: publications[0]?.trackSid ?? null,
});
logger.debug('Screen share subscribe command applied', {participantIdentity});
@@ -510,10 +533,11 @@ export class ScreenShareSubscriptionManager extends Store {
}
this.pendingResubscribePulses.delete(participantIdentity);
if (!this.isSubscriptionStillWanted(participantIdentity)) return;
if (!this.applySubscribeOperations(participantIdentity, publication, enabled, quality)) return;
const appliedQuality = this.applySubscribeOperations(participantIdentity, publication, enabled, quality);
if (appliedQuality === null) return;
this.reportActualChanged(participantIdentity, {
enabled,
quality,
quality: appliedQuality,
trackSid: publication.trackSid ?? null,
});
logger.debug('Screen share resubscribe pulse command applied after republish', {
@@ -555,13 +579,16 @@ export class ScreenShareSubscriptionManager extends Store {
private setPublicationQuality(participantIdentity: string, quality: VoiceMediaGraphVideoQuality): void {
this.withActiveScreenSharePublications(participantIdentity, (publications) => {
let appliedQuality: VoiceMediaGraphVideoQuality | null = null;
let applied = true;
for (const publication of publications) {
applied = this.applyQuality(participantIdentity, publication, quality) && applied;
const publicationQuality = this.applyQuality(participantIdentity, publication, quality);
if (publicationQuality === null) applied = false;
else appliedQuality = publicationQuality;
}
if (!applied) return;
this.reportActualChanged(participantIdentity, {quality});
logger.debug('Quality updated', {participantIdentity, quality});
this.reportActualChanged(participantIdentity, {quality: appliedQuality});
logger.debug('Quality updated', {participantIdentity, quality: appliedQuality});
});
}
}
@@ -11,6 +11,9 @@ import {
type VoiceMediaGraphSnapshot,
} from '../engine/VoiceMediaGraph';
import {voiceMediaGraphStore} from '../engine/VoiceMediaGraphStore';
import type {ScreenSharePublicationOperation} from '../utils/ScreenShareSubscriptionPolicy';
export type {ScreenSharePublicationOperation};
export const ScreenShareWatchErrorCode = {
SubscriptionSetSubscribedFailed: -2101,
@@ -19,6 +22,7 @@ export const ScreenShareWatchErrorCode = {
SubscriptionEmitTrackUpdateFailed: -2104,
ObserverAttachFailed: -2105,
ObserverDetachFailed: -2106,
SubscriptionSetVideoDimensionsFailed: -2107,
RemoteTrackSubscriptionFailed: -2202,
PublicationMissingTimeout: -2301,
SubscriptionAttachTimeout: -2302,
@@ -34,6 +38,7 @@ export type ScreenShareWatchFailureReason =
| 'subscription-set-enabled-failed'
| 'subscription-set-video-quality-failed'
| 'subscription-emit-track-update-failed'
| 'subscription-set-video-dimensions-failed'
| 'observer-attach-failed'
| 'observer-detach-failed'
| 'remote-track-subscription-failed'
@@ -64,8 +69,6 @@ export interface ScreenShareWatchFailureTarget {
source?: string | null;
}
export type ScreenSharePublicationOperation = 'setEnabled' | 'setSubscribed' | 'setVideoQuality' | 'emitTrackUpdate';
export interface ScreenShareWatchAttempt {
attemptKey: string;
startedAt: number;
@@ -92,6 +95,11 @@ export function getScreenShareWatchFailureForPublicationOperation(
code: ScreenShareWatchErrorCode.SubscriptionSetVideoQualityFailed,
reason: 'subscription-set-video-quality-failed',
};
case 'setVideoDimensions':
return {
code: ScreenShareWatchErrorCode.SubscriptionSetVideoDimensionsFailed,
reason: 'subscription-set-video-dimensions-failed',
};
case 'emitTrackUpdate':
return {
code: ScreenShareWatchErrorCode.SubscriptionEmitTrackUpdateFailed,
@@ -2,17 +2,30 @@
import {VideoQuality} from 'livekit-client';
export interface ScreenShareViewerDemandDimensions {
width: number;
height: number;
}
export interface ScreenSharePublicationTarget {
trackSid?: string | null;
isEnabled?: boolean;
isDesired?: boolean;
isSubscribed?: boolean;
videoQuality?: VideoQuality;
setEnabled?: (enabled: boolean) => void;
setSubscribed?: (subscribed: boolean) => void;
setVideoQuality?: (quality: VideoQuality) => void;
setVideoDimensions?: (dimensions: ScreenShareViewerDemandDimensions) => void;
emitTrackUpdate?: () => void;
}
export type ScreenSharePublicationOperation = 'setEnabled' | 'setSubscribed' | 'setVideoQuality' | 'emitTrackUpdate';
export type ScreenSharePublicationOperation =
| 'setEnabled'
| 'setSubscribed'
| 'setVideoQuality'
| 'setVideoDimensions'
| 'emitTrackUpdate';
export type ScreenSharePublicationErrorHandler = (
operation: ScreenSharePublicationOperation,
label: string,
@@ -24,7 +37,6 @@ export interface SyncScreenSharePublicationOptions {
label: string;
shouldSubscribe: boolean;
shouldEnable?: boolean;
videoQuality?: VideoQuality;
onError?: ScreenSharePublicationErrorHandler;
}
@@ -39,6 +51,180 @@ export interface SyncWatchedScreenSharePublicationsOptions {
onError?: ScreenSharePublicationErrorHandler;
}
const SCREEN_SHARE_VIEWER_DEMAND_DEAD_BAND_RATIO = 0.05;
export const SCREEN_SHARE_VIEWER_DEMAND_DEBOUNCE_MS = 500;
const SCREEN_SHARE_VIEWER_DEMAND_MAX_QUALITY = VideoQuality.HIGH;
const viewerDemandByTrackSid = new Map<string, Map<string, ScreenShareViewerDemandDimensions>>();
function resolveScreenShareViewerDemandPixelDensity(devicePixelRatio: number): number {
if (!Number.isFinite(devicePixelRatio) || devicePixelRatio <= 0) return 1;
return devicePixelRatio > 2 ? 2 : 1;
}
export function measureScreenShareViewerDemand(
element: {clientWidth: number; clientHeight: number} | null | undefined,
devicePixelRatio: number,
): ScreenShareViewerDemandDimensions | null {
if (!element) return null;
const density = resolveScreenShareViewerDemandPixelDensity(devicePixelRatio);
const width = Math.ceil(element.clientWidth * density);
const height = Math.ceil(element.clientHeight * density);
if (width <= 0 || height <= 0) return null;
return {width, height};
}
export function isWithinScreenShareViewerDemandDeadBand(
applied: ScreenShareViewerDemandDimensions | null,
next: ScreenShareViewerDemandDimensions,
ratio: number = SCREEN_SHARE_VIEWER_DEMAND_DEAD_BAND_RATIO,
): boolean {
if (!applied) return false;
return (
Math.abs(next.width - applied.width) <= applied.width * ratio &&
Math.abs(next.height - applied.height) <= applied.height * ratio
);
}
function allowsScreenShareViewerDemand(videoQuality: VideoQuality | undefined): boolean {
return (videoQuality ?? SCREEN_SHARE_VIEWER_DEMAND_MAX_QUALITY) === SCREEN_SHARE_VIEWER_DEMAND_MAX_QUALITY;
}
function hasScreenShareViewerDemandTrack(publication: ScreenSharePublicationTarget): boolean {
return publication.isSubscribed ?? true;
}
export function clearScreenShareViewerDemand(): void {
viewerDemandByTrackSid.clear();
}
function selectLargestScreenShareViewerDemand(
demands: Iterable<ScreenShareViewerDemandDimensions>,
): ScreenShareViewerDemandDimensions | null {
let largest: ScreenShareViewerDemandDimensions | null = null;
for (const demand of demands) {
largest =
largest === null
? demand
: {width: Math.max(largest.width, demand.width), height: Math.max(largest.height, demand.height)};
}
return largest;
}
function getScreenShareViewerDemand(trackSid: string | null | undefined): ScreenShareViewerDemandDimensions | null {
if (!trackSid) return null;
const demands = viewerDemandByTrackSid.get(trackSid);
if (!demands) return null;
return selectLargestScreenShareViewerDemand(demands.values());
}
function setPublicationVideoDimensions(
publication: ScreenSharePublicationTarget,
label: string,
dimensions: ScreenShareViewerDemandDimensions,
onError?: ScreenSharePublicationErrorHandler,
): void {
if (typeof publication.setVideoDimensions !== 'function') return;
try {
publication.setVideoDimensions(dimensions);
} catch (error) {
onError?.('setVideoDimensions', label, error);
}
}
function applyPublicationViewerDemand(
publication: ScreenSharePublicationTarget,
label: string,
onError?: ScreenSharePublicationErrorHandler,
): void {
if (!hasScreenShareViewerDemandTrack(publication) || !allowsScreenShareViewerDemand(publication.videoQuality)) return;
const demand = getScreenShareViewerDemand(publication.trackSid);
if (!demand) return;
setPublicationVideoDimensions(publication, label, demand, onError);
}
function restorePublicationVideoQuality(
publication: ScreenSharePublicationTarget,
label: string,
onError?: ScreenSharePublicationErrorHandler,
): void {
if (typeof publication.setVideoQuality !== 'function') return;
if (publication.isDesired === false || !allowsScreenShareViewerDemand(publication.videoQuality)) return;
try {
publication.setVideoQuality(SCREEN_SHARE_VIEWER_DEMAND_MAX_QUALITY);
} catch (error) {
onError?.('setVideoQuality', label, error);
}
}
export function applyScreenShareViewerDemand({
publication,
label,
viewerKey,
dimensions,
onError,
}: {
publication: ScreenSharePublicationTarget | null | undefined;
label: string;
viewerKey: string;
dimensions: ScreenShareViewerDemandDimensions;
onError?: ScreenSharePublicationErrorHandler;
}): void {
const trackSid = publication?.trackSid;
if (!publication || !trackSid) return;
let demands = viewerDemandByTrackSid.get(trackSid);
if (!demands) {
demands = new Map();
viewerDemandByTrackSid.set(trackSid, demands);
}
demands.set(viewerKey, dimensions);
applyPublicationViewerDemand(publication, label, onError);
}
export function releaseScreenShareViewerDemand({
publication,
label,
trackSid,
viewerKey,
onError,
}: {
publication: ScreenSharePublicationTarget | null | undefined;
label: string;
trackSid: string | null | undefined;
viewerKey: string;
onError?: ScreenSharePublicationErrorHandler;
}): void {
if (!trackSid) return;
const demands = viewerDemandByTrackSid.get(trackSid);
if (!demands) return;
demands.delete(viewerKey);
if (demands.size === 0) {
viewerDemandByTrackSid.delete(trackSid);
}
if (!publication || publication.trackSid !== trackSid) return;
if (demands.size === 0) {
restorePublicationVideoQuality(publication, label, onError);
return;
}
applyPublicationViewerDemand(publication, label, onError);
}
export type ScreenSharePublicationVideoRequest =
| {operation: 'setVideoQuality'; quality: VideoQuality}
| {operation: 'setVideoDimensions'; quality: VideoQuality; dimensions: ScreenShareViewerDemandDimensions};
export function resolveScreenSharePublicationVideoRequest(
publication: ScreenSharePublicationTarget,
videoQuality: VideoQuality,
): ScreenSharePublicationVideoRequest {
if (!hasScreenShareViewerDemandTrack(publication) || !allowsScreenShareViewerDemand(videoQuality)) {
return {operation: 'setVideoQuality', quality: videoQuality};
}
const dimensions = getScreenShareViewerDemand(publication.trackSid);
if (!dimensions) return {operation: 'setVideoQuality', quality: videoQuality};
return {operation: 'setVideoDimensions', quality: SCREEN_SHARE_VIEWER_DEMAND_MAX_QUALITY, dimensions};
}
function setPublicationEnabled(
publication: ScreenSharePublicationTarget,
label: string,
@@ -90,26 +276,11 @@ function setPublicationSubscribed(
}
}
function setPublicationVideoQuality(
publication: ScreenSharePublicationTarget,
label: string,
videoQuality: VideoQuality | undefined,
onError?: ScreenSharePublicationErrorHandler,
): void {
if (videoQuality === undefined || typeof publication.setVideoQuality !== 'function') return;
try {
publication.setVideoQuality(videoQuality);
} catch (error) {
onError?.('setVideoQuality', label, error);
}
}
export function syncScreenSharePublication({
publication,
label,
shouldSubscribe,
shouldEnable = shouldSubscribe,
videoQuality,
onError,
}: SyncScreenSharePublicationOptions): void {
if (!publication) return;
@@ -138,39 +309,35 @@ export function syncScreenSharePublication({
}
const forceTrackSettingsUpdate = didSubscribe && !shouldEnable && publication.isEnabled === shouldEnable;
setPublicationEnabled(publication, label, shouldEnable, forceTrackSettingsUpdate, onError);
setPublicationVideoQuality(publication, label, videoQuality, onError);
applyPublicationViewerDemand(publication, label, onError);
}
export function refreshScreenSharePublicationSubscription({
publication,
label,
shouldEnable = true,
videoQuality = VideoQuality.HIGH,
onError,
}: {
publication: ScreenSharePublicationTarget | null | undefined;
label: string;
shouldEnable?: boolean;
videoQuality?: VideoQuality;
onError?: ScreenSharePublicationErrorHandler;
}): void {
if (!publication) return;
setPublicationSubscribed(publication, label, true, onError);
setPublicationEnabled(publication, label, shouldEnable, true, onError);
setPublicationVideoQuality(publication, label, videoQuality, onError);
applyPublicationViewerDemand(publication, label, onError);
}
export function resubscribeScreenSharePublication({
publication,
label,
shouldEnable = true,
videoQuality = VideoQuality.HIGH,
onError,
}: {
publication: ScreenSharePublicationTarget | null | undefined;
label: string;
shouldEnable?: boolean;
videoQuality?: VideoQuality;
onError?: ScreenSharePublicationErrorHandler;
}): void {
if (!publication) return;
@@ -185,7 +352,7 @@ export function resubscribeScreenSharePublication({
setPublicationSubscribed(publication, label, false, onError);
setPublicationSubscribed(publication, label, true, onError);
setPublicationEnabled(publication, label, shouldEnable, true, onError);
setPublicationVideoQuality(publication, label, videoQuality, onError);
applyPublicationViewerDemand(publication, label, onError);
}
export function syncWatchedScreenSharePublications({
@@ -206,7 +373,6 @@ export function syncWatchedScreenSharePublications({
label: 'screen share video publication',
shouldSubscribe: shouldSubscribeVideo,
shouldEnable: shouldSubscribeVideo,
videoQuality: shouldSubscribeVideo ? VideoQuality.HIGH : undefined,
onError,
});
syncScreenSharePublication({