mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
refactor(voice): remove adaptive screen share quality system (#2416)
This commit is contained in:
Hampus authored and GitHub committed
1 parent
3e15b97c8c
commit
f5f60c66e7
51 files changed
+1245
-4872
No files matched your search
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
|
||||
import VoiceSettings, {
|
||||
type CameraResolution,
|
||||
@@ -69,7 +68,6 @@ type VoiceSettingsPatch = Partial<{
|
||||
screenShareSoftwareQuality: ScreenShareSoftwareQuality;
|
||||
screenShareScalabilityMode: ScreenShareScalabilityModePreference;
|
||||
screenShareBackupCodecMode: ScreenShareBackupCodecMode;
|
||||
adaptiveScreenShareQuality: boolean;
|
||||
vadThreshold: number;
|
||||
vadAutoSensitivity: boolean;
|
||||
vadEnhanced: boolean;
|
||||
@@ -179,13 +177,6 @@ function applyUpdatedVoiceSettings(
|
||||
if (settings.inputVolume !== undefined && !refreshInput) {
|
||||
MediaEngine.applyLocalInputVolume();
|
||||
}
|
||||
if (settings.adaptiveScreenShareQuality !== undefined) {
|
||||
if (settings.adaptiveScreenShareQuality) {
|
||||
AdaptiveScreenShareEngine.start(MediaEngine.room);
|
||||
} else {
|
||||
void AdaptiveScreenShareEngine.restoreConfiguredQuality();
|
||||
}
|
||||
}
|
||||
if (refreshInput) {
|
||||
refreshMicrophone();
|
||||
}
|
||||
|
||||
@@ -16,10 +16,8 @@ import {
|
||||
type StreamSettingsAudioControlLabelKey,
|
||||
selectStreamSettingsAudioMenuState,
|
||||
} from '@app/features/voice/components/StreamSettingsMenuContentStateMachine';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade';
|
||||
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import {useStoreVersion} from '@app/features/voice/engine/Store';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import {useMediaDevices} from '@app/features/voice/hooks/useMediaDevices';
|
||||
import VoiceSettings, {type ScreenshareResolution, type StreamingMode} from '@app/features/voice/state/VoiceSettings';
|
||||
@@ -139,16 +137,6 @@ const AUDIO_SETTINGS_DESCRIPTOR = msg({
|
||||
message: 'Audio settings',
|
||||
comment: 'Section header in the stream settings menu grouping audio-related toggles.',
|
||||
});
|
||||
const ADAPTIVE_QUALITY_DESCRIPTOR = msg({
|
||||
message: 'Adaptive quality',
|
||||
comment:
|
||||
'Toggle label in the stream settings menu. When enabled, productName can lower screen-share resolution if the encoder is CPU or bandwidth limited.',
|
||||
});
|
||||
const ADAPTIVE_QUALITY_ACTIVE_DESCRIPTOR = msg({
|
||||
message: 'Adjusted to {resolution} {frameRate} FPS',
|
||||
comment:
|
||||
'Inline adaptive-quality status in the stream settings menu. Shows the current automatically lowered resolution and frame rate.',
|
||||
});
|
||||
const STREAM_QUALITY_DESCRIPTOR = msg({
|
||||
message: 'Stream quality',
|
||||
comment: 'Compact submenu label for changing the resolution and frame rate of an active stream.',
|
||||
@@ -313,7 +301,6 @@ export async function pushActiveStreamSettings(
|
||||
}
|
||||
try {
|
||||
await MediaEngine.updateActiveScreenShareSettings(activeCaptureOptions, publishOptions);
|
||||
AdaptiveScreenShareEngine.start(MediaEngine.room);
|
||||
} catch (error) {
|
||||
logger.warn('Failed to push updated stream settings to the active share', error);
|
||||
}
|
||||
@@ -869,7 +856,6 @@ export const StreamSettingsMenuContent = observer(
|
||||
>
|
||||
<Trans>Hide preview thumbnail</Trans>
|
||||
</CheckboxItem>
|
||||
<AdaptiveQualityToggle data-flx="voice.stream-settings-menu-content.adaptive-quality-toggle" />
|
||||
</MenuGroup>
|
||||
</>
|
||||
);
|
||||
@@ -878,58 +864,6 @@ export const StreamSettingsMenuContent = observer(
|
||||
|
||||
StreamSettingsMenuContent.displayName = 'StreamSettingsMenuContent';
|
||||
|
||||
const RESOLUTION_LABELS: Record<ScreenshareResolution, string> = {
|
||||
low_240p: '240p',
|
||||
low_480p: '480p',
|
||||
medium: '720p',
|
||||
high: '1080p',
|
||||
ultra: '1440p',
|
||||
source: '',
|
||||
};
|
||||
|
||||
const AdaptiveQualityToggle = observer(() => {
|
||||
const {i18n} = useLingui();
|
||||
useStoreVersion(AdaptiveScreenShareEngine);
|
||||
const enabled = VoiceSettings.getAdaptiveScreenShareQuality();
|
||||
const snapshot = AdaptiveScreenShareEngine.qualitySnapshot;
|
||||
const label = i18n._(ADAPTIVE_QUALITY_DESCRIPTOR);
|
||||
const effectiveResolutionLabel =
|
||||
snapshot.effectiveResolution === 'source'
|
||||
? i18n._(SOURCE_DESCRIPTOR)
|
||||
: RESOLUTION_LABELS[snapshot.effectiveResolution];
|
||||
const adjustedStatus =
|
||||
enabled && snapshot.isAdapted
|
||||
? i18n._(ADAPTIVE_QUALITY_ACTIVE_DESCRIPTOR, {
|
||||
resolution: effectiveResolutionLabel,
|
||||
frameRate: snapshot.effectiveFrameRate,
|
||||
})
|
||||
: null;
|
||||
return (
|
||||
<CheckboxItem
|
||||
label={label}
|
||||
checked={enabled}
|
||||
onCheckedChange={(checked) => {
|
||||
VoiceSettingsCommands.update({adaptiveScreenShareQuality: checked});
|
||||
}}
|
||||
data-flx="voice.stream-settings-menu-content.adaptive-quality-toggle.checkbox-item"
|
||||
>
|
||||
<span className={styles.audioDeviceLabel} data-flx="voice.stream-settings-menu-content.adaptive-quality-label">
|
||||
<span className={styles.audioDeviceName} data-flx="voice.stream-settings-menu-content.adaptive-quality-name">
|
||||
{label}
|
||||
</span>
|
||||
{adjustedStatus && (
|
||||
<span
|
||||
className={styles.audioDeviceSubtext}
|
||||
data-flx="voice.stream-settings-menu-content.adaptive-quality-status"
|
||||
>
|
||||
{adjustedStatus}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</CheckboxItem>
|
||||
);
|
||||
});
|
||||
|
||||
const VenmicSettingsSubmenu = observer(({onSettingsChange}: {onSettingsChange?: () => void}) => {
|
||||
const {i18n} = useLingui();
|
||||
const workaround = VoiceSettings.getLinuxAudioCaptureWorkaround();
|
||||
|
||||
@@ -5,7 +5,6 @@ import Config from '@app/features/app/config/Config';
|
||||
import {getCachedDesktopTroubleshootingSettings} from '@app/features/devtools/utils/DesktopTroubleshootingUtils';
|
||||
import type {AppMetricsSnapshot, DesktopInfo, GpuInfo} from '@app/features/platform/types/Electron';
|
||||
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade';
|
||||
import ScreenShareCodecNegotiation, {
|
||||
getScreenShareCodecPreferenceOrder,
|
||||
@@ -157,7 +156,6 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
|
||||
clearInterval(id);
|
||||
};
|
||||
}, [enabled]);
|
||||
const adaptiveQualitySnapshot = AdaptiveScreenShareEngine.qualitySnapshot;
|
||||
const effectiveScreenShareSettings = resolveStreamingModeSettings(
|
||||
VoiceSettings.getStreamingMode(),
|
||||
VoiceSettings.getScreenshareResolution(),
|
||||
@@ -209,13 +207,6 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
|
||||
(getPublishedScreenShareMaxBitrateBps(localParticipant) ??
|
||||
getScreenShareBitrateBps(effectiveScreenShareSettings.resolution, effectiveScreenShareSettings.frameRate)) /
|
||||
1000000,
|
||||
adaptiveQuality: VoiceSettings.getAdaptiveScreenShareQuality(),
|
||||
adaptiveQualityAdapted: adaptiveQualitySnapshot.isAdapted,
|
||||
adaptiveQualityConfiguredResolution: adaptiveQualitySnapshot.configuredResolution,
|
||||
adaptiveQualityConfiguredFrameRate: adaptiveQualitySnapshot.configuredFrameRate,
|
||||
adaptiveQualityEffectiveResolution: adaptiveQualitySnapshot.effectiveResolution,
|
||||
adaptiveQualityEffectiveFrameRate: adaptiveQualitySnapshot.effectiveFrameRate,
|
||||
adaptiveQualityLimitationReason: adaptiveQualitySnapshot.limitationReason,
|
||||
audioSourceMode: VoiceSettings.getScreenShareAudioSourceMode(),
|
||||
audioIncludeSources: VoiceSettings.getScreenShareAudioIncludeSources(),
|
||||
audioExcludeSources: VoiceSettings.getScreenShareAudioExcludeSources(),
|
||||
|
||||
@@ -59,13 +59,6 @@ function createStatsData(): StatsForNerdsData {
|
||||
scalabilityMode: 'none',
|
||||
backupCodecMode: 'auto',
|
||||
maxBitrateMbps: 4.5,
|
||||
adaptiveQuality: true,
|
||||
adaptiveQualityAdapted: false,
|
||||
adaptiveQualityConfiguredResolution: '1080p',
|
||||
adaptiveQualityConfiguredFrameRate: 30,
|
||||
adaptiveQualityEffectiveResolution: '1080p',
|
||||
adaptiveQualityEffectiveFrameRate: 30,
|
||||
adaptiveQualityLimitationReason: 'none',
|
||||
audioSourceMode: 'none',
|
||||
audioIncludeSources: [],
|
||||
audioExcludeSources: [],
|
||||
|
||||
@@ -1,655 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import i18n from '@app/app/I18n';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
|
||||
import {Store} from '@app/features/voice/engine/Store';
|
||||
import {updateVoiceEngineV2ScreenEncodingFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import VoiceSettings, {type ScreenshareResolution, type StreamingMode} from '@app/features/voice/state/VoiceSettings';
|
||||
import {isNativeScreenShareTrack} from '@app/features/voice/utils/native_screen_capture_bridge/shared';
|
||||
import {
|
||||
getScreenShareBitrateBps,
|
||||
getScreenShareDimensions,
|
||||
resolveScreenShareFrameRate,
|
||||
SCREEN_SHARE_DEGRADATION_PREFERENCE,
|
||||
SUPPORTED_SCREEN_SHARE_FRAME_RATES,
|
||||
type SupportedScreenShareFrameRate,
|
||||
} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {msg} from '@lingui/core/macro';
|
||||
import type {LocalParticipant, LocalVideoTrack, Room, Track} from 'livekit-client';
|
||||
|
||||
const logger = new Logger('AdaptiveScreenShareEngine');
|
||||
const SCREEN_SHARE_SOURCE = VoiceTrackSource.ScreenShare as Track.Source;
|
||||
|
||||
const POLL_INTERVAL_MS = 2500;
|
||||
const STEP_DOWN_THRESHOLD = 4;
|
||||
const STEP_UP_THRESHOLD = 3;
|
||||
const FRAME_RATE_GRACE_FPS = 8;
|
||||
const STEP_DOWN_COOLDOWN_MS = 10_000;
|
||||
const STEP_UP_COOLDOWN_MS = 5_000;
|
||||
const MAX_STEP_UP_COOLDOWN_MS = 30_000;
|
||||
|
||||
const SCREEN_SHARE_240P_DESCRIPTOR = msg({
|
||||
message: '240p',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_480P_DESCRIPTOR = msg({
|
||||
message: '480p',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_720P_DESCRIPTOR = msg({
|
||||
message: '720p',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_1080P_DESCRIPTOR = msg({
|
||||
message: '1080p',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_1440P_DESCRIPTOR = msg({
|
||||
message: '1440p',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const SCREEN_SHARE_SOURCE_QUALITY_DESCRIPTOR = msg({
|
||||
message: 'source quality',
|
||||
comment: 'Resolution label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const BANDWIDTH_LIMITED_DESCRIPTOR = msg({
|
||||
message: 'bandwidth limited',
|
||||
comment: 'Reason label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const CPU_LIMITED_DESCRIPTOR = msg({
|
||||
message: 'CPU limited',
|
||||
comment: 'Reason label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const QUALITY_LIMITED_DESCRIPTOR = msg({
|
||||
message: 'quality limited',
|
||||
comment: 'Fallback reason label in adaptive screen-share quality toasts.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const LOWERED_SCREEN_SHARE_FRAME_RATE_DESCRIPTOR = msg({
|
||||
message: 'Lowered screen share to {frameRate} FPS - {reason}',
|
||||
comment:
|
||||
'Toast shown after adaptive screen-share quality lowers frame rate. {frameRate} is a number and {reason} is the limitation reason.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const LOWERED_SCREEN_SHARE_RESOLUTION_DESCRIPTOR = msg({
|
||||
message: 'Lowered screen share to {resolution} - {reason}',
|
||||
comment:
|
||||
'Toast shown after adaptive screen-share quality lowers resolution. {resolution} is a resolution label and {reason} is the limitation reason.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
const RAISED_SCREEN_SHARE_RESOLUTION_DESCRIPTOR = msg({
|
||||
message: 'Raised screen share to {resolution}',
|
||||
comment: 'Toast shown after adaptive screen-share quality raises resolution. {resolution} is a resolution label.',
|
||||
context: 'adaptive-screen-share',
|
||||
});
|
||||
|
||||
const RESOLUTION_LADDER: ReadonlyArray<ScreenshareResolution> = [
|
||||
'source',
|
||||
'ultra',
|
||||
'high',
|
||||
'medium',
|
||||
'low_480p',
|
||||
'low_240p',
|
||||
];
|
||||
|
||||
export type QualityLimitationReason = 'none' | 'cpu' | 'bandwidth' | 'other' | 'unknown';
|
||||
|
||||
export interface AdaptiveQualitySnapshot {
|
||||
configuredResolution: ScreenshareResolution;
|
||||
configuredFrameRate: SupportedScreenShareFrameRate;
|
||||
effectiveResolution: ScreenshareResolution;
|
||||
effectiveFrameRate: SupportedScreenShareFrameRate;
|
||||
limitationReason: QualityLimitationReason;
|
||||
isAdapted: boolean;
|
||||
}
|
||||
|
||||
export interface OutboundVideoAdaptationStats {
|
||||
timestamp: number;
|
||||
qualityLimitationReason: QualityLimitationReason;
|
||||
framesEncoded?: number;
|
||||
framesSent?: number;
|
||||
framesPerSecond?: number;
|
||||
frameWidth?: number;
|
||||
frameHeight?: number;
|
||||
}
|
||||
|
||||
export type PreviousOutboundVideoSample = OutboundVideoAdaptationStats | null;
|
||||
|
||||
interface ScreenShareSender {
|
||||
track: LocalVideoTrack;
|
||||
sender: RTCRtpSender;
|
||||
}
|
||||
|
||||
function getConfiguredQuality(): {
|
||||
resolution: ScreenshareResolution;
|
||||
frameRate: SupportedScreenShareFrameRate;
|
||||
} {
|
||||
return {
|
||||
resolution: VoiceSettings.getScreenshareResolution(),
|
||||
frameRate: resolveScreenShareFrameRate(VoiceSettings.getVideoFrameRate()),
|
||||
};
|
||||
}
|
||||
|
||||
function getResolutionIndex(resolution: ScreenshareResolution): number {
|
||||
return RESOLUTION_LADDER.indexOf(resolution);
|
||||
}
|
||||
|
||||
function getLowerResolution(resolution: ScreenshareResolution): ScreenshareResolution | null {
|
||||
const index = getResolutionIndex(resolution);
|
||||
if (index < 0 || index >= RESOLUTION_LADDER.length - 1) return null;
|
||||
return RESOLUTION_LADDER[index + 1];
|
||||
}
|
||||
|
||||
function getLowerFrameRate(frameRate: SupportedScreenShareFrameRate): SupportedScreenShareFrameRate | null {
|
||||
const index = SUPPORTED_SCREEN_SHARE_FRAME_RATES.indexOf(frameRate);
|
||||
if (index <= 0) return null;
|
||||
return SUPPORTED_SCREEN_SHARE_FRAME_RATES[index - 1] ?? null;
|
||||
}
|
||||
|
||||
function shouldPreferFrameRateStepDown(mode: StreamingMode, reason: QualityLimitationReason): boolean {
|
||||
if (mode === 'gaming') return false;
|
||||
if (mode === 'screenshare') return true;
|
||||
return reason === 'cpu';
|
||||
}
|
||||
|
||||
function getHigherResolution(
|
||||
resolution: ScreenshareResolution,
|
||||
configuredResolution: ScreenshareResolution,
|
||||
): ScreenshareResolution | null {
|
||||
const currentIndex = getResolutionIndex(resolution);
|
||||
const configuredIndex = getResolutionIndex(configuredResolution);
|
||||
if (currentIndex < 0 || configuredIndex < 0 || currentIndex <= configuredIndex) return null;
|
||||
return RESOLUTION_LADDER[currentIndex - 1];
|
||||
}
|
||||
|
||||
function getFrameCounter(stats: OutboundVideoAdaptationStats): number | undefined {
|
||||
return stats.framesSent ?? stats.framesEncoded;
|
||||
}
|
||||
|
||||
function formatResolutionForToast(resolution: ScreenshareResolution): string {
|
||||
switch (resolution) {
|
||||
case 'low_240p':
|
||||
return i18n._(SCREEN_SHARE_240P_DESCRIPTOR);
|
||||
case 'low_480p':
|
||||
return i18n._(SCREEN_SHARE_480P_DESCRIPTOR);
|
||||
case 'medium':
|
||||
return i18n._(SCREEN_SHARE_720P_DESCRIPTOR);
|
||||
case 'high':
|
||||
return i18n._(SCREEN_SHARE_1080P_DESCRIPTOR);
|
||||
case 'ultra':
|
||||
return i18n._(SCREEN_SHARE_1440P_DESCRIPTOR);
|
||||
case 'source':
|
||||
return i18n._(SCREEN_SHARE_SOURCE_QUALITY_DESCRIPTOR);
|
||||
}
|
||||
}
|
||||
|
||||
function formatLimitationReasonForToast(reason: QualityLimitationReason): string {
|
||||
if (reason === 'bandwidth') return i18n._(BANDWIDTH_LIMITED_DESCRIPTOR);
|
||||
if (reason === 'cpu') return i18n._(CPU_LIMITED_DESCRIPTOR);
|
||||
return i18n._(QUALITY_LIMITED_DESCRIPTOR);
|
||||
}
|
||||
|
||||
function showAdaptiveScreenShareToast(message: string): void {
|
||||
try {
|
||||
ToastCommands.createToast({
|
||||
type: 'info',
|
||||
children: message,
|
||||
timeout: 5000,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.debug('Failed to show adaptive screen-share quality toast', {error});
|
||||
}
|
||||
}
|
||||
|
||||
export function parseQualityLimitationReason(value: unknown): QualityLimitationReason {
|
||||
if (value === 'none' || value === 'cpu' || value === 'bandwidth' || value === 'other') {
|
||||
return value;
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
export function isLimitedQualityReason(reason: QualityLimitationReason): boolean {
|
||||
return reason === 'cpu' || reason === 'bandwidth';
|
||||
}
|
||||
|
||||
function isOutboundVideoReport(report: RTCStats): boolean {
|
||||
const outbound = report as RTCOutboundRtpStreamStats & {
|
||||
kind?: string;
|
||||
mediaType?: string;
|
||||
};
|
||||
if (outbound.type !== 'outbound-rtp') return false;
|
||||
return outbound.kind === 'video' || outbound.mediaType === 'video';
|
||||
}
|
||||
|
||||
export function extractOutboundVideoAdaptationStats(stats: RTCStatsReport): OutboundVideoAdaptationStats | null {
|
||||
for (const raw of stats.values()) {
|
||||
const report = raw as RTCOutboundRtpStreamStats & {
|
||||
qualityLimitationReason?: string;
|
||||
framesEncoded?: number;
|
||||
framesSent?: number;
|
||||
framesPerSecond?: number;
|
||||
frameWidth?: number;
|
||||
frameHeight?: number;
|
||||
};
|
||||
if (!isOutboundVideoReport(report)) continue;
|
||||
return {
|
||||
timestamp: report.timestamp,
|
||||
qualityLimitationReason: parseQualityLimitationReason(report.qualityLimitationReason),
|
||||
framesEncoded: typeof report.framesEncoded === 'number' ? report.framesEncoded : undefined,
|
||||
framesSent: typeof report.framesSent === 'number' ? report.framesSent : undefined,
|
||||
framesPerSecond: typeof report.framesPerSecond === 'number' ? report.framesPerSecond : undefined,
|
||||
frameWidth: typeof report.frameWidth === 'number' ? report.frameWidth : undefined,
|
||||
frameHeight: typeof report.frameHeight === 'number' ? report.frameHeight : undefined,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function computeOutboundFrameRate(
|
||||
current: OutboundVideoAdaptationStats,
|
||||
previous: PreviousOutboundVideoSample,
|
||||
): number | null {
|
||||
if (typeof current.framesPerSecond === 'number' && Number.isFinite(current.framesPerSecond)) {
|
||||
return current.framesPerSecond;
|
||||
}
|
||||
if (!previous) return null;
|
||||
const currentFrames = getFrameCounter(current);
|
||||
const previousFrames = getFrameCounter(previous);
|
||||
if (currentFrames === undefined || previousFrames === undefined) return null;
|
||||
const frameDelta = currentFrames - previousFrames;
|
||||
const timeDeltaMs = current.timestamp - previous.timestamp;
|
||||
if (frameDelta < 0 || timeDeltaMs <= 0) return null;
|
||||
return (frameDelta * 1000) / timeDeltaMs;
|
||||
}
|
||||
|
||||
export function isOutboundFrameRateBelowTarget(
|
||||
current: OutboundVideoAdaptationStats,
|
||||
previous: PreviousOutboundVideoSample,
|
||||
targetFrameRate: number,
|
||||
): boolean {
|
||||
const frameRate = computeOutboundFrameRate(current, previous);
|
||||
if (frameRate == null) return false;
|
||||
return frameRate < Math.max(1, targetFrameRate - FRAME_RATE_GRACE_FPS);
|
||||
}
|
||||
|
||||
export function computeResolutionScale(
|
||||
fromResolution: ScreenshareResolution,
|
||||
toResolution: ScreenshareResolution,
|
||||
): number {
|
||||
const from = getScreenShareDimensions(fromResolution);
|
||||
const to = getScreenShareDimensions(toResolution);
|
||||
return Math.min(to.width / from.width, to.height / from.height);
|
||||
}
|
||||
|
||||
export async function applyResolutionFrameRateAndBitrate(
|
||||
track: LocalVideoTrack,
|
||||
sender: RTCRtpSender,
|
||||
resolution: ScreenshareResolution,
|
||||
frameRate: SupportedScreenShareFrameRate,
|
||||
): Promise<void> {
|
||||
const dimensions = getScreenShareDimensions(resolution);
|
||||
const maxBitrate = getScreenShareBitrateBps(resolution, frameRate);
|
||||
if (isNativeScreenShareTrack(track.mediaStreamTrack)) {
|
||||
const updated = updateVoiceEngineV2ScreenEncodingFromMediaEngine({
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
frameRate,
|
||||
maxBitrateBps: maxBitrate,
|
||||
});
|
||||
if (!updated) {
|
||||
logger.warn('Skipped native screen-share encoding update because v2 screen state is unavailable', {
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
frameRate,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
const currentSettings = track.mediaStreamTrack.getSettings?.();
|
||||
const scaleResolutionDownBy =
|
||||
typeof currentSettings?.width === 'number' &&
|
||||
typeof currentSettings.height === 'number' &&
|
||||
currentSettings.width > 0 &&
|
||||
currentSettings.height > 0
|
||||
? Math.max(1, currentSettings.width / dimensions.width, currentSettings.height / dimensions.height)
|
||||
: undefined;
|
||||
await track.mediaStreamTrack.applyConstraints({
|
||||
width: {ideal: dimensions.width},
|
||||
height: {ideal: dimensions.height},
|
||||
frameRate: {ideal: frameRate, max: frameRate},
|
||||
});
|
||||
if (typeof track.setDegradationPreference === 'function') {
|
||||
await track.setDegradationPreference(SCREEN_SHARE_DEGRADATION_PREFERENCE);
|
||||
}
|
||||
const parameters = sender.getParameters();
|
||||
const encodings = parameters.encodings?.length ? parameters.encodings : [{}];
|
||||
parameters.degradationPreference = SCREEN_SHARE_DEGRADATION_PREFERENCE;
|
||||
parameters.encodings = encodings.map((encoding) => ({
|
||||
...encoding,
|
||||
...(scaleResolutionDownBy !== undefined ? {scaleResolutionDownBy} : {}),
|
||||
maxBitrate,
|
||||
maxFramerate: frameRate,
|
||||
priority: encoding.priority ?? 'high',
|
||||
networkPriority: encoding.networkPriority ?? 'high',
|
||||
}));
|
||||
await sender.setParameters(parameters);
|
||||
}
|
||||
|
||||
class AdaptiveScreenShareEngine extends Store {
|
||||
effectiveResolution: ScreenshareResolution | null = null;
|
||||
effectiveFrameRate: SupportedScreenShareFrameRate | null = null;
|
||||
limitationReason: QualityLimitationReason = 'none';
|
||||
isAdapted = false;
|
||||
private configuredResolution: ScreenshareResolution | null = null;
|
||||
private configuredFrameRate: SupportedScreenShareFrameRate | null = null;
|
||||
private room: Room | null = null;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private pollInFlightGeneration: number | null = null;
|
||||
private pollGeneration = 0;
|
||||
private stepDownStreak = 0;
|
||||
private stepUpStreak = 0;
|
||||
private lastStepDownAt = 0;
|
||||
private lastStepUpAt = 0;
|
||||
private stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
|
||||
private previousSample: PreviousOutboundVideoSample = null;
|
||||
|
||||
get qualitySnapshot(): AdaptiveQualitySnapshot {
|
||||
const configured = getConfiguredQuality();
|
||||
const configuredResolution = this.configuredResolution ?? configured.resolution;
|
||||
const configuredFrameRate = this.configuredFrameRate ?? configured.frameRate;
|
||||
return {
|
||||
configuredResolution,
|
||||
configuredFrameRate,
|
||||
effectiveResolution: this.effectiveResolution ?? configuredResolution,
|
||||
effectiveFrameRate: this.effectiveFrameRate ?? configuredFrameRate,
|
||||
limitationReason: this.limitationReason,
|
||||
isAdapted: this.isAdapted,
|
||||
};
|
||||
}
|
||||
|
||||
start(room: Room | null | undefined): void {
|
||||
this.stop();
|
||||
if (!room || !VoiceSettings.getAdaptiveScreenShareQuality()) {
|
||||
return;
|
||||
}
|
||||
if (!this.getScreenShareSender(room)) {
|
||||
return;
|
||||
}
|
||||
const configured = getConfiguredQuality();
|
||||
let generation = 0;
|
||||
this.update(() => {
|
||||
this.room = room;
|
||||
this.configuredResolution = configured.resolution;
|
||||
this.configuredFrameRate = configured.frameRate;
|
||||
this.effectiveResolution = configured.resolution;
|
||||
this.effectiveFrameRate = configured.frameRate;
|
||||
this.limitationReason = 'none';
|
||||
this.isAdapted = false;
|
||||
generation = ++this.pollGeneration;
|
||||
this.timer = setInterval(() => {
|
||||
void this.poll(generation);
|
||||
}, POLL_INTERVAL_MS);
|
||||
});
|
||||
void this.poll(generation);
|
||||
logger.info('Started adaptive screen share quality monitor', configured);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
}
|
||||
this.update(() => {
|
||||
this.timer = null;
|
||||
this.room = null;
|
||||
this.pollInFlightGeneration = null;
|
||||
this.pollGeneration++;
|
||||
this.stepDownStreak = 0;
|
||||
this.stepUpStreak = 0;
|
||||
this.lastStepDownAt = 0;
|
||||
this.lastStepUpAt = 0;
|
||||
this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
|
||||
this.previousSample = null;
|
||||
this.configuredResolution = null;
|
||||
this.configuredFrameRate = null;
|
||||
this.effectiveResolution = null;
|
||||
this.effectiveFrameRate = null;
|
||||
this.limitationReason = 'none';
|
||||
this.isAdapted = false;
|
||||
});
|
||||
}
|
||||
|
||||
async restoreConfiguredQuality(): Promise<void> {
|
||||
const room = this.room;
|
||||
const snapshot = this.qualitySnapshot;
|
||||
const wasAdapted = this.isAdapted;
|
||||
const screenShare = room ? this.getScreenShareSender(room) : null;
|
||||
this.stop();
|
||||
if (!wasAdapted || !screenShare) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await applyResolutionFrameRateAndBitrate(
|
||||
screenShare.track,
|
||||
screenShare.sender,
|
||||
snapshot.configuredResolution,
|
||||
snapshot.configuredFrameRate,
|
||||
);
|
||||
logger.info('Restored configured screen share quality', {
|
||||
resolution: snapshot.configuredResolution,
|
||||
frameRate: snapshot.configuredFrameRate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn('Failed to restore configured screen share quality', {error});
|
||||
}
|
||||
}
|
||||
|
||||
private getScreenShareSender(room: Room): ScreenShareSender | null {
|
||||
const participant = room.localParticipant as LocalParticipant | undefined;
|
||||
const publication = participant?.getTrackPublication(SCREEN_SHARE_SOURCE);
|
||||
const track = publication?.videoTrack as LocalVideoTrack | undefined;
|
||||
const sender = track?.sender;
|
||||
if (!track || !sender || track.mediaStreamTrack.readyState === 'ended') {
|
||||
return null;
|
||||
}
|
||||
return {track, sender};
|
||||
}
|
||||
|
||||
private async poll(generation: number): Promise<void> {
|
||||
if (this.pollInFlightGeneration != null || generation !== this.pollGeneration) return;
|
||||
const room = this.room;
|
||||
if (!room) return;
|
||||
if (!VoiceSettings.getAdaptiveScreenShareQuality()) {
|
||||
await this.restoreConfiguredQuality();
|
||||
return;
|
||||
}
|
||||
const screenShare = this.getScreenShareSender(room);
|
||||
if (!screenShare) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.pollInFlightGeneration = generation;
|
||||
try {
|
||||
const report = await screenShare.sender.getStats();
|
||||
if (generation !== this.pollGeneration) return;
|
||||
const stats = extractOutboundVideoAdaptationStats(report);
|
||||
if (!stats) return;
|
||||
const previousSample = this.previousSample;
|
||||
this.previousSample = stats;
|
||||
const targetFrameRate = this.effectiveFrameRate ?? this.configuredFrameRate ?? getConfiguredQuality().frameRate;
|
||||
const frameRateLimited = isOutboundFrameRateBelowTarget(stats, previousSample, targetFrameRate);
|
||||
const qualityLimited = isLimitedQualityReason(stats.qualityLimitationReason);
|
||||
this.update(() => {
|
||||
this.limitationReason = stats.qualityLimitationReason;
|
||||
});
|
||||
if (qualityLimited || frameRateLimited) {
|
||||
this.stepDownStreak++;
|
||||
this.stepUpStreak = 0;
|
||||
} else if (stats.qualityLimitationReason === 'none') {
|
||||
this.stepDownStreak = 0;
|
||||
if (this.isAdapted) {
|
||||
this.stepUpStreak++;
|
||||
}
|
||||
} else {
|
||||
this.stepDownStreak = 0;
|
||||
this.stepUpStreak = 0;
|
||||
}
|
||||
const now = Date.now();
|
||||
if (this.stepDownStreak >= STEP_DOWN_THRESHOLD && now - this.lastStepDownAt >= STEP_DOWN_COOLDOWN_MS) {
|
||||
await this.stepDown(screenShare, stats);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
this.stepUpStreak >= STEP_UP_THRESHOLD &&
|
||||
now - this.lastStepUpAt >= this.stepUpCooldownMs &&
|
||||
now - this.lastStepDownAt >= STEP_DOWN_COOLDOWN_MS
|
||||
) {
|
||||
await this.stepUp(screenShare);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug('Failed to poll adaptive screen share quality stats', {error});
|
||||
} finally {
|
||||
if (this.pollInFlightGeneration === generation) {
|
||||
this.pollInFlightGeneration = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async stepDown(screenShare: ScreenShareSender, stats: OutboundVideoAdaptationStats): Promise<void> {
|
||||
const reason = stats.qualityLimitationReason;
|
||||
const currentResolution =
|
||||
this.effectiveResolution ?? this.configuredResolution ?? getConfiguredQuality().resolution;
|
||||
const currentFrameRate = this.effectiveFrameRate ?? this.configuredFrameRate ?? getConfiguredQuality().frameRate;
|
||||
const streamingMode = VoiceSettings.getStreamingMode();
|
||||
const nextFrameRate = shouldPreferFrameRateStepDown(streamingMode, reason)
|
||||
? getLowerFrameRate(currentFrameRate)
|
||||
: null;
|
||||
const nextResolution = nextFrameRate ? currentResolution : getLowerResolution(currentResolution);
|
||||
if (!nextResolution) {
|
||||
this.stepDownStreak = 0;
|
||||
return;
|
||||
}
|
||||
const nextEffectiveFrameRate = nextFrameRate ?? currentFrameRate;
|
||||
const nextMaxBitrateBps = getScreenShareBitrateBps(nextResolution, nextEffectiveFrameRate);
|
||||
const stepKind = nextFrameRate ? 'framerate' : 'resolution';
|
||||
try {
|
||||
await applyResolutionFrameRateAndBitrate(
|
||||
screenShare.track,
|
||||
screenShare.sender,
|
||||
nextResolution,
|
||||
nextEffectiveFrameRate,
|
||||
);
|
||||
this.update(() => {
|
||||
this.effectiveResolution = nextResolution;
|
||||
this.effectiveFrameRate = nextEffectiveFrameRate;
|
||||
this.isAdapted = true;
|
||||
this.stepDownStreak = 0;
|
||||
this.stepUpStreak = 0;
|
||||
this.lastStepDownAt = Date.now();
|
||||
this.stepUpCooldownMs = Math.min(this.stepUpCooldownMs * 2, MAX_STEP_UP_COOLDOWN_MS);
|
||||
});
|
||||
logger.info('Reduced screen share quality adaptively', {
|
||||
fromResolution: currentResolution,
|
||||
toResolution: nextResolution,
|
||||
fromFrameRate: currentFrameRate,
|
||||
toFrameRate: nextEffectiveFrameRate,
|
||||
maxBitrateBps: nextMaxBitrateBps,
|
||||
stepKind,
|
||||
reason,
|
||||
});
|
||||
const reasonLabel = formatLimitationReasonForToast(reason);
|
||||
if (stepKind === 'framerate') {
|
||||
showAdaptiveScreenShareToast(
|
||||
i18n._(LOWERED_SCREEN_SHARE_FRAME_RATE_DESCRIPTOR, {
|
||||
frameRate: nextEffectiveFrameRate,
|
||||
reason: reasonLabel,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
showAdaptiveScreenShareToast(
|
||||
i18n._(LOWERED_SCREEN_SHARE_RESOLUTION_DESCRIPTOR, {
|
||||
resolution: formatResolutionForToast(nextResolution),
|
||||
reason: reasonLabel,
|
||||
}),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.stepDownStreak = 0;
|
||||
logger.warn('Failed to reduce screen share quality adaptively', {
|
||||
error,
|
||||
fromResolution: currentResolution,
|
||||
toResolution: nextResolution,
|
||||
fromFrameRate: currentFrameRate,
|
||||
toFrameRate: nextEffectiveFrameRate,
|
||||
maxBitrateBps: nextMaxBitrateBps,
|
||||
stepKind,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async stepUp(screenShare: ScreenShareSender): Promise<void> {
|
||||
const configured = getConfiguredQuality();
|
||||
const configuredResolution = this.configuredResolution ?? configured.resolution;
|
||||
const configuredFrameRate = this.configuredFrameRate ?? configured.frameRate;
|
||||
const currentResolution = this.effectiveResolution ?? configuredResolution;
|
||||
const nextResolution = getHigherResolution(currentResolution, configuredResolution);
|
||||
if (!nextResolution) {
|
||||
this.update(() => {
|
||||
this.effectiveResolution = configuredResolution;
|
||||
this.effectiveFrameRate = configuredFrameRate;
|
||||
this.isAdapted = false;
|
||||
this.stepUpStreak = 0;
|
||||
this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await applyResolutionFrameRateAndBitrate(
|
||||
screenShare.track,
|
||||
screenShare.sender,
|
||||
nextResolution,
|
||||
configuredFrameRate,
|
||||
);
|
||||
this.update(() => {
|
||||
this.effectiveResolution = nextResolution;
|
||||
this.effectiveFrameRate = configuredFrameRate;
|
||||
this.isAdapted = nextResolution !== configuredResolution;
|
||||
this.stepUpStreak = 0;
|
||||
this.lastStepUpAt = Date.now();
|
||||
if (!this.isAdapted) {
|
||||
this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
|
||||
}
|
||||
});
|
||||
logger.info('Raised screen share quality adaptively', {
|
||||
fromResolution: currentResolution,
|
||||
toResolution: nextResolution,
|
||||
frameRate: configuredFrameRate,
|
||||
});
|
||||
showAdaptiveScreenShareToast(
|
||||
i18n._(RAISED_SCREEN_SHARE_RESOLUTION_DESCRIPTOR, {
|
||||
resolution: formatResolutionForToast(nextResolution),
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
this.stepUpStreak = 0;
|
||||
logger.warn('Failed to raise screen share quality adaptively', {
|
||||
error,
|
||||
fromResolution: currentResolution,
|
||||
toResolution: nextResolution,
|
||||
frameRate: configuredFrameRate,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new AdaptiveScreenShareEngine();
|
||||
@@ -26,7 +26,6 @@ import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
|
||||
import Users from '@app/features/user/state/Users';
|
||||
import {getStreamKey, parseStreamKey} from '@app/features/voice/components/StreamKeys';
|
||||
import {voiceStatsDB} from '@app/features/voice/diagnostics/VoiceStatsDB';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import {
|
||||
createMediaEngineFacadeSnapshot,
|
||||
type MediaEngineFacadeEvent,
|
||||
@@ -194,7 +193,6 @@ import {
|
||||
type Room,
|
||||
RoomEvent,
|
||||
type ScreenShareCaptureOptions,
|
||||
Track,
|
||||
type TrackPublishOptions,
|
||||
} from 'livekit-client';
|
||||
import {makeObservable, observable} from 'mobx';
|
||||
@@ -506,7 +504,6 @@ class MediaEngineFacade extends Store {
|
||||
voiceEngineV2AppConnectionHostAdapter.subscribe(forwardChange);
|
||||
voiceEngineV2AppVoiceStateAdapter.subscribe(forwardChange);
|
||||
this.statsHostAdapter.subscribe(forwardChange);
|
||||
AdaptiveScreenShareEngine.subscribe(forwardChange);
|
||||
voiceEngineV2AppMediaExecutionAdapter.subscribe(forwardChange);
|
||||
voiceEngineV2AppScreenShareExecutionAdapter.subscribe(forwardChange);
|
||||
VoiceEngineV2AppSubscriptionAdapter.subscribe(forwardChange);
|
||||
@@ -1605,20 +1602,14 @@ class MediaEngineFacade extends Store {
|
||||
await this.reconcileLocalAudioState('voice room connected');
|
||||
},
|
||||
onDisconnected: () => {
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
this.stopTracking();
|
||||
this.statsHostAdapter.reset();
|
||||
},
|
||||
onReconnecting: () => {
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
this.statsHostAdapter.stopLatencyTracking();
|
||||
this.statsHostAdapter.stopStatsTracking();
|
||||
},
|
||||
onReconnected: () => {
|
||||
const room = voiceEngineV2AppConnectionHostAdapter.room;
|
||||
if (room?.localParticipant?.getTrackPublication(Track.Source.ScreenShare)?.videoTrack) {
|
||||
AdaptiveScreenShareEngine.start(room);
|
||||
}
|
||||
this.statsHostAdapter.incrementReconnectionCount();
|
||||
this.statsHostAdapter.startLatencyTracking();
|
||||
this.statsHostAdapter.startStatsTracking();
|
||||
@@ -1669,11 +1660,6 @@ class MediaEngineFacade extends Store {
|
||||
VoiceEngineV2AppSubscriptionAdapter.reconcileSubscriptions();
|
||||
VoiceEngineV2AppPermissionAdapter.applyDeafen(newRoom, getEffectiveAudioState().effectiveDeaf);
|
||||
voiceEngineV2AppMediaExecutionAdapter.applyAllLocalAudioPreferences(newRoom);
|
||||
if (newRoom.localParticipant?.getTrackPublication(Track.Source.ScreenShare)?.videoTrack) {
|
||||
AdaptiveScreenShareEngine.start(newRoom);
|
||||
} else {
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
}
|
||||
if (guildId && channelId) {
|
||||
VoiceEngineV2AppPermissionAdapter.syncWithPermissionState(guildId, channelId, newRoom);
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import type {
|
||||
VoiceEngineV2Controller,
|
||||
VoiceEngineV2Model,
|
||||
VoiceEngineV2Participant,
|
||||
VoiceEngineV2ScreenEncodingOptions,
|
||||
VoiceEngineV2Snapshot,
|
||||
VoiceEngineV2WatchedStream,
|
||||
} from '@fluxer/voice_engine_v2';
|
||||
@@ -128,36 +127,6 @@ export function getVoiceEngineV2SnapshotFromMediaEngine(): VoiceEngineV2Snapshot
|
||||
return getMediaEngine()?.voiceEngineV2Snapshot ?? null;
|
||||
}
|
||||
|
||||
export interface VoiceEngineV2ScreenEncodingUpdateFromMediaEngine {
|
||||
captureId?: string;
|
||||
width: number;
|
||||
height: number;
|
||||
frameRate?: number;
|
||||
maxBitrateBps?: number;
|
||||
}
|
||||
|
||||
function getVoiceEngineV2ActiveScreenCaptureId(controller: VoiceEngineV2Controller): string | null {
|
||||
return controller.snapshot.screen.desired?.captureId ?? controller.snapshot.screen.published?.captureId ?? null;
|
||||
}
|
||||
|
||||
export function updateVoiceEngineV2ScreenEncodingFromMediaEngine(
|
||||
options: VoiceEngineV2ScreenEncodingUpdateFromMediaEngine,
|
||||
): boolean {
|
||||
const controller = getMediaEngine()?.voiceEngineV2Controller;
|
||||
if (!controller) return false;
|
||||
const captureId = options.captureId ?? getVoiceEngineV2ActiveScreenCaptureId(controller);
|
||||
if (!captureId) return false;
|
||||
const update: VoiceEngineV2ScreenEncodingOptions = {
|
||||
captureId,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
frameRate: options.frameRate,
|
||||
maxBitrateBps: options.maxBitrateBps,
|
||||
};
|
||||
controller.updateScreenEncoding(update);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function setVoiceEngineV2ParticipantAudioLevelSpeaking(
|
||||
identity: string,
|
||||
speaking: boolean,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {Platform} from '@app/features/platform/types/Platform';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import type {NegotiationReason} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
|
||||
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
|
||||
@@ -194,7 +193,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration {
|
||||
this.adapter.ensureScreenShareKeepAliveSinkInternal(participant, track);
|
||||
this.adapter.monitorActiveScreenShareEndInternal(room, participant, track);
|
||||
this.adapter.startEncoderVerificationInternal(room, participant, publishOptions?.videoCodec, track);
|
||||
AdaptiveScreenShareEngine.start(room);
|
||||
this.adapter.applyScreenShareStateInternal(true, {reason: 'server', sendUpdate: false});
|
||||
updateLocalParticipantFromRoom(room);
|
||||
this.adapter.syncLocalStreamWatchStateInternal(true);
|
||||
@@ -338,7 +336,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration {
|
||||
typeof ctx.mediaStreamTrack.clone === 'function' ? ctx.mediaStreamTrack.clone() : ctx.mediaStreamTrack;
|
||||
this.adapter.applyScreenShareContentHintToMediaTrackInternal(state.candidateMediaStreamTrack);
|
||||
this.adapter.cancelEncoderVerificationInternal();
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
await this.announceBreakAndUnpublish(ctx, state);
|
||||
state.candidatePublication = await ctx.participant.publishTrack(
|
||||
state.candidateMediaStreamTrack,
|
||||
@@ -410,7 +407,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration {
|
||||
this.adapter.syncLocalScreenShareAudioStateInternal(ctx.participant, actual);
|
||||
if (actual) {
|
||||
this.adapter.monitorActiveScreenShareEndInternal(ctx.room, ctx.participant);
|
||||
AdaptiveScreenShareEngine.start(ctx.room);
|
||||
await this.adapter.enforceScreenShareSenderParametersInternal(ctx.participant, ctx.previousOptions);
|
||||
this.adapter.ensureScreenShareKeepAliveSinkInternal(ctx.participant);
|
||||
} else {
|
||||
|
||||
-3
@@ -9,7 +9,6 @@ import * as SoundCommands from '@app/features/ui/commands/SoundCommands';
|
||||
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
|
||||
import * as VoiceSettingsCommands from '@app/features/voice/commands/VoiceSettingsCommands';
|
||||
import {getStreamKey} from '@app/features/voice/components/StreamKeys';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import type {NegotiationReason} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import {Store} from '@app/features/voice/engine/Store';
|
||||
@@ -597,7 +596,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
handleLocalScreenShareTrackUnpublished(room: Room, playSound: boolean, publication?: LocalTrackPublication): void {
|
||||
this.clearScreenShareKeepAliveSinkInternal();
|
||||
this.cleanupActiveScreenShareEndListenerInternal();
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
const participant = room.localParticipant;
|
||||
const cleanupSnapshot = captureScreenSharePublicationCleanup(
|
||||
publication,
|
||||
@@ -848,7 +846,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
this.transitionScreenShareLifecycleInternal({type: 'share.reset'});
|
||||
this.cleanupActiveScreenShareEndListenerInternal();
|
||||
this.cancelEncoderVerificationInternal();
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
this.endedScreenShareStopInFlight = null;
|
||||
this.cleanupScreenShareAudioRoutingState();
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {isDesktop, isNativeMacOS} from '@app/features/ui/utils/NativeUtils';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
|
||||
import {
|
||||
enforceLocalMediaPublicationCap,
|
||||
@@ -180,7 +179,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
this.adapter.setStreamingPriorityInternal(false);
|
||||
this.adapter.cleanupActiveScreenShareEndListenerInternal();
|
||||
this.adapter.cancelEncoderVerificationInternal();
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
}
|
||||
SoftwareEncoderWarning.reset();
|
||||
const stopCleanupSnapshot = enabled ? null : this.adapter.getScreenShareCaptureCleanupSnapshotInternal(participant);
|
||||
@@ -837,7 +835,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
applyScreenShareState(this.adapter, false, true, true);
|
||||
this.adapter.syncLocalStreamWatchStateInternal(false);
|
||||
this.adapter.syncLocalScreenShareAudioStateInternal(participant, false);
|
||||
AdaptiveScreenShareEngine.stop();
|
||||
if (cleanupError !== undefined) {
|
||||
throw new ScreenShareRollbackIncompleteError([cleanupError]);
|
||||
}
|
||||
@@ -1084,7 +1081,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
|
||||
run: () =>
|
||||
this.adapter.startEncoderVerificationInternal(room, participant, effectivePublishOptions?.videoCodec),
|
||||
},
|
||||
{name: 'adaptive engine', run: () => AdaptiveScreenShareEngine.start(room)},
|
||||
{name: 'local stream state', run: () => applyScreenShareState(this.adapter, true, true, true)},
|
||||
{name: 'participant snapshot', run: () => updateLocalParticipantFromRoom(room)},
|
||||
{name: 'watch state', run: () => this.adapter.syncLocalStreamWatchStateInternal(true)},
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {SoundType} from '@app/features/notification/utils/SoundUtils';
|
||||
import * as SoundCommands from '@app/features/ui/commands/SoundCommands';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
|
||||
import type {
|
||||
VoiceScreenShareEvent,
|
||||
@@ -81,7 +80,6 @@ async function runScreenSharePublishPipeline(
|
||||
adapter.applyScreenShareAudioContentHintInternal(participant);
|
||||
adapter.monitorActiveScreenShareEndInternal(room, participant);
|
||||
adapter.startEncoderVerificationInternal(room, participant, pipeline.effectivePublishOptions?.videoCodec);
|
||||
AdaptiveScreenShareEngine.start(room);
|
||||
}
|
||||
|
||||
export async function runScreenShareActivationRitual(args: ScreenShareActivationRitualArgs): Promise<void> {
|
||||
@@ -158,11 +156,8 @@ export function settleScreenShareFailure(args: ScreenShareFailureSettleArgs): bo
|
||||
if (!actual) {
|
||||
args.onInactiveAfterSync?.();
|
||||
}
|
||||
if (actual) {
|
||||
if (args.monitorEndOnActive) {
|
||||
adapter.monitorActiveScreenShareEndInternal(room, participant);
|
||||
}
|
||||
AdaptiveScreenShareEngine.start(room);
|
||||
if (actual && args.monitorEndOnActive) {
|
||||
adapter.monitorActiveScreenShareEndInternal(room, participant);
|
||||
}
|
||||
if (args.playSound) {
|
||||
SoundCommands.playSound(actual ? SoundType.ScreenShareStart : SoundType.ScreenShareStop);
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
|
||||
export const logger = new Logger('VoiceEngineV2ScreenShareSupport');
|
||||
const GPU_ENCODER_REPORT_START_TIMEOUT_MS = 500;
|
||||
const SCREEN_SHARE_FULL_RESOLUTION_SCALE = 1;
|
||||
|
||||
export interface DeviceScreenShareCaptureOptions {
|
||||
videoDeviceId?: string;
|
||||
@@ -349,6 +350,14 @@ export function getPublishedScreenShareMaxBitrateBps(
|
||||
return total > 0 ? total : undefined;
|
||||
}
|
||||
|
||||
function resolveScreenShareScaleResolutionDownBy(
|
||||
encodings: ReadonlyArray<RTCRtpEncodingParameters>,
|
||||
): number | undefined {
|
||||
const simulcastLadderOwnsScale = encodings.length > 1;
|
||||
if (simulcastLadderOwnsScale) return undefined;
|
||||
return SCREEN_SHARE_FULL_RESOLUTION_SCALE;
|
||||
}
|
||||
|
||||
export async function enforceScreenShareSenderParameters(
|
||||
sender: RTCRtpSender | undefined,
|
||||
publishOptions?: TrackPublishOptions,
|
||||
@@ -370,11 +379,13 @@ export async function enforceScreenShareSenderParameters(
|
||||
screenShareEncoding?.maxBitrate !== undefined
|
||||
? distributeScreenShareBitrate(encodings, screenShareEncoding.maxBitrate)
|
||||
: undefined;
|
||||
const scaleResolutionDownBy = resolveScreenShareScaleResolutionDownBy(encodings);
|
||||
params.degradationPreference = SCREEN_SHARE_DEGRADATION_PREFERENCE;
|
||||
params.encodings = encodings.map((encoding, index) => ({
|
||||
...encoding,
|
||||
...(bitrates ? {maxBitrate: bitrates[index]} : {}),
|
||||
...(screenShareEncoding?.maxFramerate !== undefined ? {maxFramerate: screenShareEncoding.maxFramerate} : {}),
|
||||
...(scaleResolutionDownBy !== undefined ? {scaleResolutionDownBy} : {}),
|
||||
priority: screenShareEncoding?.priority ?? encoding.priority ?? 'high',
|
||||
networkPriority: screenShareEncoding?.priority ?? encoding.networkPriority ?? 'high',
|
||||
...(scalabilityMode ? {scalabilityMode} : {}),
|
||||
|
||||
@@ -109,7 +109,6 @@ type VoiceSettingsUpdate = Partial<{
|
||||
screenShareSoftwareQuality: ScreenShareSoftwareQuality;
|
||||
screenShareScalabilityMode: ScreenShareScalabilityModePreference;
|
||||
screenShareBackupCodecMode: ScreenShareBackupCodecMode;
|
||||
adaptiveScreenShareQuality: boolean;
|
||||
vadThreshold: number;
|
||||
vadAutoSensitivity: boolean;
|
||||
vadEnhanced: boolean;
|
||||
@@ -194,15 +193,13 @@ function applyPiPPopoutDefaultsMigration(parsed: Record<string, unknown>): boole
|
||||
return true;
|
||||
}
|
||||
|
||||
function applyAdaptiveScreenShareQualityMigrationV2(parsed: Record<string, unknown>): boolean {
|
||||
function applyAdaptiveScreenShareQualityRemovalMigration(parsed: Record<string, unknown>): boolean {
|
||||
let changed = false;
|
||||
if (Object.hasOwn(parsed, 'adaptiveScreenShareQuality')) {
|
||||
delete parsed.adaptiveScreenShareQuality;
|
||||
changed = true;
|
||||
}
|
||||
if (typeof parsed.adaptiveScreenShareQualityPrefV2 !== 'boolean') {
|
||||
parsed.adaptiveScreenShareQualityPrefV2 = false;
|
||||
changed = true;
|
||||
for (const key of ['adaptiveScreenShareQuality', 'adaptiveScreenShareQualityPrefV2']) {
|
||||
if (Object.hasOwn(parsed, key)) {
|
||||
delete parsed[key];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -403,7 +400,6 @@ class VoiceSettings {
|
||||
screenShareSoftwareQualityPrefV2: ScreenShareSoftwareQuality = DEFAULT_SCREEN_SHARE_SOFTWARE_QUALITY;
|
||||
screenShareScalabilityModePrefV2: ScreenShareScalabilityModePreference = DEFAULT_SCREEN_SHARE_SCALABILITY_MODE;
|
||||
screenShareBackupCodecModePrefV2: ScreenShareBackupCodecMode = DEFAULT_SCREEN_SHARE_BACKUP_CODEC_MODE;
|
||||
adaptiveScreenShareQualityPrefV2 = false;
|
||||
vadThreshold = 50;
|
||||
vadAutoSensitivity = true;
|
||||
vadEnhanced = true;
|
||||
@@ -478,7 +474,6 @@ class VoiceSettings {
|
||||
getScreenShareScalabilityModeOverride: false,
|
||||
getScreenShareBackupCodecMode: false,
|
||||
getScreenShareBackupCodecModeOverride: false,
|
||||
getAdaptiveScreenShareQuality: false,
|
||||
getVadThreshold: false,
|
||||
getVadAutoSensitivity: false,
|
||||
getVadEnhanced: false,
|
||||
@@ -512,7 +507,7 @@ class VoiceSettings {
|
||||
changed = applyLegacyVenmicKeyMigration(parsed);
|
||||
}
|
||||
changed = applyPiPPopoutDefaultsMigration(parsed) || changed;
|
||||
changed = applyAdaptiveScreenShareQualityMigrationV2(parsed) || changed;
|
||||
changed = applyAdaptiveScreenShareQualityRemovalMigration(parsed) || changed;
|
||||
changed = applyScreenShareAudioConsentMigrationV1(parsed) || changed;
|
||||
changed = applyScreenShareAudioDefaultOnMigrationV1(parsed) || changed;
|
||||
changed = applyStreamingModeDefaultMigrationV1(parsed) || changed;
|
||||
@@ -589,7 +584,6 @@ class VoiceSettings {
|
||||
'screenShareSoftwareQualityPrefV2',
|
||||
'screenShareScalabilityModePrefV2',
|
||||
'screenShareBackupCodecModePrefV2',
|
||||
'adaptiveScreenShareQualityPrefV2',
|
||||
'vadThreshold',
|
||||
'vadAutoSensitivity',
|
||||
'vadEnhanced',
|
||||
@@ -682,14 +676,6 @@ class VoiceSettings {
|
||||
this.screenShareBackupCodecModePrefV2 = value;
|
||||
}
|
||||
|
||||
get adaptiveScreenShareQuality(): boolean {
|
||||
return this.adaptiveScreenShareQualityPrefV2;
|
||||
}
|
||||
|
||||
set adaptiveScreenShareQuality(value: boolean) {
|
||||
this.adaptiveScreenShareQualityPrefV2 = value;
|
||||
}
|
||||
|
||||
getInputDeviceId(): string {
|
||||
return this.inputDeviceId;
|
||||
}
|
||||
@@ -945,10 +931,6 @@ class VoiceSettings {
|
||||
return mode === 'off' ? undefined : mode;
|
||||
}
|
||||
|
||||
getAdaptiveScreenShareQuality(): boolean {
|
||||
return this.adaptiveScreenShareQuality;
|
||||
}
|
||||
|
||||
getVadThreshold(): number {
|
||||
return this.vadThreshold;
|
||||
}
|
||||
@@ -1088,8 +1070,6 @@ class VoiceSettings {
|
||||
this.screenShareScalabilityMode = validated.screenShareScalabilityMode;
|
||||
if (validated.screenShareBackupCodecMode !== undefined)
|
||||
this.screenShareBackupCodecMode = validated.screenShareBackupCodecMode;
|
||||
if (validated.adaptiveScreenShareQuality !== undefined)
|
||||
this.adaptiveScreenShareQuality = validated.adaptiveScreenShareQuality;
|
||||
if (validated.vadThreshold !== undefined) this.vadThreshold = validated.vadThreshold;
|
||||
if (validated.vadAutoSensitivity !== undefined) this.vadAutoSensitivity = validated.vadAutoSensitivity;
|
||||
if (validated.vadEnhanced !== undefined) this.vadEnhanced = validated.vadEnhanced;
|
||||
@@ -1227,7 +1207,6 @@ class VoiceSettings {
|
||||
screenShareSoftwareQuality,
|
||||
screenShareScalabilityMode,
|
||||
screenShareBackupCodecMode,
|
||||
adaptiveScreenShareQuality: data.adaptiveScreenShareQuality ?? this.adaptiveScreenShareQuality,
|
||||
vadThreshold: Math.max(0, Math.min(100, data.vadThreshold ?? this.vadThreshold)),
|
||||
vadAutoSensitivity: data.vadAutoSensitivity ?? this.vadAutoSensitivity,
|
||||
vadEnhanced: data.vadEnhanced ?? this.vadEnhanced,
|
||||
|
||||
@@ -50,7 +50,7 @@ export function resolveScreenShareFrameRate(frameRate: number): SupportedScreenS
|
||||
return 15;
|
||||
}
|
||||
|
||||
export function getScreenShareDimensions(resolution: ScreenshareResolution): {
|
||||
function getScreenShareDimensions(resolution: ScreenshareResolution): {
|
||||
width: number;
|
||||
height: number;
|
||||
} {
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
isDesktop,
|
||||
supportsDesktopScreenShareAudioCapture,
|
||||
} from '@app/features/ui/utils/NativeUtils';
|
||||
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
|
||||
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
|
||||
import ScreenShareCodecNegotiation, {
|
||||
getScreenShareCodecPreferenceOrder,
|
||||
@@ -295,7 +294,6 @@ async function collectVoiceSettingsMetadata(): Promise<Record<string, unknown>>
|
||||
screenShareBackupCodecMode: VoiceSettings.getScreenShareBackupCodecMode(),
|
||||
screenShareMaxBitrateMbps:
|
||||
getScreenShareBitrateBps(configuredScreenShare.resolution, configuredScreenShare.frameRate) / 1000000,
|
||||
adaptiveScreenShareQuality: VoiceSettings.getAdaptiveScreenShareQuality(),
|
||||
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
|
||||
linuxAudioCapture: {
|
||||
workaround: VoiceSettings.getLinuxAudioCaptureWorkaround(),
|
||||
@@ -461,7 +459,6 @@ async function collectDesktopMetadata(): Promise<Record<string, unknown>> {
|
||||
}
|
||||
|
||||
export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
|
||||
const adaptiveQualitySnapshot = AdaptiveScreenShareEngine.qualitySnapshot;
|
||||
const effectiveScreenShareSettings = resolveStreamingModeSettings(
|
||||
VoiceSettings.getStreamingMode(),
|
||||
VoiceSettings.getScreenshareResolution(),
|
||||
@@ -513,13 +510,6 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
|
||||
(getPublishedScreenShareMaxBitrateBps(localParticipant) ??
|
||||
getScreenShareBitrateBps(effectiveScreenShareSettings.resolution, effectiveScreenShareSettings.frameRate)) /
|
||||
1000000,
|
||||
adaptiveQuality: VoiceSettings.getAdaptiveScreenShareQuality(),
|
||||
adaptiveQualityAdapted: adaptiveQualitySnapshot.isAdapted,
|
||||
adaptiveQualityConfiguredResolution: adaptiveQualitySnapshot.configuredResolution,
|
||||
adaptiveQualityConfiguredFrameRate: adaptiveQualitySnapshot.configuredFrameRate,
|
||||
adaptiveQualityEffectiveResolution: adaptiveQualitySnapshot.effectiveResolution,
|
||||
adaptiveQualityEffectiveFrameRate: adaptiveQualitySnapshot.effectiveFrameRate,
|
||||
adaptiveQualityLimitationReason: adaptiveQualitySnapshot.limitationReason,
|
||||
audioSourceMode: VoiceSettings.getScreenShareAudioSourceMode(),
|
||||
audioIncludeSources: VoiceSettings.getScreenShareAudioIncludeSources(),
|
||||
audioExcludeSources: VoiceSettings.getScreenShareAudioExcludeSources(),
|
||||
|
||||
@@ -76,13 +76,6 @@ export interface StatsForNerdsData {
|
||||
scalabilityMode: string;
|
||||
backupCodecMode: string;
|
||||
maxBitrateMbps: number;
|
||||
adaptiveQuality: boolean;
|
||||
adaptiveQualityAdapted: boolean;
|
||||
adaptiveQualityConfiguredResolution: string;
|
||||
adaptiveQualityConfiguredFrameRate: number;
|
||||
adaptiveQualityEffectiveResolution: string;
|
||||
adaptiveQualityEffectiveFrameRate: number;
|
||||
adaptiveQualityLimitationReason: string;
|
||||
audioSourceMode: string;
|
||||
audioIncludeSources: Array<Record<string, string>>;
|
||||
audioExcludeSources: Array<Record<string, string>>;
|
||||
|
||||
@@ -58,10 +58,6 @@ export function markNativeScreenShareTrack(track: MediaStreamTrack): void {
|
||||
}
|
||||
}
|
||||
|
||||
export function isNativeScreenShareTrack(track: MediaStreamTrack | undefined): boolean {
|
||||
return Boolean(track && (track as MediaStreamTrack & Record<PropertyKey, unknown>)[NATIVE_SCREEN_SHARE_TRACK_MARKER]);
|
||||
}
|
||||
|
||||
export function getVideoFrameCtor(): VideoFrameCtor | undefined {
|
||||
return (
|
||||
window as typeof window & {
|
||||
|
||||
Reference in new issue
Block a user