mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
793 lines
28 KiB
TypeScript
793 lines
28 KiB
TypeScript
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
import type {Channel} from '@app/features/channel/models/Channel';
|
|
import Users from '@app/features/user/state/Users';
|
|
import * as VoiceCallLayoutCommands from '@app/features/voice/commands/VoiceCallLayoutCommands';
|
|
import {parseStreamKey} from '@app/features/voice/components/StreamKeys';
|
|
import {usePinnedTrackRef} from '@app/features/voice/components/usePinnedTrackRef';
|
|
import {
|
|
dedupeScreenShareTracks,
|
|
splitVoiceCallRenderableTracks,
|
|
} from '@app/features/voice/components/VoiceCallTrackPartition';
|
|
import {
|
|
consolidateVoiceGridTracks,
|
|
type VoiceGridEntry,
|
|
} from '@app/features/voice/components/VoiceParticipantConsolidation';
|
|
import {
|
|
countKnownVoiceParticipants,
|
|
isKnownVoiceParticipantIdentity,
|
|
isKnownVoiceTrackRef,
|
|
} from '@app/features/voice/components/VoiceParticipantRenderUtils';
|
|
import {
|
|
compareVoiceTrackReferencesWithSnapshot,
|
|
createVoiceParticipantSortSnapshot,
|
|
syncVoiceParticipantSortSnapshot,
|
|
} from '@app/features/voice/components/VoiceParticipantSortUtils';
|
|
import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade';
|
|
import NativeVideoTileManager, {
|
|
type NativeInboundVideoTrack,
|
|
} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager';
|
|
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
|
|
import type {VoiceGatewayConnectionVoiceStates} from '@app/features/voice/engine/VoiceGatewayStateMachine';
|
|
import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge';
|
|
import {selectVoiceMediaGraphViewerStreamKeys} from '@app/features/voice/engine/VoiceMediaGraph';
|
|
import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
|
|
import {pruneInactiveWatchedStreamsForChannel} from '@app/features/voice/engine/VoiceStreamWatchState';
|
|
import {
|
|
asVoiceTrackSource,
|
|
isVoiceScreenShareSource,
|
|
VoiceTrackSource,
|
|
} from '@app/features/voice/engine/VoiceTrackSource';
|
|
import {
|
|
isVoiceEngineV2AppParticipantSpeaking,
|
|
type VoiceEngineV2AppParticipantSnapshot,
|
|
} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors';
|
|
import CallMediaPrefs from '@app/features/voice/state/CallMediaPrefs';
|
|
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
|
|
import VoiceCallLayout from '@app/features/voice/state/VoiceCallLayout';
|
|
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
|
import {
|
|
buildVoiceParticipantIdentity,
|
|
parseVoiceParticipantIdentity,
|
|
} from '@app/features/voice/utils/VoiceParticipantIdentity';
|
|
import {isTrackReference, type TrackReferenceOrPlaceholder, useMaybeRoomContext} from '@livekit/components-react';
|
|
import {Participant, type Room, RoomEvent, Track, TrackPublication, type VideoQuality} from 'livekit-client';
|
|
import {useCallback, useEffect, useMemo, useRef, useState} from 'react';
|
|
|
|
interface UseVoiceCallTracksAndLayoutArgs {
|
|
channel: Channel;
|
|
expandedUserIds: ReadonlySet<string>;
|
|
}
|
|
|
|
interface VoiceGridTrackActivity {
|
|
priority: number;
|
|
promotedOrder: number;
|
|
}
|
|
|
|
interface VoiceGridTrackActivityState {
|
|
lastActivityAt: number;
|
|
promotedOrder: number;
|
|
}
|
|
|
|
const VOICE_GRID_SPEAKER_PRIORITY_HOLD_MS = 4500;
|
|
const CAMERA_SOURCE = VoiceTrackSource.Camera as Track.Source;
|
|
const SCREEN_SHARE_SOURCE = VoiceTrackSource.ScreenShare as Track.Source;
|
|
const SCREEN_SHARE_AUDIO_SOURCE = VoiceTrackSource.ScreenShareAudio as Track.Source;
|
|
|
|
interface VoiceCallTrackSourceOption {
|
|
source: Track.Source;
|
|
withPlaceholder: boolean;
|
|
}
|
|
|
|
interface VoiceCallTrackRefsState {
|
|
tracks: Array<TrackReferenceOrPlaceholder>;
|
|
participants: Array<Participant>;
|
|
}
|
|
|
|
const VOICE_CALL_TRACK_SOURCES: ReadonlyArray<VoiceCallTrackSourceOption> = [
|
|
{source: CAMERA_SOURCE, withPlaceholder: true},
|
|
{source: SCREEN_SHARE_SOURCE, withPlaceholder: false},
|
|
];
|
|
|
|
const LIVEKIT_TRACK_UPDATE_EVENTS: ReadonlyArray<RoomEvent> = [
|
|
RoomEvent.ParticipantConnected,
|
|
RoomEvent.ParticipantDisconnected,
|
|
RoomEvent.ConnectionStateChanged,
|
|
RoomEvent.LocalTrackPublished,
|
|
RoomEvent.LocalTrackUnpublished,
|
|
RoomEvent.TrackPublished,
|
|
RoomEvent.TrackUnpublished,
|
|
RoomEvent.TrackSubscriptionStatusChanged,
|
|
RoomEvent.TrackMuted,
|
|
RoomEvent.TrackUnmuted,
|
|
RoomEvent.TrackSubscribed,
|
|
RoomEvent.TrackUnsubscribed,
|
|
RoomEvent.ActiveSpeakersChanged,
|
|
];
|
|
|
|
class NativeVoiceTrackPublication extends TrackPublication {
|
|
private subscribed: boolean;
|
|
private desired: boolean;
|
|
private enabled: boolean;
|
|
private readonly local: boolean;
|
|
private requestedVideoQuality: VideoQuality | undefined;
|
|
|
|
constructor({
|
|
trackSid,
|
|
source,
|
|
kind,
|
|
isMuted,
|
|
isLocal,
|
|
dimensions,
|
|
}: {
|
|
trackSid: string;
|
|
source: Track.Source;
|
|
kind: Track.Kind;
|
|
isMuted: boolean;
|
|
isLocal: boolean;
|
|
dimensions?: Track.Dimensions;
|
|
}) {
|
|
super(kind, trackSid, trackSid);
|
|
this.source = source;
|
|
this.metadataMuted = isMuted;
|
|
this.subscribed = !isMuted;
|
|
this.desired = !isMuted;
|
|
this.enabled = !isMuted;
|
|
this.local = isLocal;
|
|
this.dimensions = dimensions;
|
|
}
|
|
|
|
override get isLocal(): boolean {
|
|
return this.local;
|
|
}
|
|
|
|
override get isSubscribed(): boolean {
|
|
return this.subscribed;
|
|
}
|
|
|
|
get isDesired(): boolean {
|
|
return this.desired;
|
|
}
|
|
|
|
override get isEnabled(): boolean {
|
|
return this.enabled;
|
|
}
|
|
|
|
setSubscribed(subscribed: boolean): void {
|
|
this.desired = subscribed;
|
|
this.subscribed = subscribed;
|
|
}
|
|
|
|
setEnabled(enabled: boolean): void {
|
|
this.enabled = enabled;
|
|
}
|
|
|
|
setVideoQuality(quality: VideoQuality): void {
|
|
this.requestedVideoQuality = quality;
|
|
}
|
|
|
|
emitTrackUpdate(): void {
|
|
void this.requestedVideoQuality;
|
|
}
|
|
}
|
|
|
|
function difference<T>(setA: Set<T>, setB: Set<T>): Set<T> {
|
|
const result = new Set(setA);
|
|
for (const value of setB) {
|
|
result.delete(value);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function collectLiveKitTrackRefs(
|
|
room: Room,
|
|
sources: ReadonlyArray<Track.Source>,
|
|
onlySubscribedTracks: boolean,
|
|
): VoiceCallTrackRefsState {
|
|
const participants = [room.localParticipant, ...Array.from(room.remoteParticipants.values())];
|
|
const tracks: Array<TrackReferenceOrPlaceholder> = [];
|
|
for (const participant of participants) {
|
|
for (const source of sources) {
|
|
for (const publication of participant.trackPublications.values()) {
|
|
if (publication.source !== source) continue;
|
|
if (onlySubscribedTracks && !publication.track) continue;
|
|
tracks.push({participant, publication, source: publication.source});
|
|
}
|
|
}
|
|
}
|
|
return {tracks, participants};
|
|
}
|
|
|
|
function requiredLiveKitPlaceholders(
|
|
sources: ReadonlyArray<VoiceCallTrackSourceOption>,
|
|
participants: ReadonlyArray<Participant>,
|
|
): Map<Participant['identity'], Array<Track.Source>> {
|
|
const placeholderSources = sources.filter((source) => source.withPlaceholder).map((source) => source.source);
|
|
const placeholderMap = new Map<Participant['identity'], Array<Track.Source>>();
|
|
for (const participant of participants) {
|
|
const publishedSources = participant
|
|
.getTrackPublications()
|
|
.map((publication) => publication.track?.source ?? publication.source)
|
|
.filter((trackSource): trackSource is Track.Source => trackSource !== undefined);
|
|
const neededSources = Array.from(difference(new Set(placeholderSources), new Set(publishedSources)));
|
|
if (neededSources.length > 0) {
|
|
placeholderMap.set(participant.identity, neededSources);
|
|
}
|
|
}
|
|
return placeholderMap;
|
|
}
|
|
|
|
function addLiveKitPlaceholders(
|
|
tracks: ReadonlyArray<TrackReferenceOrPlaceholder>,
|
|
participants: ReadonlyArray<Participant>,
|
|
sources: ReadonlyArray<VoiceCallTrackSourceOption>,
|
|
): Array<TrackReferenceOrPlaceholder> {
|
|
const tracksWithPlaceholders = Array.from(tracks);
|
|
const placeholders = requiredLiveKitPlaceholders(sources, participants);
|
|
for (const participant of participants) {
|
|
const sourcesToAdd = placeholders.get(participant.identity);
|
|
if (!sourcesToAdd) continue;
|
|
for (const source of sourcesToAdd) {
|
|
if (
|
|
tracks.some(
|
|
(trackRef) =>
|
|
trackRef.participant.identity === participant.identity &&
|
|
isTrackReference(trackRef) &&
|
|
trackRef.publication.source === source,
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
tracksWithPlaceholders.push({participant, source});
|
|
}
|
|
}
|
|
return tracksWithPlaceholders;
|
|
}
|
|
|
|
function useOptionalLiveKitVoiceCallTrackRefs(
|
|
sources: ReadonlyArray<VoiceCallTrackSourceOption>,
|
|
options: {onlySubscribed: boolean},
|
|
): VoiceCallTrackRefsState {
|
|
const room = useMaybeRoomContext();
|
|
const [state, setState] = useState<VoiceCallTrackRefsState>({tracks: [], participants: []});
|
|
const sourceValues = useMemo(() => sources.map((source) => source.source), [sources]);
|
|
useEffect(() => {
|
|
if (!room) {
|
|
setState({tracks: [], participants: []});
|
|
return;
|
|
}
|
|
const update = () => {
|
|
setState(collectLiveKitTrackRefs(room, sourceValues, options.onlySubscribed));
|
|
};
|
|
update();
|
|
for (const event of LIVEKIT_TRACK_UPDATE_EVENTS) {
|
|
room.on(event, update);
|
|
}
|
|
return () => {
|
|
for (const event of LIVEKIT_TRACK_UPDATE_EVENTS) {
|
|
room.off(event, update);
|
|
}
|
|
};
|
|
}, [room, options.onlySubscribed, sourceValues]);
|
|
const tracks = useMemo(
|
|
() => addLiveKitPlaceholders(state.tracks, state.participants, sources),
|
|
[state.tracks, state.participants, sources],
|
|
);
|
|
return useMemo(() => ({tracks, participants: state.participants}), [tracks, state.participants]);
|
|
}
|
|
|
|
function getNativeTracksByParticipantSid(
|
|
nativeTracks: Readonly<Record<string, NativeInboundVideoTrack>>,
|
|
): Map<string, Array<NativeInboundVideoTrack>> {
|
|
const map = new Map<string, Array<NativeInboundVideoTrack>>();
|
|
const addTrack = (key: string | undefined, track: NativeInboundVideoTrack): void => {
|
|
if (!key) return;
|
|
const current = map.get(key) ?? [];
|
|
if (!current.includes(track)) {
|
|
current.push(track);
|
|
}
|
|
map.set(key, current);
|
|
};
|
|
for (const track of Object.values(nativeTracks)) {
|
|
addTrack(track.participantSid, track);
|
|
addTrack(track.participantIdentity, track);
|
|
}
|
|
return map;
|
|
}
|
|
|
|
function getNativeTracksForParticipantSnapshot(
|
|
nativeTracksByParticipantSid: Map<string, Array<NativeInboundVideoTrack>>,
|
|
snapshot: VoiceEngineV2AppParticipantSnapshot,
|
|
): Array<NativeInboundVideoTrack> {
|
|
const tracks: Array<NativeInboundVideoTrack> = [];
|
|
for (const key of [snapshot.sid, snapshot.identity]) {
|
|
if (!key) continue;
|
|
for (const track of nativeTracksByParticipantSid.get(key) ?? []) {
|
|
if (!tracks.includes(track)) {
|
|
tracks.push(track);
|
|
}
|
|
}
|
|
}
|
|
return tracks;
|
|
}
|
|
|
|
function selectNativeVideoTrack(
|
|
tracks: ReadonlyArray<NativeInboundVideoTrack>,
|
|
source: Track.Source,
|
|
): NativeInboundVideoTrack | undefined {
|
|
const wantScreenShare = asVoiceTrackSource(source) === VoiceTrackSource.ScreenShare;
|
|
return tracks.find((track) =>
|
|
wantScreenShare ? isVoiceScreenShareSource(track.source) : !isVoiceScreenShareSource(track.source),
|
|
);
|
|
}
|
|
|
|
function createNativePublication({
|
|
participantSid,
|
|
source,
|
|
isMuted,
|
|
isLocal,
|
|
nativeTrack,
|
|
}: {
|
|
participantSid: string;
|
|
source: Track.Source;
|
|
isMuted: boolean;
|
|
isLocal: boolean;
|
|
nativeTrack?: NativeInboundVideoTrack;
|
|
}): NativeVoiceTrackPublication {
|
|
const isVideo = source === CAMERA_SOURCE || source === SCREEN_SHARE_SOURCE;
|
|
return new NativeVoiceTrackPublication({
|
|
trackSid: nativeTrack?.trackSid ?? `${participantSid}:${source}`,
|
|
source,
|
|
kind: isVideo ? Track.Kind.Video : Track.Kind.Audio,
|
|
isMuted,
|
|
isLocal,
|
|
dimensions:
|
|
nativeTrack && nativeTrack.width > 0 && nativeTrack.height > 0
|
|
? {width: nativeTrack.width, height: nativeTrack.height}
|
|
: undefined,
|
|
});
|
|
}
|
|
|
|
function createNativeParticipant(
|
|
snapshot: VoiceEngineV2AppParticipantSnapshot,
|
|
nativeTracks: ReadonlyArray<NativeInboundVideoTrack>,
|
|
): Participant {
|
|
const participant = new Participant(
|
|
snapshot.sid || snapshot.identity,
|
|
snapshot.identity,
|
|
undefined,
|
|
snapshot.metadata,
|
|
{...snapshot.attributes},
|
|
);
|
|
Object.defineProperty(participant, 'isLocal', {
|
|
configurable: true,
|
|
get: () => snapshot.isLocal,
|
|
});
|
|
participant.isSpeaking = Boolean(snapshot.isSpeaking || snapshot.isAudioLevelSpeaking);
|
|
participant.audioLevel = snapshot.isAudioLevelSpeaking ? 1 : 0;
|
|
if (snapshot.lastSpokeAt) {
|
|
participant.lastSpokeAt = new Date(snapshot.lastSpokeAt);
|
|
}
|
|
participant.addTrackPublication(
|
|
createNativePublication({
|
|
participantSid: participant.sid,
|
|
source: VoiceTrackSource.Microphone as Track.Source,
|
|
isMuted: !snapshot.isMicrophoneEnabled,
|
|
isLocal: snapshot.isLocal,
|
|
}),
|
|
);
|
|
const cameraTrack = selectNativeVideoTrack(nativeTracks, CAMERA_SOURCE);
|
|
if (snapshot.isCameraEnabled || cameraTrack) {
|
|
participant.addTrackPublication(
|
|
createNativePublication({
|
|
participantSid: participant.sid,
|
|
source: CAMERA_SOURCE,
|
|
isMuted: !snapshot.isCameraEnabled && !cameraTrack,
|
|
isLocal: snapshot.isLocal,
|
|
nativeTrack: cameraTrack,
|
|
}),
|
|
);
|
|
}
|
|
const screenShareTrack = selectNativeVideoTrack(nativeTracks, SCREEN_SHARE_SOURCE);
|
|
if (snapshot.isScreenShareEnabled || screenShareTrack) {
|
|
participant.addTrackPublication(
|
|
createNativePublication({
|
|
participantSid: participant.sid,
|
|
source: SCREEN_SHARE_SOURCE,
|
|
isMuted: !snapshot.isScreenShareEnabled && !screenShareTrack,
|
|
isLocal: snapshot.isLocal,
|
|
nativeTrack: screenShareTrack,
|
|
}),
|
|
);
|
|
}
|
|
if (snapshot.isScreenShareAudioEnabled) {
|
|
participant.addTrackPublication(
|
|
createNativePublication({
|
|
participantSid: participant.sid,
|
|
source: SCREEN_SHARE_AUDIO_SOURCE,
|
|
isMuted: false,
|
|
isLocal: snapshot.isLocal,
|
|
}),
|
|
);
|
|
}
|
|
return participant;
|
|
}
|
|
|
|
function isNativeSnapshotInChannel(
|
|
snapshot: VoiceEngineV2AppParticipantSnapshot,
|
|
connectionVoiceStates: VoiceGatewayConnectionVoiceStates,
|
|
channel: Channel,
|
|
): boolean {
|
|
const connectionId = snapshot.connectionId;
|
|
if (!connectionId) return true;
|
|
const voiceState = connectionVoiceStates[connectionId];
|
|
if (!voiceState) return true;
|
|
return voiceState.channel_id === channel.id;
|
|
}
|
|
|
|
function buildNativeVoiceCallTrackRefs({
|
|
channel,
|
|
participantSnapshots,
|
|
nativeTracks,
|
|
connectionVoiceStates,
|
|
}: {
|
|
channel: Channel;
|
|
participantSnapshots: Readonly<Record<string, VoiceEngineV2AppParticipantSnapshot>>;
|
|
nativeTracks: Readonly<Record<string, NativeInboundVideoTrack>>;
|
|
connectionVoiceStates: VoiceGatewayConnectionVoiceStates;
|
|
}): VoiceCallTrackRefsState {
|
|
const nativeTracksByParticipantSid = getNativeTracksByParticipantSid(nativeTracks);
|
|
const participants = Object.values(participantSnapshots)
|
|
.filter((snapshot) => isNativeSnapshotInChannel(snapshot, connectionVoiceStates, channel))
|
|
.map((snapshot) =>
|
|
createNativeParticipant(snapshot, getNativeTracksForParticipantSnapshot(nativeTracksByParticipantSid, snapshot)),
|
|
);
|
|
const tracks: Array<TrackReferenceOrPlaceholder> = [];
|
|
for (const participant of participants) {
|
|
const cameraPublication = participant.getTrackPublication(CAMERA_SOURCE);
|
|
if (cameraPublication) {
|
|
tracks.push({participant, publication: cameraPublication, source: CAMERA_SOURCE});
|
|
} else {
|
|
tracks.push({participant, source: CAMERA_SOURCE});
|
|
}
|
|
const screenSharePublication = participant.getTrackPublication(SCREEN_SHARE_SOURCE);
|
|
if (screenSharePublication && !screenSharePublication.isMuted) {
|
|
tracks.push({participant, publication: screenSharePublication, source: SCREEN_SHARE_SOURCE});
|
|
}
|
|
}
|
|
return {tracks, participants};
|
|
}
|
|
|
|
export function buildNativeVoiceCallTrackRefForParticipant({
|
|
participantIdentity,
|
|
source,
|
|
participantSnapshots,
|
|
nativeTracks,
|
|
}: {
|
|
participantIdentity: string;
|
|
source: Track.Source;
|
|
participantSnapshots: Readonly<Record<string, VoiceEngineV2AppParticipantSnapshot>>;
|
|
nativeTracks: Readonly<Record<string, NativeInboundVideoTrack>>;
|
|
}): TrackReferenceOrPlaceholder | null {
|
|
const snapshot = participantSnapshots[participantIdentity];
|
|
if (!snapshot) return null;
|
|
const nativeTracksByParticipantSid = getNativeTracksByParticipantSid(nativeTracks);
|
|
const participant = createNativeParticipant(
|
|
snapshot,
|
|
getNativeTracksForParticipantSnapshot(nativeTracksByParticipantSid, snapshot),
|
|
);
|
|
const publication = participant.getTrackPublication(source);
|
|
if (publication) {
|
|
return {participant, publication, source};
|
|
}
|
|
return {participant, source};
|
|
}
|
|
|
|
function getTrackActivityKey(trackRef: TrackReferenceOrPlaceholder): string {
|
|
return `${trackRef.participant.identity}:${trackRef.source}`;
|
|
}
|
|
|
|
export function useVoiceCallTracksAndLayout({channel, expandedUserIds}: UseVoiceCallTracksAndLayoutArgs) {
|
|
useMediaEngineVersion();
|
|
const {layoutMode, pinnedParticipantIdentity, pinnedParticipantSource} = VoiceCallLayout;
|
|
const trackSortSnapshotRef = useRef(createVoiceParticipantSortSnapshot());
|
|
const trackActivityStateRef = useRef(new Map<string, VoiceGridTrackActivityState>());
|
|
const trackActivityNextOrderRef = useRef(0);
|
|
const compareTracks = useCallback(
|
|
(left: TrackReferenceOrPlaceholder, right: TrackReferenceOrPlaceholder) =>
|
|
compareVoiceTrackReferencesWithSnapshot(
|
|
left,
|
|
right,
|
|
trackSortSnapshotRef.current,
|
|
channel.guildId ?? null,
|
|
channel.id,
|
|
),
|
|
[channel.guildId, channel.id],
|
|
);
|
|
const liveKitTrackRefs = useOptionalLiveKitVoiceCallTrackRefs(VOICE_CALL_TRACK_SOURCES, {onlySubscribed: false});
|
|
const nativeInboundVideoTracks = NativeVideoTileManager.tracks;
|
|
const isNativeEngine = isVoiceEngineV2NativeProjectionActiveFromMediaEngine();
|
|
const connectionVoiceStates = MediaEngine.connectionVoiceStates;
|
|
const participantSnapshots = MediaEngine.participants;
|
|
const nativeTrackRefs = useMemo(
|
|
() =>
|
|
buildNativeVoiceCallTrackRefs({
|
|
channel,
|
|
participantSnapshots,
|
|
nativeTracks: nativeInboundVideoTracks,
|
|
connectionVoiceStates: connectionVoiceStates as VoiceGatewayConnectionVoiceStates,
|
|
}),
|
|
[channel, participantSnapshots, nativeInboundVideoTracks, connectionVoiceStates],
|
|
);
|
|
const tracks = isNativeEngine ? nativeTrackRefs.tracks : liveKitTrackRefs.tracks;
|
|
const participants = isNativeEngine ? nativeTrackRefs.participants : liveKitTrackRefs.participants;
|
|
const userCacheSize = Users.usersList.length;
|
|
const participantCount = useMemo(() => countKnownVoiceParticipants(participants), [participants, userCacheSize]);
|
|
const renderableTracks = useMemo(() => tracks.filter(isKnownVoiceTrackRef), [tracks, userCacheSize]);
|
|
const viewerStreamKeys = selectVoiceMediaGraphViewerStreamKeys(voiceMediaGraphStore.graph);
|
|
const screenSharePublicationMigrationVersion = ScreenSharePublicationMigration.version;
|
|
const localConnectionId = MediaEngine.connectionId;
|
|
const localSelfStream = LocalVoiceState.getSelfStream();
|
|
const {screenShareTracks, cameraTracksAll} = useMemo(
|
|
() => splitVoiceCallRenderableTracks(renderableTracks),
|
|
[renderableTracks, screenSharePublicationMigrationVersion],
|
|
);
|
|
const activeScreenShareState = useMemo(() => {
|
|
const connectionIds = new Set<string>();
|
|
const participantIdentities = new Set<string>();
|
|
for (const tr of screenShareTracks) {
|
|
participantIdentities.add(tr.participant.identity);
|
|
const parsedIdentity = parseVoiceParticipantIdentity(tr.participant.identity);
|
|
if (parsedIdentity.connectionId) {
|
|
connectionIds.add(parsedIdentity.connectionId);
|
|
}
|
|
}
|
|
return {connectionIds, participantIdentities};
|
|
}, [screenShareTracks]);
|
|
const activeScreenShareConnectionIds = activeScreenShareState.connectionIds;
|
|
const activeScreenShareParticipantIdentities = activeScreenShareState.participantIdentities;
|
|
const participantsByIdentity = useMemo(() => {
|
|
const map = new Map<string, TrackReferenceOrPlaceholder['participant']>();
|
|
for (const participant of participants) {
|
|
map.set(participant.identity, participant);
|
|
}
|
|
return map;
|
|
}, [participants]);
|
|
const virtualScreenShareTracks = useMemo<Array<TrackReferenceOrPlaceholder>>(() => {
|
|
if (viewerStreamKeys.length === 0) return [];
|
|
const virtualTracks: Array<TrackReferenceOrPlaceholder> = [];
|
|
for (const viewerStreamKey of viewerStreamKeys) {
|
|
const parsed = parseStreamKey(viewerStreamKey);
|
|
if (!parsed) continue;
|
|
if (parsed.channelId !== channel.id) continue;
|
|
const channelGuildId = channel.guildId ?? null;
|
|
if (parsed.guildId !== channelGuildId) continue;
|
|
const voiceState = connectionVoiceStates[parsed.connectionId];
|
|
if (!voiceState?.user_id) continue;
|
|
if (voiceState.channel_id !== channel.id) continue;
|
|
const isLocalStream = parsed.connectionId === localConnectionId;
|
|
const isStreamActive =
|
|
activeScreenShareConnectionIds.has(parsed.connectionId) ||
|
|
(isLocalStream ? localSelfStream : voiceState.self_stream);
|
|
if (!isStreamActive) continue;
|
|
const identity = buildVoiceParticipantIdentity(voiceState.user_id, parsed.connectionId);
|
|
if (!isKnownVoiceParticipantIdentity(identity)) continue;
|
|
if (activeScreenShareParticipantIdentities.has(identity)) continue;
|
|
const participant = participantsByIdentity.get(identity);
|
|
if (!participant) continue;
|
|
virtualTracks.push({participant, source: SCREEN_SHARE_SOURCE});
|
|
}
|
|
return virtualTracks;
|
|
}, [
|
|
activeScreenShareConnectionIds,
|
|
activeScreenShareParticipantIdentities,
|
|
channel.id,
|
|
channel.guildId,
|
|
connectionVoiceStates,
|
|
localConnectionId,
|
|
localSelfStream,
|
|
participantsByIdentity,
|
|
userCacheSize,
|
|
viewerStreamKeys,
|
|
]);
|
|
const screenShareTracksWithVirtual = useMemo(
|
|
() =>
|
|
dedupeScreenShareTracks(
|
|
virtualScreenShareTracks.length > 0 ? [...screenShareTracks, ...virtualScreenShareTracks] : screenShareTracks,
|
|
),
|
|
[screenShareTracks, virtualScreenShareTracks, screenSharePublicationMigrationVersion],
|
|
);
|
|
const trackSnapshotMembers = useMemo(
|
|
() =>
|
|
[...cameraTracksAll, ...screenShareTracksWithVirtual].map((trackRef) => {
|
|
const identity = parseVoiceParticipantIdentity(trackRef.participant.identity);
|
|
return {
|
|
participantKey: `${identity.userId}:${identity.connectionId}`,
|
|
userId: identity.userId,
|
|
};
|
|
}),
|
|
[cameraTracksAll, screenShareTracksWithVirtual],
|
|
);
|
|
syncVoiceParticipantSortSnapshot(
|
|
trackSortSnapshotRef.current,
|
|
trackSnapshotMembers,
|
|
channel.guildId ?? null,
|
|
channel.id,
|
|
);
|
|
const sortedCameraTracksAll = useMemo(
|
|
() => [...cameraTracksAll].sort(compareTracks),
|
|
[cameraTracksAll, compareTracks],
|
|
);
|
|
const filteredCameraTracks = useMemo(() => {
|
|
if (VoiceSettings.showNonVideoParticipants) return cameraTracksAll;
|
|
return cameraTracksAll.filter((tr) => {
|
|
if (!isTrackReference(tr)) return false;
|
|
if (!tr.publication) return false;
|
|
return !tr.publication.isMuted;
|
|
});
|
|
}, [cameraTracksAll, VoiceSettings.showNonVideoParticipants]);
|
|
const sortedFilteredCameraTracks = useMemo(
|
|
() => [...filteredCameraTracks].sort(compareTracks),
|
|
[filteredCameraTracks, compareTracks],
|
|
);
|
|
const sortedScreenShareTracksWithVirtual = useMemo(
|
|
() => [...screenShareTracksWithVirtual].sort(compareTracks),
|
|
[screenShareTracksWithVirtual, compareTracks],
|
|
);
|
|
const hasScreenShare = sortedScreenShareTracksWithVirtual.length > 0;
|
|
const prioritizeSpeakingParticipants = VoiceSettings.prioritizeSpeakingParticipants;
|
|
const gridTracks = useMemo(() => {
|
|
const now = Date.now();
|
|
const activityStates = trackActivityStateRef.current;
|
|
const trackRefs = hasScreenShare
|
|
? [...sortedScreenShareTracksWithVirtual, ...sortedFilteredCameraTracks]
|
|
: sortedFilteredCameraTracks;
|
|
if (!prioritizeSpeakingParticipants) {
|
|
activityStates.clear();
|
|
trackActivityNextOrderRef.current = 0;
|
|
return trackRefs;
|
|
}
|
|
const activeKeys = new Set<string>();
|
|
const activityByKey = new Map<string, VoiceGridTrackActivity>();
|
|
for (const trackRef of trackRefs) {
|
|
const key = getTrackActivityKey(trackRef);
|
|
const parsedIdentity = parseVoiceParticipantIdentity(trackRef.participant.identity);
|
|
const participantSnapshot = participantSnapshots[trackRef.participant.identity];
|
|
const voiceState = parsedIdentity.connectionId ? connectionVoiceStates[parsedIdentity.connectionId] : undefined;
|
|
const isStreaming =
|
|
asVoiceTrackSource(trackRef.source) === VoiceTrackSource.ScreenShare ||
|
|
participantSnapshot?.isScreenShareEnabled === true ||
|
|
voiceState?.self_stream === true;
|
|
const isSpeaking = isVoiceEngineV2AppParticipantSpeaking(participantSnapshot);
|
|
const lastSpokeAt = participantSnapshot?.lastSpokeAt ?? 0;
|
|
const liveActivityAt = isSpeaking ? now : 0;
|
|
const observedActivityAt = Math.max(liveActivityAt, lastSpokeAt);
|
|
let activityState = activityStates.get(key);
|
|
if (observedActivityAt > 0) {
|
|
if (!activityState) {
|
|
activityState = {
|
|
lastActivityAt: observedActivityAt,
|
|
promotedOrder: trackActivityNextOrderRef.current,
|
|
};
|
|
trackActivityNextOrderRef.current += 1;
|
|
activityStates.set(key, activityState);
|
|
} else {
|
|
activityState.lastActivityAt = Math.max(activityState.lastActivityAt, observedActivityAt);
|
|
}
|
|
}
|
|
if (activityState && now - activityState.lastActivityAt > VOICE_GRID_SPEAKER_PRIORITY_HOLD_MS) {
|
|
activityStates.delete(key);
|
|
activityState = undefined;
|
|
}
|
|
const priority = isStreaming ? 3 : isSpeaking ? 2 : activityState ? 1 : 0;
|
|
activeKeys.add(key);
|
|
activityByKey.set(key, {priority, promotedOrder: activityState?.promotedOrder ?? Number.MAX_SAFE_INTEGER});
|
|
}
|
|
for (const key of activityStates.keys()) {
|
|
if (!activeKeys.has(key)) {
|
|
activityStates.delete(key);
|
|
}
|
|
}
|
|
if (activityStates.size === 0) {
|
|
trackActivityNextOrderRef.current = 0;
|
|
}
|
|
return [...trackRefs].sort((left, right) => {
|
|
const leftActivity = activityByKey.get(getTrackActivityKey(left));
|
|
const rightActivity = activityByKey.get(getTrackActivityKey(right));
|
|
const byPriority = (rightActivity?.priority ?? 0) - (leftActivity?.priority ?? 0);
|
|
if (byPriority !== 0) return byPriority;
|
|
if ((leftActivity?.priority ?? 0) > 0 || (rightActivity?.priority ?? 0) > 0) {
|
|
const byPromotedOrder =
|
|
(leftActivity?.promotedOrder ?? Number.MAX_SAFE_INTEGER) -
|
|
(rightActivity?.promotedOrder ?? Number.MAX_SAFE_INTEGER);
|
|
if (byPromotedOrder !== 0) return byPromotedOrder;
|
|
}
|
|
return compareTracks(left, right);
|
|
});
|
|
}, [
|
|
compareTracks,
|
|
connectionVoiceStates,
|
|
hasScreenShare,
|
|
participantSnapshots,
|
|
prioritizeSpeakingParticipants,
|
|
sortedFilteredCameraTracks,
|
|
sortedScreenShareTracksWithVirtual,
|
|
]);
|
|
useEffect(() => {
|
|
pruneInactiveWatchedStreamsForChannel({
|
|
guildId: channel.guildId,
|
|
channelId: channel.id,
|
|
isStreamActive: (connectionId) => {
|
|
if (activeScreenShareConnectionIds.has(connectionId)) return true;
|
|
if (connectionId === localConnectionId) return localSelfStream;
|
|
const voiceState = connectionVoiceStates[connectionId];
|
|
return Boolean(voiceState?.self_stream && voiceState.channel_id === channel.id);
|
|
},
|
|
});
|
|
}, [
|
|
activeScreenShareConnectionIds,
|
|
channel.guildId,
|
|
channel.id,
|
|
connectionVoiceStates,
|
|
localConnectionId,
|
|
localSelfStream,
|
|
viewerStreamKeys,
|
|
]);
|
|
useEffect(() => {
|
|
const callId = MediaEngine.connectionId ?? '';
|
|
const currentUserId = Users.currentUser?.id;
|
|
if (!callId || !currentUserId) return;
|
|
for (const tr of cameraTracksAll) {
|
|
if (!isTrackReference(tr)) continue;
|
|
if (asVoiceTrackSource(tr.source) !== VoiceTrackSource.Camera) continue;
|
|
const identity = tr.participant.identity || '';
|
|
if (!identity.startsWith(`user_${currentUserId}_`)) continue;
|
|
const {connectionId} = parseVoiceParticipantIdentity(identity);
|
|
if (connectionId !== localConnectionId) continue;
|
|
const isPublishing = Boolean(tr.publication && !tr.publication.isMuted);
|
|
const disabled = !VoiceSettings.showMyOwnCamera && isPublishing;
|
|
CallMediaPrefs.setVideoDisabled(callId, identity, disabled);
|
|
}
|
|
}, [cameraTracksAll, VoiceSettings.showMyOwnCamera, localConnectionId]);
|
|
useEffect(() => {
|
|
if (!pinnedParticipantIdentity) return;
|
|
if (cameraTracksAll.length === 0) return;
|
|
const stillExists = cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity);
|
|
if (!stillExists) VoiceCallLayoutCommands.setPinnedParticipant(null);
|
|
}, [cameraTracksAll, pinnedParticipantIdentity]);
|
|
const {
|
|
mainTrack: focusMainTrack,
|
|
carouselTracks,
|
|
pipTrack,
|
|
} = usePinnedTrackRef({
|
|
layoutMode,
|
|
pinnedParticipantIdentity,
|
|
pinnedParticipantSource,
|
|
filteredCameraTracks: sortedFilteredCameraTracks,
|
|
cameraTracksAll: sortedCameraTracksAll,
|
|
screenShareTracks: sortedScreenShareTracksWithVirtual,
|
|
compareTracks,
|
|
participantSnapshots,
|
|
});
|
|
const gridEntries = useMemo<Array<VoiceGridEntry>>(
|
|
() => consolidateVoiceGridTracks({tracks: gridTracks, expandedUserIds}),
|
|
[gridTracks, expandedUserIds],
|
|
);
|
|
return {
|
|
tracks: renderableTracks,
|
|
screenShareTracks: sortedScreenShareTracksWithVirtual,
|
|
cameraTracksAll: sortedCameraTracksAll,
|
|
filteredCameraTracks: sortedFilteredCameraTracks,
|
|
hasScreenShare,
|
|
gridEntries,
|
|
participantCount,
|
|
layoutMode,
|
|
pinnedParticipantIdentity,
|
|
pinnedParticipantSource,
|
|
focusMainTrack,
|
|
carouselTracks,
|
|
pipTrack,
|
|
channel,
|
|
};
|
|
}
|