Compare commits

...
128 changed files with 5741 additions and 2018 deletions
+4 -4
View File
@@ -314,8 +314,8 @@ jobs:
path: |
~/.cache/rebar3
fluxer_gateway/_build
!fluxer_gateway/_build/default/lib/fluxer_gateway
!fluxer_gateway/_build/test/lib/fluxer_gateway
!fluxer_gateway/_build/default/lib/fluxer_gateway/**
!fluxer_gateway/_build/test/lib/fluxer_gateway/**
key: >-
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
'fluxer_gateway/rebar.config') }}
@@ -345,8 +345,8 @@ jobs:
path: |
~/.cache/rebar3
fluxer_gateway/_build
!fluxer_gateway/_build/default/lib/fluxer_gateway
!fluxer_gateway/_build/test/lib/fluxer_gateway
!fluxer_gateway/_build/default/lib/fluxer_gateway/**
!fluxer_gateway/_build/test/lib/fluxer_gateway/**
key: >-
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
'fluxer_gateway/rebar.config') }}
+2 -16
View File
@@ -12287,14 +12287,7 @@
"max_concurrent_guild_starts": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
"gateway_dispatch_relay_shards": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
"gateway_dispatch_relay_max_queue": {"type": "integer", "minimum": 0, "maximum": 1000000, "format": "int32"},
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"},
"voice_reconciliation_v3_percentage": {"type": "number", "minimum": 0, "maximum": 100},
"voice_reconciliation_v3_interval_ms": {
"type": "integer",
"minimum": 500,
"maximum": 60000,
"format": "int32"
}
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"}
}
},
"InstanceConfigUpdateRequest": {
@@ -12534,14 +12527,7 @@
"max_concurrent_guild_starts": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
"gateway_dispatch_relay_shards": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
"gateway_dispatch_relay_max_queue": {"type": "integer", "minimum": 0, "maximum": 1000000, "format": "int32"},
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"},
"voice_reconciliation_v3_percentage": {"type": "number", "minimum": 0, "maximum": 100},
"voice_reconciliation_v3_interval_ms": {
"type": "integer",
"minimum": 500,
"maximum": 60000,
"format": "int32"
}
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"}
}
},
"BrandingAssetUploadRequest": {
@@ -30,8 +30,8 @@ import {phraseBlocklistCache} from '../../middleware/PhraseBlocklistCache';
import {profileSubstringBlocklistCache} from '../../middleware/ProfileSubstringBlocklistCache';
import {urlBlocklistCache} from '../../middleware/UrlBlocklistCache';
import {
getIpBanBlastRadiusVerdict,
getSuspiciousIpSkipReason,
hasHighCgnatBlastRadiusRisk,
isSingleIpBanCandidate,
} from '../../risk/IpBanCgnatGuard';
import {isIpBanExempt} from '../../risk/IpBanExemptions';
@@ -292,7 +292,7 @@ export class AdminBanManagementService {
return false;
}
try {
const highRisk = await hasHighCgnatBlastRadiusRisk(ip, this.deps.ipInfoService, {
const {cgnat: highRisk} = await getIpBanBlastRadiusVerdict(ip, this.deps.ipInfoService, {
source: 'admin.ip_ban',
reason: 'pre_write_cgnat_guard',
});
@@ -401,7 +401,7 @@ export class GuildDiscoveryService extends IGuildDiscoveryService {
const language =
params.primaryLanguage && isValidDiscoveryLanguage(params.primaryLanguage) ? params.primaryLanguage : undefined;
const tag = params.tag && params.tag.trim().length > 0 ? normalizeDiscoveryTag(params.tag) : undefined;
const sortBy = params.sortBy === 'member_count' ? 'memberCount' : 'relevance';
const sortBy = params.sortBy === 'relevance' ? 'relevance' : 'memberCount';
const filters: GuildSearchFilters = {
isDiscoverable: true,
discoveryCategory: params.categoryId,
@@ -444,7 +444,6 @@ export class GuildDiscoveryService extends IGuildDiscoveryService {
for (const guild of guilds) {
const counts = freshCounts.get(BigInt(guild.id) as GuildID);
if (counts) {
guild.member_count = counts.memberCount;
guild.online_count = counts.onlineCount;
}
}
@@ -19,7 +19,7 @@ import type {UserCacheService} from '../../infrastructure/UserCacheService';
import {Logger} from '../../Logger';
import type {RequestCache} from '../../middleware/RequestCacheMiddleware';
import type {GuildBan} from '../../models/GuildBan';
import {hasHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../../risk/IpBanCgnatGuard';
import {getIpBanBlastRadiusVerdict, isSingleIpBanCandidate} from '../../risk/IpBanCgnatGuard';
import {isIpBanExempt} from '../../risk/IpBanExemptions';
import type {IUserRepository} from '../../user/IUserRepository';
import type {WorkerTaskName} from '../../worker/WorkerLaneConfig';
@@ -237,19 +237,20 @@ export class GuildModerationService {
return true;
}
try {
const highRisk = await hasHighCgnatBlastRadiusRisk(userIp, this.ipInfoService, {
const {cgnat, sharedAccess} = await getIpBanBlastRadiusVerdict(userIp, this.ipInfoService, {
source: 'guild.ip_ban',
reason: 'join_cgnat_guard',
});
const highRisk = cgnat || sharedAccess;
if (highRisk) {
Logger.warn(
{userIp, bannedIp},
'Skipping guild IP ban match because IPInfo indicates high CGNAT blast-radius risk',
'Skipping guild IP ban match because IPInfo indicates high shared-network blast-radius risk',
);
}
return !highRisk;
} catch (error) {
Logger.warn({error, userIp, bannedIp}, 'IPInfo CGNAT guard failed while checking guild IP ban');
Logger.warn({error, userIp, bannedIp}, 'IPInfo blast-radius guard failed while checking guild IP ban');
return true;
}
}
@@ -68,23 +68,10 @@ export class GuildSearchService {
const includeNsfwRequested = searchParams.include_nsfw ?? false;
const canUserAccessNsfw =
guildIsAgeRestricted || includeNsfwRequested ? await this.getCanUserAccessNsfw(userId) : false;
if (guildIsAgeRestricted) {
if (!canUserAccessNsfw) {
throw new NsfwContentRequiresAgeVerificationError();
}
if (!includeNsfwRequested) {
const hitsPerPage = searchParams.hits_per_page ?? 25;
const page = searchParams.page ?? 1;
return {
channels: [],
messages: [],
total: 0,
hits_per_page: hitsPerPage,
page,
};
}
if (guildIsAgeRestricted && !canUserAccessNsfw) {
throw new NsfwContentRequiresAgeVerificationError();
}
const canIncludeNsfw = includeNsfwRequested && canUserAccessNsfw;
const canIncludeNsfw = canUserAccessNsfw && (includeNsfwRequested || guildIsAgeRestricted);
const guildNsfw = guildData?.nsfw ?? false;
const channels = await this.channelRepository.listChannels(channelIds);
const channelMap = new Map<string, Channel>();
@@ -14,7 +14,7 @@ import type {GuildID, RoleID, UserID} from '../../../BrandedTypes';
import {guildIdToRoleId} from '../../../BrandedTypes';
import {Logger} from '../../../Logger';
import type {GuildMember} from '../../../models/GuildMember';
import {hasHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../../../risk/IpBanCgnatGuard';
import {getIpBanBlastRadiusVerdict, isSingleIpBanCandidate} from '../../../risk/IpBanCgnatGuard';
import {isIpBanExempt} from '../../../risk/IpBanExemptions';
import type {IUserRepository} from '../../../user/IUserRepository';
import type {IGuildRepositoryAggregate} from '../../repositories/IGuildRepositoryAggregate';
@@ -119,14 +119,15 @@ export class GuildMemberValidationService {
return true;
}
try {
const highRisk = await hasHighCgnatBlastRadiusRisk(userIp, this.ipInfoService, {
const {cgnat, sharedAccess} = await getIpBanBlastRadiusVerdict(userIp, this.ipInfoService, {
source: 'guild.member_ip_ban',
reason: 'join_cgnat_guard',
});
const highRisk = cgnat || sharedAccess;
if (highRisk) {
Logger.warn(
{userIp, bannedIp},
'Skipping guild member IP ban match because IPInfo indicates high CGNAT blast-radius risk',
'Skipping guild member IP ban match because IPInfo indicates high shared-network blast-radius risk',
);
}
return !highRisk;
@@ -8,13 +8,21 @@ import type {
DiscoveryCategoryResponse,
DiscoveryGuildListResponse,
} from '@fluxer/schema/src/domains/guild/GuildDiscoverySchemas';
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
import {createTestAccount, setUserACLs} from '../../auth/tests/AuthTestUtils';
import type {GuildID} from '../../BrandedTypes';
import {createTestBotAccount} from '../../bot/tests/BotTestUtils';
import {setInjectedGatewayService} from '../../middleware/ServiceRegistry';
import {getGuildRepository} from '../../middleware/ServiceSingletons';
import {banUser} from '../../moderation/tests/ModerationTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import {NoopGatewayService} from '../../test/NoopGatewayService';
import {HTTP_STATUS, TEST_IDS} from '../../test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '../../test/TestRequestBuilder';
import syncDiscoveryIndex from '../../worker/tasks/SyncDiscoveryIndex';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../../worker/WorkerContext';
import {createGuild, getUserGuilds} from './GuildTestUtils';
async function setGuildMemberCount(harness: ApiTestHarness, guildId: string, memberCount: number): Promise<void> {
@@ -24,6 +32,30 @@ async function setGuildMemberCount(harness: ApiTestHarness, guildId: string, mem
.execute();
}
interface LiveGuildCounts {
memberCount: number;
onlineCount: number;
}
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
class LiveCountsGatewayService extends NoopGatewayService {
constructor(private readonly liveCounts: Map<string, LiveGuildCounts>) {
super();
}
override async getDiscoveryGuildCounts(guildIds: Array<GuildID>): Promise<Map<GuildID, LiveGuildCounts>> {
const counts = new Map<GuildID, LiveGuildCounts>();
for (const guildId of guildIds) {
const live = this.liveCounts.get(guildId.toString());
if (live) {
counts.set(guildId, live);
}
}
return counts;
}
}
async function applyAndApprove(
harness: ApiTestHarness,
ownerToken: string,
@@ -44,12 +76,39 @@ async function applyAndApprove(
.execute();
}
async function createApprovedDiscoveryGuild(
harness: ApiTestHarness,
adminToken: string,
name: string,
memberCount: number,
): Promise<string> {
const owner = await createTestAccount(harness);
const guild = await createGuild(harness, owner.token, name);
await setGuildMemberCount(harness, guild.id, memberCount);
await applyAndApprove(
harness,
owner.token,
adminToken,
guild.id,
`${name} welcomes everyone`,
DiscoveryCategories.GAMING,
);
return guild.id;
}
function expectNonIncreasing(counts: Array<number>): void {
for (let index = 1; index < counts.length; index++) {
expect(counts[index]).toBeLessThanOrEqual(counts[index - 1]);
}
}
describe('Discovery Search and Join', () => {
let harness: ApiTestHarness;
beforeEach(async () => {
harness = await createApiTestHarness({search: 'enabled'});
});
afterEach(async () => {
clearWorkerDependencies();
await harness?.shutdown();
});
describe('categories', () => {
@@ -257,6 +316,82 @@ describe('Discovery Search and Join', () => {
.execute();
expect(results.guilds.length).toBeLessThanOrEqual(2);
});
test('should order results by the member count it reports back', async () => {
const liveCounts = new Map<string, LiveGuildCounts>();
setInjectedGatewayService(new LiveCountsGatewayService(liveCounts));
const admin = await createTestAccount(harness);
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
const guildIds: Array<string> = [];
for (const memberCount of [50, 40, 30, 20, 10]) {
guildIds.push(
await createApprovedDiscoveryGuild(harness, admin.token, `Ordered Guild ${memberCount}`, memberCount),
);
}
liveCounts.set(guildIds[0], {memberCount: 5, onlineCount: 3});
liveCounts.set(guildIds[4], {memberCount: 500, onlineCount: 7});
const searcher = await createTestAccount(harness);
const results = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
.get('/discovery/guilds?sort_by=member_count&limit=48')
.expect(HTTP_STATUS.OK)
.execute();
expect(results.guilds.map((guild) => guild.id)).toEqual(guildIds);
expectNonIncreasing(results.guilds.map((guild) => guild.member_count));
expect(results.guilds[0].online_count).toBe(3);
expect(results.guilds[4].online_count).toBe(7);
});
test('should rank by member count when the client omits sort_by', async () => {
const admin = await createTestAccount(harness);
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
const guildsByCount = new Map<number, string>();
for (const memberCount of [30, 10, 20]) {
guildsByCount.set(
memberCount,
await createApprovedDiscoveryGuild(harness, admin.token, `Unsorted Guild ${memberCount}`, memberCount),
);
}
const searcher = await createTestAccount(harness);
const results = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
.get('/discovery/guilds?limit=48')
.expect(HTTP_STATUS.OK)
.execute();
expect(results.guilds.map((guild) => guild.id)).toEqual([
guildsByCount.get(30),
guildsByCount.get(20),
guildsByCount.get(10),
]);
expectNonIncreasing(results.guilds.map((guild) => guild.member_count));
});
test('should not repeat guilds across pages when the discovery index is resynced', async () => {
const liveCounts = new Map<string, LiveGuildCounts>();
const gatewayService = new LiveCountsGatewayService(liveCounts);
setInjectedGatewayService(gatewayService);
const admin = await createTestAccount(harness);
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
const guildIds: Array<string> = [];
for (const [index, memberCount] of [60, 50, 40, 40, 30, 30].entries()) {
guildIds.push(await createApprovedDiscoveryGuild(harness, admin.token, `Paged Guild ${index}`, memberCount));
}
const searcher = await createTestAccount(harness);
const firstPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=0')
.expect(HTTP_STATUS.OK)
.execute();
expect(firstPage.guilds.map((guild) => guild.id)).toEqual([guildIds[0], guildIds[1]]);
liveCounts.set(guildIds[0], {memberCount: 5, onlineCount: 0});
setWorkerDependenciesForTest({guildRepository: getGuildRepository(), gatewayService});
await syncDiscoveryIndex({}, WORKER_HELPERS);
const secondPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=2')
.expect(HTTP_STATUS.OK)
.execute();
const thirdPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=4')
.expect(HTTP_STATUS.OK)
.execute();
const paged = [...firstPage.guilds, ...secondPage.guilds, ...thirdPage.guilds].map((guild) => guild.id);
expect(new Set(paged).size).toBe(paged.length);
expect([...paged].sort()).toEqual([...guildIds].sort());
});
test('should require login to search', async () => {
await createBuilderWithoutAuth(harness).get('/discovery/guilds').expect(HTTP_STATUS.UNAUTHORIZED).execute();
});
@@ -324,9 +324,6 @@ export class LiveKitService extends ILiveKitService {
participants: participants.map((participant) => ({identity: participant.identity})),
};
} catch (error) {
if (LiveKitService.isHttp404(error)) {
return {status: 'ok', participants: []};
}
Logger.warn({error, regionId, serverId, roomName}, 'LiveKit listParticipants failed');
const status = LiveKitService.getHttpStatus(error);
const isRetryable = status != null && status >= 500;
@@ -2,7 +2,8 @@
import {AccessToken, TrackSource} from 'livekit-server-sdk';
import {describe, expect, it} from 'vitest';
import {computeLiveKitPublishSources, VOICE_TOKEN_TTL_SECONDS} from '../LiveKitService';
import {createChannelID, createGuildID} from '../../BrandedTypes';
import {computeLiveKitPublishSources, LiveKitService, VOICE_TOKEN_TTL_SECONDS} from '../LiveKitService';
function decodeJwtPayload(token: string): Record<string, unknown> {
const [, payload] = token.split('.');
@@ -57,3 +58,78 @@ describe('LiveKitService publish permissions', () => {
expect(exp - nowSeconds).toBeGreaterThan(0);
});
});
class FakeTwirpError extends Error {
status: number;
code?: string;
constructor(message: string, status: number, code?: string) {
super(message);
this.name = 'TwirpError';
this.status = status;
this.code = code;
}
}
function createServiceWithRoomServiceClient(roomServiceClient: unknown): LiveKitService {
const service = Object.create(LiveKitService.prototype) as LiveKitService;
Reflect.set(
service,
'serverClients',
new Map([
[
'region-1',
new Map([
[
'region-1-server-1',
{
endpoint: 'ws://livekit.test/livekit',
apiKey: 'test-key',
apiSecret: 'test-secret',
isActive: true,
roomServiceClient,
},
],
]),
],
]),
);
return service;
}
describe('LiveKitService listParticipants', () => {
const params = {
guildId: createGuildID(1n),
channelId: createChannelID(2n),
regionId: 'region-1',
serverId: 'region-1-server-1',
};
it('reports a 404 as an unreadable room instead of an empty one', async () => {
const service = createServiceWithRoomServiceClient({
listParticipants: async () => {
throw new FakeTwirpError('not_found', 404, 'not_found');
},
});
const result = await service.listParticipants(params);
expect(result.status).toBe('error');
});
it('reports a bad_route 404 as an unreadable room instead of an empty one', async () => {
const service = createServiceWithRoomServiceClient({
listParticipants: async () => {
throw new FakeTwirpError('invalid path prefix', 404, 'bad_route');
},
});
const result = await service.listParticipants(params);
expect(result.status).toBe('error');
expect(result.status === 'error' && result.retryable).toBe(false);
});
it('still reports a genuinely empty room as empty', async () => {
const service = createServiceWithRoomServiceClient({
listParticipants: async () => [],
});
const result = await service.listParticipants(params);
expect(result).toEqual({status: 'ok', participants: []});
});
});
@@ -63,8 +63,6 @@ describe('GatewayRolloutConfigPublisher', () => {
gateway_dispatch_relay_shards: 32,
gateway_dispatch_relay_max_queue: 50000,
voice_e2ee_scope: 'guild_feature_only',
voice_reconciliation_v3_percentage: 100,
voice_reconciliation_v3_interval_ms: 2000,
};
await publisher.publish(config);
@@ -42,8 +42,6 @@ const DEFAULT_GATEWAY_ROLLOUT_CONFIG: GatewayRolloutConfig = {
gateway_dispatch_relay_shards: 32,
gateway_dispatch_relay_max_queue: 50000,
voice_e2ee_scope: 'guild_feature_only',
voice_reconciliation_v3_percentage: 100,
voice_reconciliation_v3_interval_ms: 2000,
};
export type InstanceRegistrationMode = 'open' | 'approval' | 'closed';
export interface InstanceRegistrationConfig {
+16 -8
View File
@@ -7,9 +7,14 @@ import {isTrustedCommercialPrivacyProvider} from './TrustedPrivacyProviders';
const VERDICT_CACHE_TTL_MS = 60 * 60 * 1000;
interface IpBanBlastRadiusVerdict {
cgnat: boolean;
sharedAccess: boolean;
}
interface CachedVerdict {
expiresAtMs: number;
highRisk: boolean;
verdict: IpBanBlastRadiusVerdict;
}
const verdictCache = new Map<string, CachedVerdict>();
@@ -48,7 +53,7 @@ export function getSuspiciousIpSkipReason(result: IpInfoLookupResult): Suspiciou
return null;
}
function isHighSharedAccessBlastRadiusRisk(result: IpInfoLookupResult): boolean {
export function isHighSharedAccessBlastRadiusRisk(result: IpInfoLookupResult): boolean {
if (result.flags.isHosting || isAnonymousAccess(result)) {
return false;
}
@@ -60,29 +65,32 @@ export function isSingleIpBanCandidate(value: string): boolean {
return parseIpBanEntry(value)?.type === 'single';
}
export async function hasHighCgnatBlastRadiusRisk(
export async function getIpBanBlastRadiusVerdict(
ip: string,
ipInfoService: IpInfoService,
context: {
source: string;
reason: string;
},
): Promise<boolean> {
): Promise<IpBanBlastRadiusVerdict> {
const now = Date.now();
const cacheKey = getSameIpDecisionKey(ip) ?? ip;
const cached = verdictCache.get(cacheKey);
if (cached && cached.expiresAtMs > now) {
return cached.highRisk;
return cached.verdict;
}
const result = await ipInfoService.lookup(ip, {
source: context.source,
reason: context.reason,
metadata: {policy: 'ip_ban_cgnat_guard'},
});
const highRisk = isHighCgnatBlastRadiusRisk(result);
const verdict: IpBanBlastRadiusVerdict = {
cgnat: isHighCgnatBlastRadiusRisk(result),
sharedAccess: isHighSharedAccessBlastRadiusRisk(result),
};
verdictCache.set(cacheKey, {
highRisk,
verdict,
expiresAtMs: now + VERDICT_CACHE_TTL_MS,
});
return highRisk;
return verdict;
}
@@ -2,7 +2,11 @@
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
import {describe, expect, it} from 'vitest';
import {isHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../IpBanCgnatGuard';
import {
isHighCgnatBlastRadiusRisk,
isHighSharedAccessBlastRadiusRisk,
isSingleIpBanCandidate,
} from '../IpBanCgnatGuard';
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
return {
@@ -96,4 +100,63 @@ describe('IpBanCgnatGuard', () => {
),
).toBe(false);
});
it('flags satellite, anycast and education networks as high blast-radius risk', () => {
expect(
isHighSharedAccessBlastRadiusRisk(
ipInfoResult({
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
}),
),
).toBe(true);
expect(
isHighSharedAccessBlastRadiusRisk(
ipInfoResult({
flags: {isAnycast: true, isHosting: false, isMobile: false, isSatellite: false},
}),
),
).toBe(true);
expect(
isHighSharedAccessBlastRadiusRisk(
ipInfoResult({asn: {asn: 'AS64500', number: 64500, name: 'Test University', domain: null, type: 'education'}}),
),
).toBe(true);
});
it('does not flag ordinary residential networks as shared-access risk', () => {
expect(isHighSharedAccessBlastRadiusRisk(ipInfoResult())).toBe(false);
});
it('does not treat shared-access networks as CGNAT risk', () => {
expect(
isHighCgnatBlastRadiusRisk(
ipInfoResult({
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
}),
),
).toBe(false);
});
it('does not exempt hosting or anonymous shared-access infrastructure', () => {
expect(
isHighSharedAccessBlastRadiusRisk(
ipInfoResult({
flags: {isAnycast: true, isHosting: true, isMobile: false, isSatellite: false},
}),
),
).toBe(false);
expect(
isHighSharedAccessBlastRadiusRisk(
ipInfoResult({
anonymous: {
isAnonymous: true,
providerName: 'Example VPN',
isVpn: true,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
percentDaysSeen: null,
},
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
}),
),
).toBe(false);
});
});
@@ -344,7 +344,7 @@ describe('Message Search Permissions', () => {
}
expect(result.messages.length).toBe(0);
});
test('age-restricted guild requires include_nsfw: true', async () => {
test('age-restricted guild is searchable by an adult member without include_nsfw', async () => {
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
const guild = await createGuild(harness, owner.token, 'Age Restricted Search Guild');
const systemChannelId = guild.system_channel_id!;
@@ -363,7 +363,7 @@ describe('Message Search Permissions', () => {
if (!isSearchResult(excluded)) {
expect.fail('Expected search result but got indexing response');
}
expect(excluded.messages.length).toBe(0);
expect(excluded.messages.some((m) => m.channel_id === systemChannelId)).toBe(true);
const included = await createBuilder<MessageSearchResponse>(harness, owner.token)
.post('/search/messages')
.body({
@@ -379,6 +379,30 @@ describe('Message Search Permissions', () => {
expect(included.messages.length).toBeGreaterThan(0);
expect(included.messages.some((m) => m.channel_id === systemChannelId)).toBe(true);
});
test('age-restricted guild is searchable in a channel pinned to nsfw_override: false', async () => {
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
const guild = await createGuild(harness, owner.token, 'Age Restricted Override Guild');
const channel = await createBuilder<{id: string; nsfw_override?: boolean | null}>(harness, owner.token)
.post(`/guilds/${guild.id}/channels`)
.body({name: 'override-channel', type: ChannelTypes.GUILD_TEXT, nsfw: false})
.execute();
expect(channel.nsfw_override).toBe(false);
await sendChannelMessage(harness, owner.token, channel.id, 'age restricted override searchable message');
await updateGuild(harness, owner.token, guild.id, {nsfw_level: GuildNSFWLevel.AGE_RESTRICTED});
await markGuildChannelsAsIndexed(harness, owner.token, guild.id);
const result = await createBuilder<MessageSearchResponse>(harness, owner.token)
.post('/search/messages')
.body({
content: 'age restricted override searchable',
context_guild_id: guild.id,
})
.expect(HTTP_STATUS.OK)
.execute();
if (!isSearchResult(result)) {
expect.fail('Expected search result but got indexing response');
}
expect(result.messages.some((m) => m.channel_id === channel.id)).toBe(true);
});
test('underage user cannot search messages in an age-restricted guild', async () => {
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
const underageMember = await createTestAccount(harness, {dateOfBirth: '2012-01-01'});
@@ -441,16 +441,25 @@ function collectGuildText(doc: SearchableGuild): Array<string | null> {
return [doc.name, doc.vanityUrlCode, doc.discoveryDescription, ...doc.discoveryTags];
}
const sortGuildsByCreatedAt = sortNumericField<SearchableGuild, GuildSearchFilters>('createdAt', 'asc');
const sortGuildsByMemberCount = sortNumericField<SearchableGuild, GuildSearchFilters>('memberCount', 'desc');
function sortGuilds(left: SearchableGuild, right: SearchableGuild, filters: GuildSearchFilters, query: string): number {
const sorter = filters.sortBy === 'memberCount' ? sortGuildsByMemberCount : sortGuildsByCreatedAt;
const delta = sorter(left, right, filters, query);
if (delta !== 0) return delta;
const leftId = BigInt(left.id);
const rightId = BigInt(right.id);
if (leftId === rightId) return 0;
return leftId > rightId ? -1 : 1;
}
class InMemoryGuildSearchService
extends InMemorySearchServiceBase<GuildSearchFilters, SearchableGuild>
implements IGuildSearchService
{
constructor() {
super(
matchesGuildFilters,
collectGuildText,
sortNumericField<SearchableGuild, GuildSearchFilters>('createdAt', 'asc'),
);
super(matchesGuildFilters, collectGuildText, sortGuilds);
}
async indexGuild(guild: Guild, discovery?: GuildDiscoveryContext): Promise<void> {
@@ -2,7 +2,6 @@
import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import type {GuildID} from '../../BrandedTypes';
import {GuildDiscoveryRepository} from '../../guild/repositories/GuildDiscoveryRepository';
import {getGuildSearchService} from '../../SearchFactory';
import {mapWithConcurrency} from '../../utils/ConcurrencyUtils';
@@ -17,7 +16,7 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
helpers.logger.warn('Search service not available, skipping discovery index sync');
return;
}
const {guildRepository, gatewayService} = getWorkerDependencies();
const {guildRepository} = getWorkerDependencies();
const discoveryRepository = new GuildDiscoveryRepository();
const approvedRows = await discoveryRepository.listByStatus(DiscoveryApplicationStatus.APPROVED);
if (approvedRows.length === 0) {
@@ -25,21 +24,6 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
return;
}
const guildIds = approvedRows.map((row) => row.guild_id);
let freshCounts = new Map<
GuildID,
{
memberCount: number;
onlineCount: number;
}
>();
try {
freshCounts = await gatewayService.getDiscoveryGuildCounts(guildIds);
} catch (error) {
helpers.logger.warn(
{error: error instanceof Error ? error.message : String(error)},
'Failed to fetch fresh guild counts from gateway, using database values',
);
}
let synced = 0;
for (let i = 0; i < guildIds.length; i += BATCH_SIZE) {
const batch = guildIds.slice(i, i + BATCH_SIZE);
@@ -54,7 +38,7 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
if (!guild) return null;
const discoveryRow = discoveryRows[index];
if (!discoveryRow || discoveryRow.status !== DiscoveryApplicationStatus.APPROVED) return null;
return {guild, discoveryRow, counts: freshCounts.get(guildId)};
return {guild, discoveryRow};
})
.filter((update): update is NonNullable<typeof update> => update != null);
await mapWithConcurrency(updates, UPDATE_CONCURRENCY, (update) =>
@@ -63,7 +47,6 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
categoryId: update.discoveryRow.category_type,
primaryLanguage: update.discoveryRow.primary_language ?? null,
tags: update.discoveryRow.custom_tags ?? [],
memberCount: update.counts?.memberCount,
}),
);
synced += updates.length;
+17 -1
View File
@@ -28,6 +28,12 @@ const MODULE_REGISTRY_TEST_FILES = [
'src/api/risk/__tests__/AccountPolicyService.test.ts',
];
const INSTANCE_POLICY_TEST_FILES = [
'src/api/admin/tests/InstanceConfigPendingRegistrationApproval.test.ts',
'src/api/auth/tests/DeferredPhoneGate.test.ts',
'src/api/instance/tests/SingleCommunityService.test.ts',
];
const sharedExclude = [
...configDefaults.exclude,
'pkgs/**',
@@ -76,7 +82,7 @@ export default defineConfig({
...sharedTestConfig,
name: 'api',
include: ['src/**/*.{test,spec}.{ts,tsx}'],
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES],
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES, ...INSTANCE_POLICY_TEST_FILES],
isolate: false,
},
},
@@ -90,6 +96,16 @@ export default defineConfig({
isolate: true,
},
},
{
plugins: [tsconfigPaths()],
test: {
...sharedTestConfig,
name: 'api-instance-policy',
include: INSTANCE_POLICY_TEST_FILES,
exclude: sharedExclude,
isolate: true,
},
},
],
},
});
+36 -85
View File
@@ -7,114 +7,65 @@
## Fluxer modifications
Changes applied on top of the upstream v2.17.2 source. Previously maintained as
a pnpm patch at `patches/[email protected]`; now maintained as regular
source edits in this package.
Changes applied on top of the upstream v2.17.2 source. These were previously a pnpm patch at `patches/[email protected]` and are now plain source edits in this package.
1. **AV1 E2EE support** (`src/e2ee/worker/av1Crypto.ts`, `FrameCryptor.ts`, `e2ee.worker.ts`)
OBU-level AV1 encryption and decryption for end-to-end encrypted voice/video.
OBU-level AV1 encryption and decryption for end-to-end encrypted voice and video.
2. **UpdateTrackContext message** (`src/e2ee/types.ts`, worker dispatch)
Replaced `updateCodec` with richer `updateTrackContext` carrying participant
identity and track ID, preventing codec mismatch on track reuse.
`updateCodec` replaced with `updateTrackContext`, which carries participant identity and track id so a reused track cannot pick up the wrong codec.
3. **E2EEManager state tracking** (`src/e2ee/E2eeManager.ts`)
Added `getE2EETransformState()` / `setE2EETransformState()` for proper
transform lifecycle management.
4. **Screenshare scalability mode** (`src/room/participant/LocalParticipant.ts`)
Preserve caller-supplied `scalabilityMode` for screen shares instead of forcing
`L3T3_KEY`, so VP9/AV1 screen shares can use the browser's compatible default
unless Fluxer explicitly requests an SVC layer layout.
`getE2EETransformState()` and `setE2EETransformState()` for transform lifecycle management.
4. **Screen share scalability mode** (`src/room/participant/LocalParticipant.ts`)
A caller-supplied `scalabilityMode` is preserved for screen shares instead of being forced to `L3T3_KEY`, so VP9 and AV1 screen shares use the browser default unless Fluxer asks for a specific SVC layout.
5. **E2EE frame layout guards** (`src/e2ee/worker/FrameCryptor.ts`)
Validate encrypted frame trailer, IV, tag, and clear-prefix bounds before
constructing typed-array views, and drop malformed encrypted frames without
tearing down the transform stream.
Encrypted frame trailer, IV, tag and clear-prefix bounds are validated before any typed-array view is constructed, and malformed frames are dropped without tearing down the transform stream.
6. **Encrypted backup codec publishing** (`src/room/participant/LocalParticipant.ts`, `src/e2ee/E2eeManager.ts`)
Allows backup codec tracks to be advertised and published while E2EE is
enabled, and attaches sender transforms to backup codec senders using their
cloned media track ID and codec.
Backup codec tracks can be advertised and published while E2EE is on, with sender transforms attached using the cloned media track id and codec.
7. **Publisher codec preferences** (`src/room/RTCEngine.ts`)
Applies `RTCRtpTransceiver.setCodecPreferences()` to publisher transceivers
so browser SDP follows the selected primary or backup codec, and prefers
H.264 profiles that use Chromium's external/hardware encoder before the
OpenH264 software profile.
`setCodecPreferences()` is applied to publisher transceivers so the browser's SDP follows the selected primary or backup codec. H.264 profiles rank Baseline `42001f` first, then Constrained Baseline `42e01f`, then everything else. Main, High and Constrained High rank last on purpose.
livekit-server registers H.264 High `640032` on the publisher peer connection but filters it off the subscriber peer connection, and its `CodecParametersFuzzySearch` falls back to a mime-only match. A High publication therefore reaches subscribers under their `42e01f` payload type and decodes to nothing on a Constrained-Baseline-only decoder such as Firefox's OpenH264 GMP. `42001f` is the one profile Chromium's accelerated encoder factory advertises that such a decoder can still handle, because Chromium's VAAPI encoder and OpenH264 both write a Constrained Baseline SPS for `H264PROFILE_BASELINE`.
The trade is that livekit-server does not register `42001f` either, so Windows and macOS negotiate `42e01f`, which Chromium's accelerated encoder factory does not advertise there (`kPlatformH264CbpEncoding` is off by default on Windows, and `IsH264ConstrainedBaselineProfileAvailableForAcceleratedEncoder` returns false on Apple). Those publishers fall back to software H.264, which is what an unpatched browser does anyway. Linux, ChromeOS and Android keep hardware encoding.
8. **Media publishing defaults** (`src/room/defaults.ts`, `src/room/utils.ts`, `src/room/track/options.ts`)
Falls back to H.264, then VP9, VP8, AV1, and HEVC/H.265 according to actual
sender capabilities, pairs advanced codecs with H.264 backup simulcast, and
uses maintain-resolution screen-share defaults with a 4K60-ready bitrate cap.
The order puts AV1 and HEVC last because both are opt-in in Fluxer, so a
fallback inside `publishTrack` must not land on a codec the user did not
enable. Fluxer picks the codec itself before publishing, so this list only
applies when the client overrides the request, such as the reconnect
republish that runs outside Fluxer's own flows.
Codec fallback follows actual sender capabilities in the order H.264, VP9, VP8, AV1, HEVC. Advanced codecs are paired with an H.264 backup simulcast, and screen shares default to maintain-resolution with a 4K60-ready bitrate cap. AV1 and HEVC come last because both are opt-in in Fluxer, so a fallback inside `publishTrack` must not land on a codec the user did not enable. Fluxer picks the codec itself before publishing, so this order only applies when the client overrides the request, such as the reconnect republish that runs outside Fluxer's own flows.
9. **High-fidelity Opus SDP munging** (`src/room/PCTransport.ts`)
Forces Opus RED/FEC, stereo signaling, 10 ms packet time, no DTX, and a
510 kbps maximum average bitrate in local offers and remote answers.
Local offers and remote answers are munged to force Opus RED and FEC, 10 ms packet time, no DTX, and a 510 kbps maximum average bitrate. Stereo signalling stays opt-in. `stereo=1` and `sprop-stereo=1` are added only for publications whose `TrackBitrateInfo.stereo` is set (studio mode above the stereo bitrate threshold, and screen-share audio) and for the subscriber mids the server advertised as stereo, so a mono microphone is not encoded and decoded as a two-channel stream.
10. **Remote audio volume restore at exactly zero** (`src/room/track/RemoteAudioTrack.ts`)
`attach()`, `connectWebAudio()` and `getVolume()` guarded the remembered
`elementVolume` with a truthiness check, so a track deliberately held at `0`
came back at full volume whenever it was re-attached or its Web Audio graph
was rebuilt. All three guards now test `!== undefined`. Note that remote
gains above `1.0` are only legal because `setVolume()` takes the Web Audio
`gainNode` branch; the `el.volume` branch would throw `IndexSizeError`.
`webAudioMix` must stay unconditional.
`attach()`, `connectWebAudio()` and `getVolume()` guarded the remembered `elementVolume` with a truthiness check, so a track deliberately held at `0` came back at full volume whenever it was re-attached or its Web Audio graph was rebuilt. All three now test `!== undefined`. Remote gains above `1.0` are only legal because `setVolume()` takes the Web Audio `gainNode` branch, as the `el.volume` branch would throw `IndexSizeError`, so `webAudioMix` must stay unconditional.
11. **Processor teardown before source stop** (`src/room/track/LocalTrack.ts`)
`stop()` called `super.stop()` first, killing the source `MediaStreamTrack`
and closing the readable feeding a track processor before `processor.destroy()`
ran. A camera-effect worker therefore saw input EOF before its owner's stop
command and reported an operational failure during an ordinary camera-off.
The processor is now captured, detached, and its teardown initiated before
`super.stop()`.
12. **Transactional source and processor swaps** (`src/room/track/LocalTrack.ts`,
`LocalVideoTrack.ts`, `LocalAudioTrack.ts`)
`setMediaStreamTrack()` applied the new source, restarted the processor and
re-armed the sender with no unwind path, so a failure anywhere in the middle
left a half-applied track: listeners moved, elements detached, sender pointing
at a dead track. It now takes `SetMediaStreamTrackOptions`
(`force`, `deferEndedListener`, `preservePreviousTrack`) and, on failure,
restores the previous source, constraints, `enabled` state, listeners,
processor and sender, throwing `TrackInvalidError` when the previous source is
no longer `live` because an ended track cannot be restored. Both errors are
surfaced together as an `AggregateError` when the unwind itself fails.
`stageTrackReplacement()` / `commitStagedTrackReplacement()` expose a two-phase
swap: the candidate becomes the active source with its `ended` listener
deferred and the previous source preserved, and only the commit adopts the
`ended` listener and clears the staged identity, so a caller can validate its
publication before the swap is observable. `replaceTrack()` and `restart()`
guard the `providedByUser` flip behind a `replacementCommitted` flag.
`restart()` still detaches and stops the previous source before calling
`getUserMedia()`, as upstream does, because Safari ends a freshly acquired
track with a capture failure while the old track for the same device is
live. `setSimulcastTrackSender()` routes an already-installed processor's
`processedTrack` to a newly registered secondary sender so a backup codec
never publishes raw frames while the primary is processed.
Processor install and teardown in all three classes roll the processed/raw
sender track back, including `LocalVideoTrack`'s secondary simulcast senders,
and aggregate every cleanup failure instead of discarding it.
`stop()` called `super.stop()` first, killing the source `MediaStreamTrack` and closing the readable that feeds a track processor before `processor.destroy()` ran. A camera-effect worker therefore saw input EOF before its owner's stop command and reported an operational failure during an ordinary camera-off. The processor is now captured, detached and torn down before `super.stop()`.
13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`,
`src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
`x-google-start-bitrate` was reachable only by AV1 and VP9, gated twice: the
publish path registered a track bitrate only for SVC codecs, and the offer
munging returned early for everything else. H264, H265 and VP8 therefore
opened at the Chromium default and had to ramp, which showed up as a 3000 kbps
screen share encoding at 346 kbps twenty seconds in. The bitrate is now
registered for every video codec from the highest encoding
(`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and
the offer munging applies the start bitrate whenever a max bitrate is known.
The dependency descriptor extension stays SVC-only.
`appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and
`setTrackCodecBitrate()` now replaces an entry for the same cid or transceiver
instead of appending, since `trackBitrates` is never cleared.
12. **Transactional source and processor swaps** (`src/room/track/LocalTrack.ts`, `LocalVideoTrack.ts`, `LocalAudioTrack.ts`)
`setMediaStreamTrack()` applied the new source, restarted the processor and re-armed the sender with no unwind path, so a failure part-way through left a half-applied track with listeners moved, elements detached and the sender pointing at a dead track. It now takes `SetMediaStreamTrackOptions` (`force`, `deferEndedListener`, `preservePreviousTrack`) and, on failure, restores the previous source, constraints, `enabled` state, listeners, processor and sender. It throws `TrackInvalidError` when the previous source is no longer `live`, because an ended track cannot be restored, and surfaces both failures as an `AggregateError` when the unwind itself fails.
`stageTrackReplacement()` and `commitStagedTrackReplacement()` add a two-phase swap. The candidate becomes the active source with its `ended` listener deferred and the previous source preserved, and only the commit adopts the `ended` listener and clears the staged identity, so a caller can validate its publication before the swap is observable. `replaceTrack()` and `restart()` guard the `providedByUser` flip behind a `replacementCommitted` flag. `restart()` still detaches and stops the previous source before calling `getUserMedia()`, as upstream does, because Safari ends a freshly acquired track with a capture failure while the old track for the same device is still live. `setSimulcastTrackSender()` routes an installed processor's `processedTrack` to a newly registered secondary sender so a backup codec never publishes raw frames while the primary is processed. Processor install and teardown in all three classes roll the processed and raw sender track back, including `LocalVideoTrack`'s secondary simulcast senders, and aggregate every cleanup failure instead of discarding it.
13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`, `src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
`x-google-start-bitrate` was reachable only by AV1 and VP9 because it was gated twice. The publish path registered a track bitrate only for SVC codecs, and the offer munging returned early for everything else. H.264, H.265 and VP8 therefore opened at the Chromium default and had to ramp, which showed up as a 3000 kbps screen share encoding at 346 kbps twenty seconds in. The bitrate is now registered for every video codec from the highest encoding (`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and the offer munging applies it whenever a max bitrate is known. The dependency descriptor extension stays SVC-only. `appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and `setTrackCodecBitrate()` replaces an entry for the same cid or transceiver instead of appending, since `trackBitrates` is never cleared.
## Updating from upstream
@@ -3,12 +3,43 @@
// SPDX-License-Identifier: Apache-2.0
import type {MediaDescription} from 'sdp-transform';
import {describe, expect, it} from 'vitest';
import {appendStartBitrateToFmtp} from './PCTransport.ts';
import type {TrackBitrateInfo} from './PCTransport.ts';
import {appendStartBitrateToFmtp, collectStereoMids, ensureAudioNackAndStereo, ensureOpusFmtp} from './PCTransport.ts';
function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {
return {fmtp: entries} as unknown as MediaDescription;
}
function opusMedia(config: string, mid = '0'): MediaDescription {
return {
type: 'audio',
mid,
port: 9,
protocol: 'UDP/TLS/RTP/SAVPF',
rtp: [{payload: 109, codec: 'opus', rate: 48000, encoding: 2}],
fmtp: [{payload: 109, config}],
} as unknown as MediaDescription;
}
function offerMedia(mid: string, trackId: string): MediaDescription {
const media = opusMedia('useinbandfec=1', mid);
media.msid = `- ${trackId}`;
return media;
}
function audioBitrateInfo(mid: string | null, trackId: string, stereo: boolean): TrackBitrateInfo {
return {
transceiver: {mid, sender: {track: {id: trackId}}} as unknown as RTCRtpTransceiver,
codec: 'opus',
maxbr: 320,
stereo,
};
}
function opusConfig(media: MediaDescription): string {
return media.fmtp.find((fmtp) => fmtp.payload === 109)?.config ?? '';
}
describe('appendStartBitrateToFmtp', () => {
it('appends the start bitrate to a non-SVC codec fmtp line', () => {
const media = mediaWithFmtp([
@@ -44,3 +75,72 @@ describe('appendStartBitrateToFmtp', () => {
expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
});
});
describe('ensureOpusFmtp', () => {
it('does not force stereo on a mono publication', () => {
const media = opusMedia('maxplaybackrate=48000;stereo=0;useinbandfec=1');
ensureOpusFmtp(media, 48000, false);
const config = opusConfig(media);
expect(config).toContain('minptime=10');
expect(config).toContain('useinbandfec=1');
expect(config).toContain('usedtx=0');
expect(config).toContain('maxaveragebitrate=48000');
expect(config).not.toContain('stereo=1');
});
it('keeps stereo for a stereo publication', () => {
const media = opusMedia('maxplaybackrate=48000;useinbandfec=1');
ensureOpusFmtp(media, 320000, true);
const config = opusConfig(media);
expect(config).toContain('stereo=1');
expect(config).toContain('sprop-stereo=1');
expect(config).toContain('maxaveragebitrate=320000');
});
it('preserves a stereo parameter the server negotiated', () => {
const media = opusMedia('minptime=10;stereo=1');
ensureOpusFmtp(media, 48000, false);
expect(opusConfig(media)).toContain('stereo=1');
});
});
describe('ensureAudioNackAndStereo', () => {
it('only stamps stereo on the listed mids', () => {
const mono = opusMedia('useinbandfec=1', '0');
ensureAudioNackAndStereo(mono as never, ['1'], []);
expect(opusConfig(mono)).not.toContain('stereo=1');
const stereo = opusMedia('useinbandfec=1', '1');
ensureAudioNackAndStereo(stereo as never, ['1'], []);
expect(opusConfig(stereo)).toContain('stereo=1');
expect(opusConfig(stereo)).toContain('sprop-stereo=1');
});
});
describe('collectStereoMids', () => {
it('matches the offer media section by msid before the transceiver has a mid', () => {
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
expect(collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media)).toEqual(['1']);
});
it('stamps stereo on the first offer for a new stereo publication', () => {
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
const stereoMids = collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media);
for (const m of media) {
ensureAudioNackAndStereo(m as never, stereoMids, []);
}
expect(opusConfig(media[0]!)).not.toContain('stereo=1');
expect(opusConfig(media[1]!)).toContain('stereo=1');
expect(opusConfig(media[1]!)).toContain('sprop-stereo=1');
});
it('uses the assigned mid once renegotiation has one', () => {
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
expect(collectStereoMids([audioBitrateInfo('1', 'screenshare-track', true)], media)).toEqual(['1']);
});
it('leaves mono publications out', () => {
const media = [offerMedia('0', 'mic-track')];
expect(collectStereoMids([audioBitrateInfo(null, 'mic-track', false)], media)).toEqual([]);
});
});
@@ -11,11 +11,12 @@ import {NegotiationError, UnexpectedConnectionState} from './errors.ts';
import type {LoggerOptions} from './types.ts';
import {ddExtensionURI, isFireFox, isSafari, isSVCCodec} from './utils.ts';
interface TrackBitrateInfo {
export interface TrackBitrateInfo {
cid?: string;
transceiver?: RTCRtpTransceiver;
codec: string;
maxbr: number;
stereo?: boolean;
}
const startBitrateFraction = 0.7;
@@ -25,8 +26,6 @@ const requiredOpusFmtpParameters = {
minptime: '10',
useinbandfec: '1',
usedtx: '0',
stereo: '1',
'sprop-stereo': '1',
};
const debounceInterval = 20;
export const PCEvents = {
@@ -156,20 +155,18 @@ export default class PCTransport extends EventEmitter {
sdpParsed.media.forEach((media) => {
const mid = getMidString(media.mid!);
if (media.type === 'audio') {
ensureOpusFmtp(media);
this.trackBitrates.some((trackbr): boolean => {
if (!trackbr.transceiver || mid !== trackbr.transceiver.mid) {
return false;
}
const codecPayload = getCodecPayload(media, trackbr.codec);
if (codecPayload === 0) {
return true;
}
if (trackbr.codec.toLowerCase() === 'opus') {
ensureOpusFmtp(media, trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps);
}
return true;
});
const trackbr = this.trackBitrates.find(
(br) => br.transceiver !== undefined && mid === br.transceiver.mid && br.codec.toLowerCase() === 'opus',
);
if (trackbr && getCodecPayload(media, trackbr.codec) !== 0) {
ensureOpusFmtp(
media,
trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps,
trackbr.stereo === true,
);
} else {
ensureOpusFmtp(media);
}
}
});
mungedSDP = write(sdpParsed);
@@ -240,10 +237,11 @@ export default class PCTransport extends EventEmitter {
const offer = await this.pc.createOffer(options);
this.log.debug('original offer', {sdp: offer.sdp, ...this.logContext});
const sdpParsed = parse(offer.sdp ?? '');
const stereoMids = collectStereoMids(this.trackBitrates, sdpParsed.media);
sdpParsed.media.forEach((media) => {
ensureIPAddrMatchVersion(media);
if (media.type === 'audio') {
ensureAudioNackAndStereo(media, ['all'], []);
ensureAudioNackAndStereo(media, stereoMids, []);
} else if (media.type === 'video') {
this.trackBitrates.some((trackbr): boolean => {
if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
@@ -587,7 +585,11 @@ function ensureAudioRedFmtp(media: MediaDescription, opusPayload: number): void
}
}
function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number = opusMaxAverageBitrateBps): number {
export function ensureOpusFmtp(
media: MediaDescription,
maxAverageBitrateBps: number = opusMaxAverageBitrateBps,
stereo = false,
): number {
const opusPayload = getCodecPayload(media, 'opus');
if (opusPayload <= 0) return 0;
media.ptime = opusPacketTimeMs;
@@ -596,6 +598,10 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
for (const [key, value] of Object.entries(requiredOpusFmtpParameters)) {
config = setFmtpParameter(config, key, value);
}
if (stereo) {
config = setFmtpParameter(config, 'stereo', '1');
config = setFmtpParameter(config, 'sprop-stereo', '1');
}
if (maxAverageBitrateBps > 0) {
config = setFmtpParameter(config, 'maxaveragebitrate', String(maxAverageBitrateBps));
}
@@ -604,18 +610,18 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
return opusPayload;
}
function ensureAudioNackAndStereo(
export function ensureAudioNackAndStereo(
media: {
type: string;
port: number;
protocol: string;
payloads?: string | undefined;
} & MediaDescription,
_stereoMids: Array<string>,
stereoMids: Array<string>,
nackMids: Array<string>,
) {
const mid = getMidString(media.mid!);
const opusPayload = ensureOpusFmtp(media);
const opusPayload = ensureOpusFmtp(media, opusMaxAverageBitrateBps, stereoMids.includes(mid));
if (opusPayload > 0) {
if (!media.rtcpFb) {
media.rtcpFb = [];
@@ -629,6 +635,32 @@ function ensureAudioNackAndStereo(
}
}
export function collectStereoMids(
trackBitrates: Array<TrackBitrateInfo>,
media: Array<MediaDescription>,
): Array<string> {
const stereoMids: Array<string> = [];
for (const trackbr of trackBitrates) {
if (trackbr.stereo !== true || !trackbr.transceiver) {
continue;
}
if (trackbr.transceiver.mid) {
stereoMids.push(getMidString(trackbr.transceiver.mid));
continue;
}
const trackId = trackbr.transceiver.sender.track?.id;
if (trackId === undefined) {
continue;
}
for (const m of media) {
if (m.type === 'audio' && m.mid !== undefined && m.msid?.includes(trackId)) {
stereoMids.push(getMidString(m.mid));
}
}
}
return stereoMids;
}
function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
stereoMids: Array<string>;
nackMids: Array<string>;
@@ -27,7 +27,59 @@ describe('selectPublisherCodecPreferences', () => {
const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f');
const rtx = codec('video/rtx');
const preferences = selectPublisherCodecPreferences('h264', [openH264, rtx, externalBaseline, highProfile]);
expect(preferences).toEqual([externalBaseline, highProfile, openH264, rtx]);
expect(preferences).toEqual([externalBaseline, openH264, highProfile, rtx]);
});
it('ranks Constrained Baseline above Main, High and Constrained High', () => {
const constrainedBaseline = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
);
const mainProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f');
const highProfileLevel31 = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f',
);
const highProfileLevel51 = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',
);
const constrainedHigh = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f',
);
const preferences = selectPublisherCodecPreferences('h264', [
mainProfile,
highProfileLevel31,
highProfileLevel51,
constrainedHigh,
constrainedBaseline,
]);
expect(preferences).toEqual([
constrainedBaseline,
mainProfile,
highProfileLevel31,
highProfileLevel51,
constrainedHigh,
]);
});
it('keeps Constrained Baseline packetization-mode=1 ahead of Constrained Baseline packetization-mode=0 and High', () => {
const constrainedBaselineMode0 = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',
);
const constrainedBaselineMode1 = codec(
'video/H264',
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
);
const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033');
const preferences = selectPublisherCodecPreferences('h264', [
highProfile,
constrainedBaselineMode0,
constrainedBaselineMode1,
]);
expect(preferences).toEqual([constrainedBaselineMode1, constrainedBaselineMode0, highProfile]);
});
it('keeps non-H.264 codecs in browser capability order and appends RTX', () => {
@@ -96,7 +96,7 @@ const videoCodecMimeTypes: Record<VideoCodec, Array<string>> = {
vp8: ['video/vp8'],
};
const h264OpenH264ProfileLevelId = '42e01f';
const h264PreferredHardwareProfileLevelIds = new Set(['42001f', '4d001f', '64001f']);
const h264PreferredHardwareProfileLevelIds = new Set(['42001f']);
type RtpCodecCapability = RTCRtpCapabilities['codecs'][number] & {sdpFmtpLine?: string};
enum PCState {
@@ -1656,8 +1656,8 @@ function getH264PublisherCodecScore(codec: RtpCodecCapability): number {
const packetizationMode = getFmtpParameter(codec.sdpFmtpLine, 'packetization-mode');
const packetizationScore = packetizationMode === '1' ? 0 : 1;
if (profileLevelId && h264PreferredHardwareProfileLevelIds.has(profileLevelId)) return packetizationScore;
if (profileLevelId && profileLevelId !== h264OpenH264ProfileLevelId) return 10 + packetizationScore;
if (profileLevelId === h264OpenH264ProfileLevelId) return 20 + packetizationScore;
if (profileLevelId === h264OpenH264ProfileLevelId) return 10 + packetizationScore;
if (profileLevelId) return 20 + packetizationScore;
return 30 + packetizationScore;
}
@@ -1040,6 +1040,7 @@ export default class LocalParticipant extends Participant {
transceiver: trackTransceiver,
codec: 'opus',
maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
stereo: isStereo,
});
}
} else if (track.codec) {
@@ -25,7 +25,6 @@ import * as ModalCommands from '@app/features/ui/commands/ModalCommands';
import {modal} from '@app/features/ui/commands/ModalCommands';
import * as PopoutCommands from '@app/features/ui/commands/PopoutCommands';
import * as TextCopyCommands from '@app/features/ui/commands/TextCopyCommands';
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
import {MockAvatar} from '@app/features/ui/components/MockAvatar';
import {StatusIndicator} from '@app/features/ui/components/StatusIndicator';
import FocusRing from '@app/features/ui/focus_ring/FocusRing';
@@ -52,9 +51,8 @@ import Users from '@app/features/user/state/Users';
import * as NicknameUtils from '@app/features/user/utils/NicknameUtils';
import * as ProfileDisplayUtils from '@app/features/user/utils/ProfileDisplayUtils';
import {createMockProfile} from '@app/features/user/utils/ProfileUtils';
import {COPIED_STATS_JSON_DESCRIPTOR} from '@app/features/voice/components/StatsForNerdsCopyDescriptors';
import {copyVoiceDiagnostics} from '@app/features/voice/commands/VoiceDiagnosticsCommands';
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import {buildStatsForNerdsCopyPayload, collectStatsForNerdsSnapshot} from '@app/features/voice/utils/StatsForNerdsCopy';
import {MEDIA_PROXY_PROFILE_BANNER_SIZE_POPOUT} from '@fluxer/constants/src/MediaProxyAssetSizes';
import {StatusTypes} from '@fluxer/constants/src/StatusConstants';
import type {MessageDescriptor} from '@lingui/core';
@@ -462,12 +460,7 @@ export const UserAreaPopout = observer(() => {
});
}, [i18n]);
const handleCopyStats = useCallback(() => {
const data = collectStatsForNerdsSnapshot();
void buildStatsForNerdsCopyPayload(data).then((payload) => {
void navigator.clipboard.writeText(JSON.stringify(payload, null, 2)).then(() => {
ToastCommands.createToast({type: 'success', children: i18n._(COPIED_STATS_JSON_DESCRIPTOR)});
});
});
void copyVoiceDiagnostics(i18n);
}, [i18n]);
const handleCopyUserTag = useCallback(() => {
if (!currentUser) {
@@ -1277,6 +1277,7 @@ const SKELETON_NAGBAR_ROW_SHAPES: Record<NagbarType, SkeletonNagbarRowShape> = {
[NagbarType.PREMIUM_ONBOARDING]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.GIFT_INVENTORY]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.DESKTOP_DOWNLOAD]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.DESKTOP_UPDATE_READY]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.GUILD_MEMBERSHIP_CTA]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.VISIONARY_MFA]: {tone: SkeletonNagbarTone.BRAND, hasActions: true},
[NagbarType.VOICE_SESSION_RESTORE]: {tone: SkeletonNagbarTone.VOICE, hasActions: true},
@@ -11,6 +11,7 @@ import Config from '@app/features/app/config/Config';
import {isClientReconnecting} from '@app/features/app/state/ClientReadiness';
import Initialization from '@app/features/app/state/Initialization';
import RuntimeConfig from '@app/features/app/state/RuntimeConfig';
import Updater from '@app/features/app/state/Updater';
import Authentication from '@app/features/auth/state/Authentication';
import Channels from '@app/features/channel/state/Channels';
import DeveloperOptions from '@app/features/devtools/state/DeveloperOptions';
@@ -230,6 +231,7 @@ export const useNagbarConditions = (): NagbarConditions => {
const canShowLinuxInputAccess = NativePermission.shouldShowLinuxInputAccessNagbar;
const canShowSoftwareEncoder = SoftwareEncoderWarning.showWarning;
const canShowStreamerMode = StreamerMode.shouldShowNagbar;
const canShowDesktopUpdateReady = Updater.shouldShowUpdateReadyNagbar;
const canShowBuildEnvironment =
!BUILD_ENVIRONMENT_HIDDEN_RELEASE_CHANNELS.has(Config.PUBLIC_RELEASE_CHANNEL) &&
!nagbarState.buildEnvironmentDismissedThisSession;
@@ -288,6 +290,7 @@ export const useNagbarConditions = (): NagbarConditions => {
canShowLinuxInputAccess,
canShowSoftwareEncoder,
canShowStreamerMode,
canShowDesktopUpdateReady,
};
};
export const useActiveNagbars = (conditions: NagbarConditions): Array<NagbarState> => {
@@ -407,6 +410,12 @@ export const useActiveNagbars = (conditions: NagbarConditions): Array<NagbarStat
visible: conditions.canShowStreamerMode,
dismissible: true,
},
{
type: NagbarType.DESKTOP_UPDATE_READY,
priority: -1.5,
visible: conditions.canShowDesktopUpdateReady,
dismissible: true,
},
];
return selectVisibleNagbars(nagbars);
}, [conditions]);
@@ -15,6 +15,7 @@ export const NagbarType = {
PREMIUM_ONBOARDING: 'premium-onboarding',
GIFT_INVENTORY: 'gift-inventory',
DESKTOP_DOWNLOAD: 'desktop-download',
DESKTOP_UPDATE_READY: 'desktop-update-ready',
GUILD_MEMBERSHIP_CTA: 'guild-membership-cta',
VISIONARY_MFA: 'visionary-mfa',
VOICE_SESSION_RESTORE: 'voice-session-restore',
@@ -50,6 +51,7 @@ export interface NagbarConditions {
canShowPremiumOnboarding: boolean;
canShowGiftInventory: boolean;
canShowDesktopDownload: boolean;
canShowDesktopUpdateReady: boolean;
canShowGuildMembershipCta: boolean;
canShowVisionaryMfa: boolean;
canShowVoiceSessionRestore: boolean;
@@ -7,6 +7,7 @@ import {ConnectionNagbar} from '@app/features/app/components/layout/app_layout/n
import {CorruptedInstallationNagbar} from '@app/features/app/components/layout/app_layout/nagbars/CorruptedInstallationNagbar';
import {DesktopDownloadNagbar} from '@app/features/app/components/layout/app_layout/nagbars/DesktopDownloadNagbar';
import {DesktopNotificationNagbar} from '@app/features/app/components/layout/app_layout/nagbars/DesktopNotificationNagbar';
import {DesktopUpdateReadyNagbar} from '@app/features/app/components/layout/app_layout/nagbars/DesktopUpdateReadyNagbar';
import {EmailVerificationNagbar} from '@app/features/app/components/layout/app_layout/nagbars/EmailVerificationNagbar';
import {GiftInventoryNagbar} from '@app/features/app/components/layout/app_layout/nagbars/GiftInventoryNagbar';
import {GuildMembershipCtaNagbar} from '@app/features/app/components/layout/app_layout/nagbars/GuildMembershipCtaNagbar';
@@ -153,6 +154,14 @@ export const NagbarContainer: React.FC<NagbarContainerProps> = observer(({nagbar
data-flx="app.app-layout.nagbar-container.desktop-download-nagbar"
/>
);
case NagbarType.DESKTOP_UPDATE_READY:
return (
<DesktopUpdateReadyNagbar
key={nagbar.type}
isMobile={mobileLayout.enabled}
data-flx="app.app-layout.nagbar-container.desktop-update-ready-nagbar"
/>
);
case NagbarType.GUILD_MEMBERSHIP_CTA:
return (
<GuildMembershipCtaNagbar
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Nagbar} from '@app/features/app/components/layout/Nagbar';
import {NagbarButton} from '@app/features/app/components/layout/NagbarButton';
import {NagbarContent} from '@app/features/app/components/layout/NagbarContent';
import {NAGBAR_TONES, NagbarToneKind} from '@app/features/app/components/layout/NagbarTones';
import {PRODUCT_NAME} from '@app/features/app/config/I18nDisplayConstants';
import Updater from '@app/features/app/state/Updater';
import {
DESKTOP_VERSION_HAS_BEEN_DOWNLOADED_DESCRIPTOR,
RESTART_FLUXER_DESCRIPTOR,
THE_DESKTOP_UPDATE_HAS_BEEN_DOWNLOADED_DESCRIPTOR,
} from '@app/features/updater/commands/UpdaterModalCommands';
import {useLingui} from '@lingui/react/macro';
import {observer} from 'mobx-react-lite';
export const DesktopUpdateReadyNagbar = observer(({isMobile}: {isMobile: boolean}) => {
const {i18n} = useLingui();
const version = Updater.updateInfo.native.version;
return (
<Nagbar
isMobile={isMobile}
backgroundColor={NAGBAR_TONES[NagbarToneKind.BRAND].backgroundColor}
textColor={NAGBAR_TONES[NagbarToneKind.BRAND].textColor}
dismissible
onDismiss={Updater.dismissUpdateReadyNagbar}
data-flx="app.app-layout.nagbars.desktop-update-ready-nagbar.nagbar"
>
<NagbarContent
isMobile={isMobile}
onDismiss={Updater.dismissUpdateReadyNagbar}
message={
version
? i18n._(DESKTOP_VERSION_HAS_BEEN_DOWNLOADED_DESCRIPTOR, {version, productName: PRODUCT_NAME})
: i18n._(THE_DESKTOP_UPDATE_HAS_BEEN_DOWNLOADED_DESCRIPTOR, {productName: PRODUCT_NAME})
}
actions={
<NagbarButton
isMobile={isMobile}
onClick={() => void Updater.applyUpdate()}
data-flx="app.app-layout.nagbars.desktop-update-ready-nagbar.restart-button"
>
{i18n._(RESTART_FLUXER_DESCRIPTOR, {productName: PRODUCT_NAME})}
</NagbarButton>
}
data-flx="app.app-layout.nagbars.desktop-update-ready-nagbar.nagbar-content"
/>
</Nagbar>
);
});
@@ -0,0 +1,112 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {installVoiceMenuTestBootstrap} from '@app/features/ui/action_menu/items/__fixtures__/VoiceMenuTestBootstrap';
import type {UpdaterEvent} from '@app/types/electron.d';
import {afterEach, describe, expect, test, vi} from 'vitest';
const {pushUpdateReadyModal} = vi.hoisted(() => ({pushUpdateReadyModal: vi.fn()}));
vi.mock('@app/features/updater/commands/UpdaterModalCommands', () => ({
pushDesktopUpdateDownloadFailedModal: vi.fn(),
pushDesktopUpdateInstallFailedModal: vi.fn(),
pushManualUpdateAvailableModal: vi.fn(),
pushUnsupportedUpdateModal: vi.fn(),
pushUpdateAvailableModal: vi.fn(),
pushUpdateCheckFailedModal: vi.fn(),
pushUpdateReadyModal,
pushUpToDateModal: vi.fn(),
}));
vi.mock('@lingui/core/macro', () => ({
msg: (descriptor: {message: string}) => descriptor,
}));
vi.mock('@app/features/platform/utils/ClientInfo', () => ({
getClientInfo: () =>
Promise.resolve({desktopVersion: '1.0.0', desktopChannel: 'canary', desktopArch: 'x64', arch: 'x64'}),
}));
installVoiceMenuTestBootstrap();
let nativeEventListener: ((event: UpdaterEvent) => void) | null = null;
let onUpdaterCheck: (() => void) | null = null;
let loadedUpdater: {dispose: () => void} | null = null;
function installElectronApi(): void {
nativeEventListener = null;
onUpdaterCheck = null;
(window as unknown as {electron: unknown}).electron = {
platform: 'win32',
buildChannel: 'canary',
onUpdaterEvent: (listener: (event: UpdaterEvent) => void) => {
nativeEventListener = listener;
return () => {
nativeEventListener = null;
};
},
updaterCheck: () => {
onUpdaterCheck?.();
return Promise.resolve();
},
updaterDownload: () => Promise.resolve(),
updaterInstall: () => Promise.resolve(),
};
}
function emit(event: UpdaterEvent): void {
if (!nativeEventListener) throw new Error('Updater never subscribed to native updater events');
nativeEventListener(event);
}
async function loadUpdater() {
vi.resetModules();
installElectronApi();
const {default: Updater} = await import('@app/features/app/state/Updater');
loadedUpdater = Updater;
await vi.waitFor(() => {
expect(nativeEventListener).not.toBeNull();
expect(Updater.lastCheckedAt).not.toBeNull();
});
pushUpdateReadyModal.mockClear();
return Updater;
}
function emitUserDownloadCompletion(version: string): void {
emit({type: 'available', context: 'user', version, downloadSize: 1000, downloadStarted: true});
emit({type: 'downloaded', context: 'user', version});
}
afterEach(() => {
loadedUpdater?.dispose();
loadedUpdater = null;
});
describe('updater update-ready surface', () => {
test('does not push a blocking modal when a user-initiated download finishes outside a check', async () => {
const Updater = await loadUpdater();
emitUserDownloadCompletion('2.0.0');
expect(Updater.nativeUpdateReady).toBe(true);
expect(pushUpdateReadyModal).not.toHaveBeenCalled();
});
test('announces the ready update through a dismissible nagbar instead', async () => {
const Updater = await loadUpdater();
expect(Updater.shouldShowUpdateReadyNagbar).toBe(false);
emitUserDownloadCompletion('2.0.0');
expect(Updater.shouldShowUpdateReadyNagbar).toBe(true);
Updater.dismissUpdateReadyNagbar();
expect(Updater.shouldShowUpdateReadyNagbar).toBe(false);
emit({type: 'downloaded', context: 'background', version: '2.1.0'});
expect(Updater.shouldShowUpdateReadyNagbar).toBe(true);
});
test('still answers a user-initiated check with the update ready modal', async () => {
const Updater = await loadUpdater();
emitUserDownloadCompletion('2.0.0');
pushUpdateReadyModal.mockClear();
onUpdaterCheck = () => emit({type: 'available', context: 'user', version: '2.0.0', downloadStarted: false});
await Updater.checkForUpdates(true, true);
expect(pushUpdateReadyModal).toHaveBeenCalledTimes(1);
});
});
+17 -3
View File
@@ -131,6 +131,7 @@ class Updater {
private backgroundCheckInterval: number | null = null;
private backgroundCheckCleanups: Array<() => void> = [];
private unsubscribeNativeEvents: (() => void) | null = null;
private updateReadyNagbarDismissedVersion: string | null = null;
constructor() {
makeAutoObservable(this, {}, {autoBind: true});
@@ -193,6 +194,18 @@ class Updater {
return this.updateInfo.native.available && this.updateInfo.native.downloaded;
}
get shouldShowUpdateReadyNagbar(): boolean {
return (
this.nativeUpdateReady &&
!this.updateInfo.native.installing &&
this.updateReadyNagbarDismissedVersion !== this.updateReadyNagbarVersionKey
);
}
private get updateReadyNagbarVersionKey(): string {
return this.updateInfo.native.version ?? 'unknown';
}
get nativeDownloadInFlight(): boolean {
return this.updateInfo.native.downloading && !this.updateInfo.native.downloaded;
}
@@ -348,9 +361,6 @@ class Updater {
break;
}
this.transition({type: 'native.downloaded', version: event.version ?? null});
if (shouldShowImmediateUserResult) {
this.showCurrentUpdateState();
}
break;
case 'progress':
if (!shouldSurfaceNativeDesktopUpdate || !this.nativeDownloadProgressSupported) {
@@ -661,6 +671,10 @@ class Updater {
}
}
dismissUpdateReadyNagbar(): void {
this.updateReadyNagbarDismissedVersion = this.updateReadyNagbarVersionKey;
}
reset(): void {
this.transition({type: 'reset'});
}
@@ -99,9 +99,10 @@ export const DiscoveryPage = observer(function DiscoveryPage() {
return Math.min(SKELETON_DISCOVERY_MAX_COLUMNS, Math.max(1, columnsThatFit));
}, [containerWidth, zoomLevel]);
const guilds = Discovery.guilds;
const loadedCount = Discovery.loadedCount;
const searchActive = Discovery.query.length > 0;
const rowCount = columns > 0 ? Math.ceil(guilds.length / columns) : 0;
const hasMore = guilds.length < Discovery.total;
const hasMore = loadedCount < Discovery.total;
const virtualizer = useVirtualizer({
count: rowCount,
getScrollElement: () => scrollerRef.current?.getViewportElement() ?? null,
@@ -114,10 +115,10 @@ export const DiscoveryPage = observer(function DiscoveryPage() {
return;
}
void Discovery.search({
offset: guilds.length,
offset: loadedCount,
limit: PAGE_SIZE,
});
}, [guilds.length, hasMore]);
}, [loadedCount, hasMore]);
useEffect(() => {
const items = virtualizer.getVirtualItems();
const lastItem = items[items.length - 1];
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {DiscoveryGuild, DiscoverySearchResponse} from '@app/features/discovery/commands/DiscoveryCommands';
import * as DiscoveryCommands from '@app/features/discovery/commands/DiscoveryCommands';
import {beforeEach, describe, expect, test, vi} from 'vitest';
import Discovery from './Discovery';
vi.mock('@app/features/discovery/commands/DiscoveryCommands', () => ({
searchGuilds: vi.fn(),
getCategories: vi.fn(),
}));
function guild(id: string, memberCount: number): DiscoveryGuild {
return {
id,
name: `Guild ${id}`,
icon: null,
banner: null,
description: null,
category_type: 0,
primary_language: null,
custom_tags: [],
member_count: memberCount,
online_count: 0,
features: [],
verification_level: 0,
};
}
function page(guilds: Array<DiscoveryGuild>, total: number): DiscoverySearchResponse {
return {guilds, total, categoryCounts: null};
}
describe('Discovery.search', () => {
beforeEach(() => {
vi.mocked(DiscoveryCommands.searchGuilds).mockReset();
Discovery.reset();
});
test('does not append a guild that a previous page already returned', async () => {
vi.mocked(DiscoveryCommands.searchGuilds)
.mockResolvedValueOnce(page([guild('1', 50), guild('2', 40)], 4))
.mockResolvedValueOnce(page([guild('2', 40), guild('3', 30)], 4));
await Discovery.search({limit: 2, offset: 0});
await Discovery.search({limit: 2, offset: Discovery.loadedCount});
expect(Discovery.guilds.map((entry) => entry.id)).toEqual(['1', '2', '3']);
});
test('advances the pagination offset by what the server returned, not by what survived deduping', async () => {
vi.mocked(DiscoveryCommands.searchGuilds)
.mockResolvedValueOnce(page([guild('1', 50), guild('2', 40)], 4))
.mockResolvedValueOnce(page([guild('2', 40), guild('3', 30)], 4));
await Discovery.search({limit: 2, offset: 0});
await Discovery.search({limit: 2, offset: Discovery.loadedCount});
expect(Discovery.loadedCount).toBe(4);
});
});
@@ -8,6 +8,7 @@ const DEFAULT_DISCOVERY_PAGE_SIZE = 36;
class Discovery {
guilds: Array<DiscoveryGuild> = [];
loadedCount = 0;
total = 0;
loading = false;
error = false;
@@ -53,6 +54,7 @@ class Discovery {
this.error = false;
if (offset === 0 && searchModeChanged) {
this.guilds = [];
this.loadedCount = 0;
this.total = 0;
}
this.query = query;
@@ -78,8 +80,10 @@ class Discovery {
if (offset === 0) {
this.guilds = result.guilds;
} else {
this.guilds = [...this.guilds, ...result.guilds];
const seen = new Set(this.guilds.map((guild) => guild.id));
this.guilds = [...this.guilds, ...result.guilds.filter((guild) => !seen.has(guild.id))];
}
this.loadedCount = offset + result.guilds.length;
this.total = result.total;
this.categoryCounts = result.categoryCounts;
this.loading = false;
@@ -136,6 +140,7 @@ class Discovery {
reset(): void {
this.activeSearchToken += 1;
this.guilds = [];
this.loadedCount = 0;
this.total = 0;
this.categoryCounts = null;
this.loading = false;
@@ -0,0 +1,113 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {installVoiceMenuTestBootstrap} from '@app/features/ui/action_menu/items/__fixtures__/VoiceMenuTestBootstrap';
import type {GuildMemberData} from '@fluxer/schema/src/domains/guild/GuildMemberSchemas';
import type {Guild} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
import {describe, expect, it, vi} from 'vitest';
vi.mock('@lingui/core/macro', () => {
const descriptor = (value: unknown): unknown => (typeof value === 'string' ? {message: value} : value);
return {msg: descriptor, t: descriptor, plural: () => '', select: () => '', selectOrdinal: () => ''};
});
vi.mock('@app/features/gateway/transport/GatewayConnection', () => ({default: {socket: null}}));
vi.mock('@app/features/channel/state/Channels', () => ({default: {getChannel: () => null}}));
vi.mock('@app/features/guild/state/Guilds', () => ({
default: {
getGuild: (guildId: string) => ({
id: guildId,
disabledOperations: 0,
roles: {
'10': {id: '10', permissions: 0n, position: 0},
'20': {id: '20', permissions: 0n, position: 5},
'30': {id: '30', permissions: 0n, position: 9},
},
}),
},
}));
installVoiceMenuTestBootstrap();
const {canManageTargetUser} = await import('@app/features/permissions/utils/PermissionUtils');
const {default: GuildMembers} = await import('@app/features/member/state/GuildMembers');
const {default: MemberSidebar} = await import('@app/features/member/state/MemberSidebar');
const MODERATOR_ROLE = {id: '20', permissions: 0n, position: 5};
let nextGuildId = 0;
function makeMember(userId: string, roles: Array<string>, nick: string | null = null): GuildMemberData {
return {
user: {
id: userId,
username: `user-${userId}`,
discriminator: '0001',
global_name: null,
avatar: null,
avatar_color: null,
flags: 0,
},
nick,
roles,
joined_at: '2026-01-01T00:00:00.000Z',
};
}
function makeGuild(guildId: string): Guild {
return {id: guildId, owner_id: 'owner'} as unknown as Guild;
}
function syncMemberList(guildId: string, channelId: string, members: Array<GuildMemberData>): void {
MemberSidebar.subscribeToChannel(guildId, channelId, [[0, 99]]);
MemberSidebar.handleListUpdate({
guildId,
listId: 'everyone',
channelId,
memberCount: members.length,
onlineCount: members.length,
groups: [{id: 'online', count: members.length}],
ops: [
{
op: 'SYNC',
range: [0, members.length],
items: [{group: {id: 'online', count: members.length}}, ...members.map((member) => ({member}))],
},
],
});
}
function setupGuild(members: Array<GuildMemberData>): string {
const guildId = `guild-${++nextGuildId}`;
syncMemberList(guildId, `channel-${guildId}`, members);
return guildId;
}
describe('MemberSidebar member list hydration', () => {
it('makes membership known for members that only ever appeared in the member list', () => {
const guildId = setupGuild([makeMember('target-low', []), makeMember('target-high', ['30'])]);
expect(GuildMembers.isMembershipKnown(guildId, 'target-low')).toBe(true);
expect(GuildMembers.getMember(guildId, 'target-high')?.roles.has('30')).toBe(true);
});
it('lets the role hierarchy check pass for a sidebar member who has never posted', () => {
const guildId = setupGuild([makeMember('target-low', [])]);
expect(canManageTargetUser(makeGuild(guildId), 'me', MODERATOR_ROLE, 'target-low')).toBe(true);
});
it('still refuses a sidebar member who outranks you', () => {
const guildId = setupGuild([makeMember('target-high', ['30'])]);
expect(canManageTargetUser(makeGuild(guildId), 'me', MODERATOR_ROLE, 'target-high')).toBe(false);
});
it('does not create members for group header rows', () => {
const guildId = setupGuild([makeMember('target-low', []), makeMember('target-high', ['30'])]);
expect(GuildMembers.getMemberCount(guildId)).toBe(2);
});
it('leaves an already known member untouched', () => {
const guildId = `guild-${++nextGuildId}`;
GuildMembers.hydrateIfMissing(guildId, makeMember('target-low', ['20'], 'authoritative'));
syncMemberList(guildId, `channel-${guildId}`, [makeMember('target-low', [], 'stale')]);
expect(GuildMembers.getMember(guildId, 'target-low')?.nick).toBe('authoritative');
});
});
@@ -5,6 +5,7 @@ import Channels from '@app/features/channel/state/Channels';
import GatewayConnection from '@app/features/gateway/transport/GatewayConnection';
import Guilds from '@app/features/guild/state/Guilds';
import {GuildMember} from '@app/features/member/models/GuildMember';
import GuildMembers from '@app/features/member/state/GuildMembers';
import {getHydratedMemberListRangesFromNormalized} from '@app/features/member/utils/MemberListHydration';
import {deriveMemberListIdentity} from '@app/features/member/utils/MemberListIdentity';
import {
@@ -695,6 +696,7 @@ class MemberSidebar {
}
userIdRowCounts.set(userId, (userIdRowCounts.get(userId) ?? 0) + 1);
newMembersByUserId.set(userId, member);
GuildMembers.hydrateIfMissing(guildId, member);
const memberItem = this.convertItem(guildId, row);
if (memberItem) {
newItems.set(rowIndex, memberItem);
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, test, vi} from 'vitest';
let nativeMacOS = false;
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
isNativeMacOS: () => nativeMacOS,
}));
const {isTextInputKeyEvent} = await import('@app/features/platform/utils/IsTextInputKeyEvent');
const keyEvent = (init: Partial<KeyboardEvent>): KeyboardEvent =>
({key: '', ctrlKey: false, metaKey: false, altKey: false, shiftKey: false, ...init}) as KeyboardEvent;
beforeEach(() => {
nativeMacOS = false;
});
describe('isTextInputKeyEvent', () => {
test('Alt+digit is a shortcut chord, not typing, off macOS', () => {
for (const key of ['1', '2', '3', '4', '5', '6', '7', '8', '9']) {
expect(isTextInputKeyEvent(keyEvent({key, altKey: true}))).toBe(false);
}
});
test('Alt+letter is a shortcut chord, not typing, off macOS', () => {
expect(isTextInputKeyEvent(keyEvent({key: 'a', altKey: true}))).toBe(false);
});
test('Option-composed characters stay typing on macOS', () => {
nativeMacOS = true;
expect(isTextInputKeyEvent(keyEvent({key: '¡', altKey: true}))).toBe(true);
expect(isTextInputKeyEvent(keyEvent({key: '@', altKey: true}))).toBe(true);
});
test('a bare printable key is still typing', () => {
expect(isTextInputKeyEvent(keyEvent({key: 'a'}))).toBe(true);
expect(isTextInputKeyEvent(keyEvent({key: '1'}))).toBe(true);
});
test('ctrl and meta chords and named keys are still not typing', () => {
expect(isTextInputKeyEvent(keyEvent({key: '1', ctrlKey: true}))).toBe(false);
expect(isTextInputKeyEvent(keyEvent({key: '1', metaKey: true}))).toBe(false);
expect(isTextInputKeyEvent(keyEvent({key: 'ArrowUp'}))).toBe(false);
expect(isTextInputKeyEvent(keyEvent({key: 'ArrowUp', altKey: true}))).toBe(false);
});
test('a dead key still counts as typing while Alt is held', () => {
expect(isTextInputKeyEvent(keyEvent({key: 'Dead', altKey: true}))).toBe(true);
});
});
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {isNativeMacOS} from '@app/features/ui/utils/NativeUtils';
export function isTextInputKeyEvent(event: KeyboardEvent): boolean {
const {key, ctrlKey, metaKey} = event;
const {key, ctrlKey, metaKey, altKey} = event;
if (!key || key === 'Unidentified') {
return false;
}
@@ -11,6 +13,9 @@ export function isTextInputKeyEvent(event: KeyboardEvent): boolean {
if (key === 'Dead') {
return true;
}
if (altKey && !isNativeMacOS()) {
return false;
}
if (key.length > 1 && NAMED_KEY_PATTERN.test(key)) {
return false;
}
@@ -4,7 +4,7 @@
import {installVoiceMenuTestBootstrap} from '@app/features/ui/action_menu/items/__fixtures__/VoiceMenuTestBootstrap';
import type {VoiceParticipantMenuScreenShareSource} from '@app/features/ui/action_menu/items/VoiceParticipantMenuTypes';
import type {I18n} from '@lingui/core';
import {expect, test, vi} from 'vitest';
import {beforeEach, expect, test, vi} from 'vitest';
vi.mock('@lingui/core/macro', () => {
const descriptor = (value: unknown): unknown => (typeof value === 'string' ? {message: value} : value);
@@ -38,6 +38,9 @@ vi.mock('@app/features/voice/state/StreamAudioPrefs', () => ({
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {showMyOwnScreenShare: false, pauseOwnScreenSharePreviewOnUnfocus: false},
}));
vi.mock('@app/features/voice/commands/VoiceDiagnosticsCommands', () => ({
copyVoiceDiagnostics: vi.fn(async () => undefined),
}));
vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({
update: vi.fn(),
}));
@@ -47,6 +50,7 @@ installVoiceMenuTestBootstrap();
const {buildVoiceParticipantStreamMenu} = await import(
'@app/features/ui/action_menu/items/VoiceParticipantStreamMenuBuilder'
);
const {copyVoiceDiagnostics} = await import('@app/features/voice/commands/VoiceDiagnosticsCommands');
const i18n = {
locale: 'en',
@@ -56,6 +60,7 @@ const i18n = {
interface MenuLeaf {
label?: string;
items?: Array<MenuLeaf>;
onClick?: () => void;
}
function streamMenu(source: VoiceParticipantMenuScreenShareSource): Array<{items: Array<MenuLeaf>}> {
@@ -99,18 +104,44 @@ const WATCHED_REMOTE_STREAM_SOURCE: VoiceParticipantMenuScreenShareSource = {
state: {kind: 'remote-watched', hasAudio: true, onStopWatching: () => undefined},
};
const UNWATCHED_REMOTE_STREAM_SOURCE: VoiceParticipantMenuScreenShareSource = {
kind: 'screen-share',
streamKey: 'stream-key',
state: {kind: 'remote-unwatched', onWatch: () => undefined},
};
beforeEach(() => {
vi.mocked(copyVoiceDiagnostics).mockClear();
});
test('own stream keeps a More options submenu with the screen-share preferences', () => {
const groups = streamMenu(OWN_STREAM_SOURCE);
const moreOptions = findLeaf(groups, 'More options');
expect(moreOptions).not.toBeNull();
expect(findLeaf(groups, 'Show my screen share')).not.toBeNull();
expect(findLeaf(groups, 'Copy stats JSON')).not.toBeNull();
expect(findLeaf(groups, 'Report Problem')).toBeNull();
});
test('remote watched stream omits the now-empty More options submenu and keeps audio controls', () => {
test('remote watched stream keeps a More options submenu with the diagnostics entry and audio controls', () => {
const groups = streamMenu(WATCHED_REMOTE_STREAM_SOURCE);
expect(findLeaf(groups, 'More options')).toBeNull();
expect(findLeaf(groups, 'More options')).not.toBeNull();
expect(findLeaf(groups, 'Copy stats JSON')).not.toBeNull();
expect(findLeaf(groups, 'Mute')).not.toBeNull();
expect(findLeaf(groups, 'Stream volume')).not.toBeNull();
expect(findLeaf(groups, 'Report Problem')).toBeNull();
});
test('remote unwatched stream keeps the diagnostics entry', () => {
const groups = streamMenu(UNWATCHED_REMOTE_STREAM_SOURCE);
expect(findLeaf(groups, 'More options')).not.toBeNull();
expect(findLeaf(groups, 'Copy stats JSON')).not.toBeNull();
});
test('the diagnostics entry on a remote stream copies voice diagnostics', () => {
const groups = streamMenu(WATCHED_REMOTE_STREAM_SOURCE);
const copyStats = findLeaf(groups, 'Copy stats JSON');
expect(copyStats).not.toBeNull();
copyStats?.onClick?.();
expect(vi.mocked(copyVoiceDiagnostics)).toHaveBeenCalledTimes(1);
});
@@ -30,9 +30,11 @@ import type {
MenuSliderType,
MenuSubmenuItemType,
} from '@app/features/ui/menu_bottom_sheet/MenuBottomSheet';
import {copyVoiceDiagnostics} from '@app/features/voice/commands/VoiceDiagnosticsCommands';
import * as VoiceSettingsCommands from '@app/features/voice/commands/VoiceSettingsCommands';
import {changeActiveScreenShare, stopActiveScreenShare} from '@app/features/voice/components/ActiveScreenShareMenu';
import {openScreenSharePreviewPrivacyModal} from '@app/features/voice/components/modals/ScreenSharePickerModal';
import {COPY_STATS_JSON_DESCRIPTOR} from '@app/features/voice/components/StatsForNerdsCopyDescriptors';
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import PopoutWindowManager, {isVoicePopoutSupported} from '@app/features/voice/state/PopoutWindowManager';
@@ -211,28 +213,39 @@ function buildStreamPopoutAction(options: VoiceParticipantStreamMenuBuilderOptio
function buildStreamMoreOptions(options: VoiceParticipantStreamMenuBuilderOptions): MenuSubmenuItemType | null {
const {i18n, guildId, source, showMyOwnScreenShare, pauseOwnScreenSharePreviewOnUnfocus, onClose} = options;
if (source.state.kind !== 'own') return null;
const items: Array<MenuItemType | MenuCheckboxType> = [
{
label: i18n._(SHOW_MY_SCREEN_SHARE_DESCRIPTOR),
checked: showMyOwnScreenShare,
onChange: (checked: boolean) => VoiceSettingsCommands.update({showMyOwnScreenShare: checked}),
},
{
label: i18n._(PAUSE_OWN_STREAM_PREVIEW_DESCRIPTOR),
checked: pauseOwnScreenSharePreviewOnUnfocus,
onChange: (checked: boolean) => VoiceSettingsCommands.update({pauseOwnScreenSharePreviewOnUnfocus: checked}),
},
];
if (guildId === undefined) {
items.push({
label: i18n._(SCREEN_SHARE_PRIVACY_DESCRIPTOR),
onClick: () => {
onClose();
openScreenSharePreviewPrivacyModal();
const items: Array<MenuItemType | MenuCheckboxType> = [];
if (source.state.kind === 'own') {
items.push(
{
label: i18n._(SHOW_MY_SCREEN_SHARE_DESCRIPTOR),
checked: showMyOwnScreenShare,
onChange: (checked: boolean) => VoiceSettingsCommands.update({showMyOwnScreenShare: checked}),
},
});
{
label: i18n._(PAUSE_OWN_STREAM_PREVIEW_DESCRIPTOR),
checked: pauseOwnScreenSharePreviewOnUnfocus,
onChange: (checked: boolean) => VoiceSettingsCommands.update({pauseOwnScreenSharePreviewOnUnfocus: checked}),
},
);
if (guildId === undefined) {
items.push({
label: i18n._(SCREEN_SHARE_PRIVACY_DESCRIPTOR),
onClick: () => {
onClose();
openScreenSharePreviewPrivacyModal();
},
});
}
}
items.push({
label: i18n._(COPY_STATS_JSON_DESCRIPTOR),
onClick: () => {
onClose();
void copyVoiceDiagnostics(i18n).catch((error) => {
logger.error('Failed to copy voice diagnostics from participant menu', error);
});
},
});
if (items.length === 0) return null;
return {
label: i18n._(MORE_OPTIONS_DESCRIPTOR),
@@ -334,6 +347,17 @@ export const VoiceParticipantOwnStreamMenuTail: React.FC<VoiceParticipantOwnStre
{i18n._(SCREEN_SHARE_PRIVACY_DESCRIPTOR)}
</MenuItem>
)}
<MenuItem
onClick={() => {
onClose();
void copyVoiceDiagnostics(i18n).catch((error) => {
logger.error('Failed to copy voice diagnostics from participant menu', error);
});
}}
data-flx="ui.action-menu.items.voice-participant-stream-menu-builder.voice-participant-own-stream-menu-tail.menu-item.close--2"
>
{i18n._(COPY_STATS_JSON_DESCRIPTOR)}
</MenuItem>
</MenuGroup>
)}
data-flx="ui.action-menu.items.voice-participant-stream-menu-builder.more-options-submenu"
@@ -88,16 +88,16 @@ const DESKTOP_UPDATE_READY_DESCRIPTOR = msg({
message: 'Desktop update ready',
comment: 'Modal title shown when a desktop app update has finished downloading.',
});
const DESKTOP_VERSION_HAS_BEEN_DOWNLOADED_DESCRIPTOR = msg({
export const DESKTOP_VERSION_HAS_BEEN_DOWNLOADED_DESCRIPTOR = msg({
message: 'Desktop version {version} has been downloaded. Restart {productName} to finish installing.',
comment:
'Desktop updater modal body. The version placeholder is the downloaded app version; productName is the app name.',
});
const THE_DESKTOP_UPDATE_HAS_BEEN_DOWNLOADED_DESCRIPTOR = msg({
export const THE_DESKTOP_UPDATE_HAS_BEEN_DOWNLOADED_DESCRIPTOR = msg({
message: 'The desktop update has been downloaded. Restart {productName} to finish installing.',
comment: 'Desktop updater modal body when the downloaded version is unknown. productName is the app name.',
});
const RESTART_FLUXER_DESCRIPTOR = msg({
export const RESTART_FLUXER_DESCRIPTOR = msg({
message: 'Restart {productName}',
comment: 'Button label that restarts the app to apply a desktop update. productName is the app name.',
});
@@ -1,55 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Button} from '@app/features/ui/button/Button';
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
import {
COPIED_STATS_JSON_DESCRIPTOR,
COPY_STATS_JSON_DESCRIPTOR,
} from '@app/features/voice/components/StatsForNerdsCopyDescriptors';
import {useStatsForNerds} from '@app/features/voice/components/useStatsForNerds';
import {buildStatsForNerdsCopyPayload} from '@app/features/voice/utils/StatsForNerdsCopy';
import {useLingui} from '@lingui/react/macro';
import {CopySimpleIcon} from '@phosphor-icons/react';
import type React from 'react';
import {useCallback, useState} from 'react';
export const StatsForNerdsCopyButton: React.FC = () => {
const {i18n} = useLingui();
const data = useStatsForNerds();
const [copying, setCopying] = useState(false);
const handleCopy = useCallback(async () => {
if (copying) return;
setCopying(true);
try {
let payload: Record<string, unknown>;
try {
payload = await buildStatsForNerdsCopyPayload(data);
} catch {
payload = {
schemaVersion: 1,
createdAt: new Date().toISOString(),
statsForNerds: data,
};
}
await navigator.clipboard.writeText(JSON.stringify(payload, null, 2));
ToastCommands.createToast({
type: 'success',
children: i18n._(COPIED_STATS_JSON_DESCRIPTOR),
});
} finally {
setCopying(false);
}
}, [copying, data, i18n]);
return (
<Button
variant="secondary"
fitContent
leftIcon={<CopySimpleIcon size={16} data-flx="user.stats-for-nerds-copy-button.copy-icon" />}
submitting={copying}
onClick={() => void handleCopy()}
data-flx="user.stats-for-nerds-copy-button.copy"
>
{i18n._(COPY_STATS_JSON_DESCRIPTOR)}
</Button>
);
};
@@ -0,0 +1,22 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
import {COPIED_STATS_JSON_DESCRIPTOR} from '@app/features/voice/components/StatsForNerdsCopyDescriptors';
import {buildStatsForNerdsCopyPayload, collectStatsForNerdsSnapshot} from '@app/features/voice/utils/StatsForNerdsCopy';
import type {I18n} from '@lingui/core';
export async function copyVoiceDiagnostics(i18n: I18n): Promise<void> {
const data = collectStatsForNerdsSnapshot();
let payload: Record<string, unknown>;
try {
payload = await buildStatsForNerdsCopyPayload(data);
} catch {
payload = {
schemaVersion: 1,
createdAt: new Date().toISOString(),
statsForNerds: data,
};
}
await navigator.clipboard.writeText(JSON.stringify(payload, null, 2));
ToastCommands.createToast({type: 'success', children: i18n._(COPIED_STATS_JSON_DESCRIPTOR)});
}
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, it, vi} from 'vitest';
const refreshMicrophoneFromSettings = vi.fn();
const refreshCameraBackgroundFromSettings = vi.fn();
const refreshCameraCaptureFromSettings = vi.fn();
const refreshScreenShareCodecNegotiationFromSettings = vi.fn();
const settings = {
cameraResolution: 720,
videoDeviceId: 'camera',
screenshareResolution: 1080,
videoFrameRate: 60,
streamingMode: 'smooth',
screenShareContentHint: 'auto',
preferredScreenShareCodec: 'auto',
screenShareAv1OptIn: true,
screenShareHevcOptIn: false,
screenShareEncoderMode: 'auto',
screenShareSoftwareQuality: 'auto',
screenShareScalabilityMode: 'auto',
screenShareBackupCodecMode: 'auto',
openH264Enabled: false,
};
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
default: {
room: null,
refreshMicrophoneFromSettings,
refreshCameraBackgroundFromSettings,
refreshCameraCaptureFromSettings,
refreshScreenShareCodecNegotiationFromSettings,
setScreenShareAudioMuted: vi.fn(),
applyAllLocalAudioPreferences: vi.fn(),
applyLocalInputVolume: vi.fn(),
},
}));
vi.mock('@app/features/voice/engine/ScreenShareUnderperformance', () => ({
noteDeliberateScreenShareQualityChange: vi.fn(),
}));
vi.mock('@app/features/voice/utils/VoiceProcessingProfile', () => ({
getActiveInputDeviceLabel: () => null,
}));
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
updateSettings: (patch: Record<string, unknown>) => Object.assign(settings, patch),
get cameraResolution() {
return settings.cameraResolution;
},
get videoDeviceId() {
return settings.videoDeviceId;
},
getScreenshareResolution: () => settings.screenshareResolution,
getVideoFrameRate: () => settings.videoFrameRate,
getStreamingMode: () => settings.streamingMode,
getScreenShareContentHint: () => settings.screenShareContentHint,
getPreferredScreenShareCodec: () => settings.preferredScreenShareCodec,
getScreenShareAv1OptIn: () => settings.screenShareAv1OptIn,
getScreenShareHevcOptIn: () => settings.screenShareHevcOptIn,
getScreenShareEncoderMode: () => settings.screenShareEncoderMode,
getScreenShareSoftwareQuality: () => settings.screenShareSoftwareQuality,
getScreenShareScalabilityMode: () => settings.screenShareScalabilityMode,
getScreenShareBackupCodecMode: () => settings.screenShareBackupCodecMode,
getOpenH264Enabled: () => settings.openH264Enabled,
},
}));
const VoiceSettingsCommands = await import('./VoiceSettingsCommands');
describe('screen-share codec settings reactions', () => {
beforeEach(() => {
settings.screenShareAv1OptIn = true;
settings.screenShareHevcOptIn = false;
settings.preferredScreenShareCodec = 'auto';
refreshScreenShareCodecNegotiationFromSettings.mockClear();
});
it('renegotiates the live screen share when the AV1 opt-in is turned off', () => {
VoiceSettingsCommands.update({screenShareAv1OptIn: false});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('renegotiates the live screen share when the HEVC opt-in changes', () => {
VoiceSettingsCommands.update({screenShareHevcOptIn: true});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('renegotiates the live screen share when the preferred codec changes', () => {
VoiceSettingsCommands.update({preferredScreenShareCodec: 'vp9'});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('leaves the negotiation alone for unrelated screen share settings', () => {
VoiceSettingsCommands.update({videoFrameRate: 30});
expect(refreshScreenShareCodecNegotiationFromSettings).not.toHaveBeenCalled();
});
});
@@ -79,6 +79,7 @@ type VoiceSettingsPatch = Partial<{
screenShareAudioSourceMode: 'none' | 'system' | 'specific';
screenShareAudioIncludeSources: Array<Record<string, string>>;
screenShareAudioExcludeSources: Array<Record<string, string>>;
screenShareDeviceAudioUsesMicrophone: boolean;
openH264Enabled: boolean;
}>;
@@ -102,6 +103,11 @@ const CAMERA_BACKGROUND_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
'mirrorCamera',
];
const CAMERA_CAPTURE_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = ['cameraResolution', 'videoDeviceId'];
const SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
'preferredScreenShareCodec',
'screenShareAv1OptIn',
'screenShareHevcOptIn',
];
function refreshMicrophone(): void {
MediaEngine.refreshMicrophoneFromSettings();
}
@@ -114,6 +120,10 @@ function refreshCameraCapture(): void {
MediaEngine.refreshCameraCaptureFromSettings();
}
function refreshScreenShareCodecNegotiation(): void {
MediaEngine.refreshScreenShareCodecNegotiationFromSettings();
}
function shouldRefreshMicrophone(settings: VoiceSettingsPatch): boolean {
return MICROPHONE_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
@@ -122,6 +132,10 @@ function shouldRefreshCameraBackground(settings: VoiceSettingsPatch): boolean {
return CAMERA_BACKGROUND_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
function shouldRefreshScreenShareCodecNegotiation(settings: VoiceSettingsPatch): boolean {
return SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
function readCameraCaptureRefreshValues(): VoiceSettingsPatch {
return {
cameraResolution: VoiceSettings.cameraResolution,
@@ -195,6 +209,9 @@ function applyUpdatedVoiceSettings(
if (shouldRefreshCameraCapture(settings, cameraCaptureBefore, readCameraCaptureRefreshValues())) {
refreshCameraCapture();
}
if (shouldRefreshScreenShareCodecNegotiation(settings)) {
refreshScreenShareCodecNegotiation();
}
}
export function setVoiceProcessingModeForDeviceLabel(label: string, mode: VoiceProcessingMode): void {
@@ -141,6 +141,7 @@ export const AudioSourcePickerLinuxSubmenu = observer((props: AudioSourcePickerL
const sourceMode = VoiceSettings.getScreenShareAudioSourceMode();
const includeSources = VoiceSettings.getScreenShareAudioIncludeSources();
const excludeSources = VoiceSettings.getScreenShareAudioExcludeSources();
const usesDeviceMicrophone = VoiceSettings.getScreenShareDeviceAudioUsesMicrophone();
const granular = VoiceSettings.getLinuxAudioCaptureGranularSelect();
const deviceSelect = VoiceSettings.getLinuxAudioCaptureDeviceSelect();
const ignoreVirtual = VoiceSettings.getLinuxAudioCaptureIgnoreVirtual();
@@ -167,12 +168,13 @@ export const AudioSourcePickerLinuxSubmenu = observer((props: AudioSourcePickerL
onSelectionChange?.('window');
}, [onSelectionChange]);
const handlePickSystem = useCallback(() => {
VoiceSettingsCommands.update({
screenShareAudioSourceMode: 'system',
screenShareAudioIncludeSources: [],
});
VoiceSettingsCommands.update(
isDeviceShare
? {screenShareDeviceAudioUsesMicrophone: true}
: {screenShareAudioSourceMode: 'system', screenShareAudioIncludeSources: []},
);
onSelectionChange?.(widenedScope);
}, [onSelectionChange, widenedScope]);
}, [isDeviceShare, onSelectionChange, widenedScope]);
const handlePickNone = useCallback(() => {
VoiceSettingsCommands.update({
screenShareAudioSourceMode: 'none',
@@ -189,10 +191,11 @@ export const AudioSourcePickerLinuxSubmenu = observer((props: AudioSourcePickerL
VoiceSettingsCommands.update({
screenShareAudioSourceMode: nextSources.length > 0 ? 'specific' : 'system',
screenShareAudioIncludeSources: nextSources,
...(isDeviceShare ? {screenShareDeviceAudioUsesMicrophone: false} : {}),
});
onSelectionChange?.(widenedScope);
},
[includeSources, onSelectionChange, widenedScope],
[includeSources, isDeviceShare, onSelectionChange, widenedScope],
);
const handleToggleExcludeApp = useCallback(
(item: LinuxAudioSourceItem) => {
@@ -217,9 +220,10 @@ export const AudioSourcePickerLinuxSubmenu = observer((props: AudioSourcePickerL
microphoneLabel,
displayShareEnvironment,
windowAudioScope,
usesDeviceMicrophone,
});
const showsWideSourceLists = !offersWindowScope || resolvedScope === 'system';
const wideSourceIsSelected = isDeviceShare ? !routesSelectedSources : sourceMode === 'system';
const wideSourceIsSelected = isDeviceShare ? usesDeviceMicrophone || !routesSelectedSources : sourceMode === 'system';
if (!snapshot.available && !snapshot.loading) {
return null;
}
@@ -2,13 +2,9 @@
import {msg} from '@lingui/core/macro';
export const COPY_AS_JSON_DESCRIPTOR = msg({
message: 'Copy as JSON',
comment: 'Aria label for a button that copies stats for nerds data as JSON to the clipboard.',
});
export const COPY_STATS_JSON_DESCRIPTOR = msg({
message: 'Copy stats JSON',
comment: 'Button label in audio and video settings. Copies stats for nerds data as JSON to the clipboard.',
comment: 'Menu item label in the screen share menu. Copies stats for nerds data as JSON to the clipboard.',
});
export const COPIED_STATS_JSON_DESCRIPTOR = msg({
message: 'Copied stats JSON to clipboard',
@@ -15,6 +15,7 @@ import {
} from '@app/features/i18n/utils/CommonMessageDescriptors';
import Favorites from '@app/features/messaging/state/Favorites';
import {goBackOr} from '@app/features/platform/components/router/NavigationAdapter';
import {STREAM_VOLUME_DESCRIPTOR} from '@app/features/ui/action_menu/items/voice_participant_menu_data/shared';
import {BottomSheet} from '@app/features/ui/bottom_sheet/BottomSheet';
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
import FocusRing from '@app/features/ui/focus_ring/FocusRing';
@@ -636,6 +637,7 @@ const VoiceCallViewInner = observer(
iconSize={18}
className={styles.voiceHeaderIconButton}
position="below"
ariaLabel={i18n._(STREAM_VOLUME_DESCRIPTOR)}
data-flx="voice.voice-call-view.voice-call-view-inner.hud-stream-volume-control"
/>
)}
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import Accessibility from '@app/features/accessibility/state/Accessibility';
import {LongPressable} from '@app/features/app/components/LongPressable';
import Channels from '@app/features/channel/state/Channels';
import {WATCH_STREAM_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';
@@ -7,6 +8,7 @@ import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';
import Permission from '@app/features/permissions/state/Permission';
import {dimColor} from '@app/features/theme/utils/ColorUtils';
import type {VoiceParticipantMenuSource} from '@app/features/ui/action_menu/items/VoiceParticipantMenuTypes';
import {STREAM_VOLUME_DESCRIPTOR} from '@app/features/ui/action_menu/items/voice_participant_menu_data/shared';
import {UserContextMenu} from '@app/features/ui/action_menu/UserContextMenu';
import {VoiceParticipantContextMenu} from '@app/features/ui/action_menu/VoiceParticipantContextMenu';
import {Button} from '@app/features/ui/button/Button';
@@ -57,6 +59,7 @@ import {
selectVoiceParticipantTileCameraActive,
selectVoiceParticipantTileScreenShareState,
shouldShowCameraBuffering,
shouldShowTileStreamAudioControls,
type VoiceParticipantTileScreenShareSignals,
} from '@app/features/voice/components/VoiceParticipantTileStateMachine';
import {useVoiceTileGroup} from '@app/features/voice/components/VoiceTileGroupContext';
@@ -296,6 +299,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
: getVoiceDeafenedStatusLabel(i18n, isCurrentUser);
const deafenStatusClassName = isModeratorDeafened ? styles.participantIconRed : styles.participantIconMuted;
const isActuallySpeaking = displayState.speaking;
const shouldAnimateTileAvatar = isActuallySpeaking && !Accessibility.useReducedMotion;
const isMobileExperience = isMobileExperienceEnabled();
const [bottomSheetOpen, setBottomSheetOpen] = useState(false);
const tileGroup = useVoiceTileGroup();
@@ -632,7 +636,21 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
const participantDisplayName =
(participantUser ? NicknameUtils.getNickname(participantUser, guildId, channelId) : participant.name) ||
i18n._(UNKNOWN_USER_DESCRIPTOR);
const showStreamAudioControls = isScreenShare && !isOwnScreenShare && isWatching && hasScreenShareAudio;
const showStreamAudioControls = shouldShowTileStreamAudioControls({
isScreenShare,
isOwnScreenShare,
isWatching,
hasScreenShareAudio,
isFocusedPlaceholderTile,
presentation,
});
const showTileSpectatorPill = !isFocusPresentationTile && isScreenShare && viewerUsers.length > 0;
const showTileControlPill =
!isFocusedPlaceholderTile &&
(showStreamAudioControls ||
showTileSpectatorPill ||
(isGridTile && groupHiddenCount > 0) ||
showDeviceCollapseControl);
const viewerStreamCount = graphViewerStreamKeys.length;
const addStreamTooltipText = plural(
{count: viewerStreamCount},
@@ -940,6 +958,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
user={participantUser}
size={TILE_AVATAR_BASE}
mediaSize={TILE_AVATAR_MEDIA_SIZE}
forceAnimate={shouldAnimateTileAvatar}
className={styles.avatarFlexShrink}
style={TILE_AVATAR_STYLE}
guildId={guildId}
@@ -962,6 +981,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
placeholderStyle,
previewUrl,
screenSharePlaceholderStyle,
shouldAnimateTileAvatar,
trackRef,
shouldHideOwnScreenShareVideo,
]);
@@ -1195,154 +1215,150 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
/>
</div>
)}
{!isFocusPresentationTile &&
!isFocusedPlaceholderTile &&
(showStreamAudioControls ||
(isScreenShare && viewerUsers.length > 0) ||
(isGridTile && groupHiddenCount > 0) ||
showDeviceCollapseControl) && (
<div
className={clsx(
voiceCallStyles.tileControlPill,
isScreenShare && viewerUsers.length > 0 && voiceCallStyles.tileControlPillPersistent,
)}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tile-control-pill"
>
{showStreamAudioControls && (
<div
className={clsx(voiceCallStyles.tileControlPillSlot, isStreamMuted && styles.streamAudioSlotMuted)}
role="group"
onClick={(e) => e.stopPropagation()}
onKeyDown={(e) => e.stopPropagation()}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-audio-volume"
{showTileControlPill && (
<div
className={clsx(
voiceCallStyles.tileControlPill,
showTileSpectatorPill && voiceCallStyles.tileControlPillPersistent,
)}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tile-control-pill"
>
{showStreamAudioControls && (
<div
className={clsx(voiceCallStyles.tileControlPillSlot, isStreamMuted && styles.streamAudioSlotMuted)}
role="group"
onClick={(e) => e.stopPropagation()}
onKeyDown={(e) => e.stopPropagation()}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-audio-volume"
>
<MediaVerticalVolumeControl
volume={streamVolume / 100}
isMuted={isStreamMuted}
maxVolume={VOICE_VOLUME_MAX_SLIDER_VOLUME}
onVolumeChange={handleStreamVolumeChange}
onToggleMute={handleStreamAudioToggle}
iconSize={14}
position="below"
ariaLabel={i18n._(STREAM_VOLUME_DESCRIPTOR)}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-audio-volume-control"
/>
</div>
)}
{showDeviceCollapseControl && (
<Tooltip
text={groupCollapseTooltip}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-tooltip"
>
<FocusRing
offset={-2}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-focus-ring"
>
<MediaVerticalVolumeControl
volume={streamVolume / 100}
isMuted={isStreamMuted}
maxVolume={VOICE_VOLUME_MAX_SLIDER_VOLUME}
onVolumeChange={handleStreamVolumeChange}
onToggleMute={handleStreamAudioToggle}
iconSize={14}
position="below"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-audio-volume-control"
<div
role="button"
tabIndex={0}
className={clsx(voiceCallStyles.tileControlPillSlot, styles.groupExpandPillSlot)}
onClick={handleExpandGroup}
onKeyDown={(event) => {
if (!isKeyboardActivationKey(event.key)) return;
event.preventDefault();
handleExpandGroup(event);
}}
aria-label={groupCollapseTooltip}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-pill"
>
<span
className={styles.groupExpandPillSign}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill-sign"
>
-
</span>
{groupDeviceConnectionCount}
</div>
</FocusRing>
</Tooltip>
)}
{isGridTile && groupHiddenCount > 0 && (
<Tooltip
text={groupExpandTooltip}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-tooltip"
>
<FocusRing
offset={-2}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-focus-ring"
>
<div
role="button"
tabIndex={0}
className={clsx(voiceCallStyles.tileControlPillSlot, styles.groupExpandPillSlot)}
onClick={handleExpandGroup}
onKeyDown={(event) => {
if (!isKeyboardActivationKey(event.key)) return;
event.preventDefault();
handleExpandGroup(event);
}}
aria-label={groupExpandTooltip}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill"
>
<span
className={styles.groupExpandPillSign}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill-sign--2"
>
+
</span>
{groupHiddenCount}
</div>
</FocusRing>
</Tooltip>
)}
{isOwnScreenShare && viewerUsers.length > 0 && streamUnderperformanceReason && (
<Tooltip
text={getStreamUnderperformanceLabel(i18n, streamUnderperformanceReason)}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tooltip"
>
<div
className={clsx(voiceCallStyles.tileControlPillSlot, styles.streamUnderperformanceSlot)}
role="img"
aria-label={i18n._(STREAM_NOT_KEEPING_UP_DESCRIPTOR)}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-underperformance-slot"
>
<WarningIcon
weight="fill"
className={styles.tilePillIcon}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tile-pill-icon"
/>
</div>
)}
{showDeviceCollapseControl && (
<Tooltip
text={groupCollapseTooltip}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-tooltip"
</Tooltip>
)}
{showTileSpectatorPill && (
<StreamSpectatorsPopout
viewerUsers={viewerUsers}
spectatorEntries={spectatorEntries}
guildId={guildId}
channelId={channelId}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-spectators-popout"
>
<div
className={clsx(voiceCallStyles.tileControlPillSlot, voiceCallStyles.tileControlPillViewerSlot)}
role="img"
aria-label={i18n._(WATCHING_DESCRIPTOR, {length: viewerUsers.length})}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-count"
>
<FocusRing
offset={-2}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-focus-ring"
>
<div
role="button"
tabIndex={0}
className={clsx(voiceCallStyles.tileControlPillSlot, styles.groupExpandPillSlot)}
onClick={handleExpandGroup}
onKeyDown={(event) => {
if (!isKeyboardActivationKey(event.key)) return;
event.preventDefault();
handleExpandGroup(event);
}}
aria-label={groupCollapseTooltip}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-collapse-pill"
>
<span
className={styles.groupExpandPillSign}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill-sign"
>
-
</span>
{groupDeviceConnectionCount}
</div>
</FocusRing>
</Tooltip>
)}
{isGridTile && groupHiddenCount > 0 && (
<Tooltip
text={groupExpandTooltip}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-tooltip"
>
<FocusRing
offset={-2}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-focus-ring"
>
<div
role="button"
tabIndex={0}
className={clsx(voiceCallStyles.tileControlPillSlot, styles.groupExpandPillSlot)}
onClick={handleExpandGroup}
onKeyDown={(event) => {
if (!isKeyboardActivationKey(event.key)) return;
event.preventDefault();
handleExpandGroup(event);
}}
aria-label={groupExpandTooltip}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill"
>
<span
className={styles.groupExpandPillSign}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.group-expand-pill-sign--2"
>
+
</span>
{groupHiddenCount}
</div>
</FocusRing>
</Tooltip>
)}
{isOwnScreenShare && viewerUsers.length > 0 && streamUnderperformanceReason && (
<Tooltip
text={getStreamUnderperformanceLabel(i18n, streamUnderperformanceReason)}
position="top"
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tooltip"
>
<div
className={clsx(voiceCallStyles.tileControlPillSlot, styles.streamUnderperformanceSlot)}
role="img"
aria-label={i18n._(STREAM_NOT_KEEPING_UP_DESCRIPTOR)}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-underperformance-slot"
>
<WarningIcon
weight="fill"
className={styles.tilePillIcon}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.tile-pill-icon"
/>
</div>
</Tooltip>
)}
{isScreenShare && viewerUsers.length > 0 && (
<StreamSpectatorsPopout
viewerUsers={viewerUsers}
spectatorEntries={spectatorEntries}
guildId={guildId}
channelId={channelId}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.stream-spectators-popout"
>
<div
className={clsx(voiceCallStyles.tileControlPillSlot, voiceCallStyles.tileControlPillViewerSlot)}
role="img"
aria-label={i18n._(WATCHING_DESCRIPTOR, {length: viewerUsers.length})}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-count"
>
<EyeIcon
weight="fill"
className={styles.tilePillIcon}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-icon"
/>
<span data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-count-text">
{viewerUsers.length}
</span>
</div>
</StreamSpectatorsPopout>
)}
</div>
)}
<EyeIcon
weight="fill"
className={styles.tilePillIcon}
data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-icon"
/>
<span data-flx="voice.voice-participant-tile.voice-participant-tile-inner.viewer-count-text">
{viewerUsers.length}
</span>
</div>
</StreamSpectatorsPopout>
)}
</div>
)}
{showParticipantMetadata && (
<div
className={voiceCallStyles.lkParticipantMetadata}
@@ -30,6 +30,23 @@ describe('VoiceParticipantTile stability', () => {
expect(css).not.toMatch(/transform:\s*scale/);
expect(css).not.toContain('will-change: transform');
});
it('drives the tile avatar animation from the speaking signal so animated avatars play without hover', () => {
const tileSource = sourceFile('VoiceParticipantTile.tsx');
const avatarElement = tileSource.match(/<Avatar\b[\s\S]*?\/>/)?.[0];
expect(avatarElement).toBeDefined();
const forceAnimateExpression = avatarElement?.match(/forceAnimate=\{([^}]+)\}/)?.[1]?.trim();
expect(forceAnimateExpression).toBeDefined();
const animateFlagSource =
forceAnimateExpression === 'isActuallySpeaking'
? 'isActuallySpeaking'
: tileSource.match(new RegExp(`const ${forceAnimateExpression} = ([^;]+);`))?.[1];
expect(animateFlagSource).toContain('isActuallySpeaking');
expect(animateFlagSource).toContain('!Accessibility.useReducedMotion');
expect(tileSource).toContain("import Accessibility from '@app/features/accessibility/state/Accessibility';");
const afterAvatarElement = tileSource.slice(tileSource.indexOf(avatarElement!) + avatarElement!.length);
const mediaNodeDeps = afterAvatarElement.match(/\}, \[([\s\S]*?)\]\);/)?.[1] ?? '';
expect(mediaNodeDeps).toContain(forceAnimateExpression);
});
it('keeps the fullscreen call surface mounted while the media room catches up to a channel switch', () => {
const voiceCallViewSource = sourceFile('VoiceCallView.tsx');
const guildChannelViewSource = appSourceFile('features/channel/components/channel_view/GuildChannelView.tsx');
@@ -6,10 +6,12 @@ import {
selectVoiceParticipantTileCameraActive,
selectVoiceParticipantTileScreenShareState,
shouldShowCameraBuffering,
shouldShowTileStreamAudioControls,
shouldShowWatchFailed,
type VoiceParticipantTileCameraActiveSignals,
type VoiceParticipantTileCameraBufferingSignals,
type VoiceParticipantTileScreenShareSignals,
type VoiceParticipantTileStreamAudioSignals,
} from '@app/features/voice/components/VoiceParticipantTileStateMachine';
import {
getAppliedScreenShareFrameRate,
@@ -84,6 +86,50 @@ function cameraActiveSignals(
};
}
function streamAudioSignals(
overrides: Partial<VoiceParticipantTileStreamAudioSignals> = {},
): VoiceParticipantTileStreamAudioSignals {
return {
isScreenShare: true,
isOwnScreenShare: false,
isWatching: true,
hasScreenShareAudio: true,
isFocusedPlaceholderTile: false,
presentation: 'grid',
...overrides,
};
}
describe('VoiceParticipantTileStateMachine stream audio controls', () => {
it('keeps the per-stream volume control on the watched stream once it becomes the focused tile', () => {
expect(shouldShowTileStreamAudioControls(streamAudioSignals({presentation: 'focus-main'}))).toBe(true);
});
it('shows the per-stream volume control on a watched grid tile', () => {
expect(shouldShowTileStreamAudioControls(streamAudioSignals({presentation: 'grid'}))).toBe(true);
});
it('leaves the control off carousel thumbnails, own shares, and placeholder tiles', () => {
expect(shouldShowTileStreamAudioControls(streamAudioSignals({presentation: 'focus-secondary'}))).toBe(false);
expect(shouldShowTileStreamAudioControls(streamAudioSignals({isOwnScreenShare: true}))).toBe(false);
expect(
shouldShowTileStreamAudioControls(
streamAudioSignals({presentation: 'focus-main', isFocusedPlaceholderTile: true}),
),
).toBe(false);
expect(shouldShowTileStreamAudioControls(streamAudioSignals({isScreenShare: false}))).toBe(false);
});
it('leaves the control off streams that are not watched or carry no audio', () => {
expect(shouldShowTileStreamAudioControls(streamAudioSignals({presentation: 'focus-main', isWatching: false}))).toBe(
false,
);
expect(
shouldShowTileStreamAudioControls(streamAudioSignals({presentation: 'focus-main', hasScreenShareAudio: false})),
).toBe(false);
});
});
describe('VoiceParticipantTileStateMachine camera buffering state', () => {
it('shows buffering while an active camera publication has no video', () => {
expect(shouldShowCameraBuffering(cameraSignals())).toBe(true);
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {VoiceParticipantTilePresentation} from '@app/features/voice/components/voice_participant_tile/shared';
import type {VoiceMediaGraphStreamTileState} from '@app/features/voice/engine/VoiceMediaGraphTileState';
import {assign, getInitialSnapshot, setup, transition} from 'xstate';
@@ -31,6 +32,15 @@ export interface VoiceParticipantTileCameraBufferingSignals {
hasRenderedVideoFrame: boolean;
}
export interface VoiceParticipantTileStreamAudioSignals {
isScreenShare: boolean;
isOwnScreenShare: boolean;
isWatching: boolean;
hasScreenShareAudio: boolean;
isFocusedPlaceholderTile: boolean;
presentation: VoiceParticipantTilePresentation;
}
export interface VoiceParticipantTileCameraActiveSignals {
isCameraTile: boolean;
isOwnContent: boolean;
@@ -141,6 +151,15 @@ export function shouldShowWatchPrompt(signals: VoiceParticipantTileScreenShareSi
return !signals.isFocusPresentationTile;
}
export function shouldShowTileStreamAudioControls(signals: VoiceParticipantTileStreamAudioSignals): boolean {
if (!signals.isScreenShare) return false;
if (signals.isOwnScreenShare) return false;
if (signals.isFocusedPlaceholderTile) return false;
if (!signals.isWatching) return false;
if (!signals.hasScreenShareAudio) return false;
return signals.presentation === 'grid' || signals.presentation === 'focus-main';
}
export function shouldShowCameraBuffering(signals: VoiceParticipantTileCameraBufferingSignals): boolean {
if (signals.isScreenShare) return false;
if (signals.isFocusedPlaceholderTile) return false;
@@ -1492,6 +1492,7 @@ const ScreenSharePickerModalLoadedContent = observer(
microphoneLabel,
displayShareEnvironment,
windowAudioScope,
usesDeviceMicrophone: VoiceSettings.getScreenShareDeviceAudioUsesMicrophone(),
})
: activeTab === 'devices'
? i18n._(DEVICE_AUDIO_MUTED_DESCRIPTOR)
@@ -9,7 +9,6 @@ import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/Med
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
@@ -17,6 +16,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {VoiceEngineV2PerTrackStats, VoiceEngineV2TransportInfo} from '@fluxer/voice_engine_v2';
@@ -216,8 +216,8 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
@@ -0,0 +1,183 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
isScreenShareVideoSubscriptionRecoveryWanted,
ScreenShareVideoSubscriptionRecoveryCoordinator,
type ScreenShareVideoSubscriptionRecoveryGraph,
type ScreenShareVideoSubscriptionRecoveryPublication,
type ScreenShareVideoSubscriptionRecoveryScheduler,
} from '@app/features/voice/components/voice_participant_tile/ScreenShareVideoSubscriptionRecovery';
import {
createVoiceMediaGraphSnapshot,
transitionVoiceMediaGraph,
VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE,
type VoiceMediaGraphFailure,
type VoiceMediaGraphSnapshot,
} from '@app/features/voice/engine/VoiceMediaGraph';
import {ScreenShareWatchErrorCode} from '@app/features/voice/state/ScreenShareWatchFailures';
import {describe, expect, it, vi} from 'vitest';
const STREAM_KEY = 'dm:channel-a:connection-a';
const PARTICIPANT_IDENTITY = 'user_1_connection-a';
const TRACK_SID = 'TR_screen_share';
const FIRST_FRAME_TIMEOUT_FAILURE: Partial<VoiceMediaGraphFailure> = {
code: ScreenShareWatchErrorCode.FirstFrameTimeout,
reason: 'first-frame-timeout',
};
const REMOTE_SUBSCRIPTION_FAILURE: Partial<VoiceMediaGraphFailure> = {
code: ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed,
reason: 'remote-track-subscription-failed',
};
function createScheduler() {
let nextId = 1;
const callbacks = new Map<number, () => void>();
const scheduledDelays: Array<number> = [];
const scheduler: ScreenShareVideoSubscriptionRecoveryScheduler = {
setTimeout: (callback, delayMs) => {
const timeoutId = nextId;
nextId += 1;
callbacks.set(timeoutId, callback);
scheduledDelays.push(delayMs);
return timeoutId;
},
clearTimeout: (timeoutId) => {
callbacks.delete(timeoutId);
},
};
return {
scheduler,
pendingCount: () => callbacks.size,
scheduledDelays: () => [...scheduledDelays],
fire: () => {
const next = callbacks.entries().next();
if (next.done) return;
callbacks.delete(next.value[0]);
next.value[1]();
},
};
}
function createWatchedSnapshot(failure: Partial<VoiceMediaGraphFailure> | null): VoiceMediaGraphSnapshot {
let snapshot: VoiceMediaGraphSnapshot = createVoiceMediaGraphSnapshot();
snapshot = transitionVoiceMediaGraph(snapshot, {type: 'watchIntent.add', key: STREAM_KEY});
snapshot = transitionVoiceMediaGraph(snapshot, {type: 'watch.started', streamKey: STREAM_KEY});
if (!failure) return snapshot;
return transitionVoiceMediaGraph(snapshot, {
type: 'failure.reported',
failure: {
code: -2202,
reason: 'remote-track-subscription-failed',
reportedAt: 1000,
source: VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE,
streamKey: STREAM_KEY,
participantIdentity: PARTICIPANT_IDENTITY,
...failure,
},
});
}
function createGraphPort(snapshot: VoiceMediaGraphSnapshot): ScreenShareVideoSubscriptionRecoveryGraph {
return {
getGraphSnapshot: () => snapshot,
nowMs: () => 1000,
transition: () => undefined,
};
}
function createReceivablePublication(): ScreenShareVideoSubscriptionRecoveryPublication {
return {
trackSid: TRACK_SID,
isSubscribed: true,
track: {mediaStreamTrack: {readyState: 'live', muted: false}},
};
}
function acquireWatchingTile(failure: Partial<VoiceMediaGraphFailure> | null) {
const snapshot = createWatchedSnapshot(failure);
const {scheduler, fire, pendingCount, scheduledDelays} = createScheduler();
const coordinator = new ScreenShareVideoSubscriptionRecoveryCoordinator(scheduler, createGraphPort(snapshot));
const recover = vi.fn();
coordinator.acquire({
key: TRACK_SID,
publication: createReceivablePublication(),
streamKey: STREAM_KEY,
participantIdentity: PARTICIPANT_IDENTITY,
isStillWanted: () => isScreenShareVideoSubscriptionRecoveryWanted(snapshot, STREAM_KEY),
recover,
});
return {coordinator, fire, pendingCount, scheduledDelays, recover};
}
describe('isScreenShareVideoSubscriptionRecoveryWanted', () => {
it('keeps the lease armed after a first-frame timeout so recovery can still run', () => {
expect(
isScreenShareVideoSubscriptionRecoveryWanted(createWatchedSnapshot(FIRST_FRAME_TIMEOUT_FAILURE), STREAM_KEY),
).toBe(true);
});
it('drops the lease for failures other than a first-frame timeout', () => {
expect(
isScreenShareVideoSubscriptionRecoveryWanted(createWatchedSnapshot(REMOTE_SUBSCRIPTION_FAILURE), STREAM_KEY),
).toBe(false);
});
it('drops the lease once the stream is no longer watched', () => {
expect(isScreenShareVideoSubscriptionRecoveryWanted(createVoiceMediaGraphSnapshot(), STREAM_KEY)).toBe(false);
});
});
describe('ScreenShareVideoSubscriptionRecoveryCoordinator first-frame recovery', () => {
it('keeps resubscribing while a first-frame timeout stands, up to the recovery limit', () => {
const {coordinator, fire, pendingCount, recover} = acquireWatchingTile(FIRST_FRAME_TIMEOUT_FAILURE);
fire();
expect(recover).toHaveBeenCalledTimes(1);
expect(recover).toHaveBeenCalledWith('resubscribe');
expect(pendingCount()).toBe(1);
fire();
fire();
expect(recover).toHaveBeenCalledTimes(3);
expect(recover).toHaveBeenNthCalledWith(3, 'resubscribe');
expect(pendingCount()).toBe(1);
fire();
expect(recover).toHaveBeenCalledTimes(3);
expect(coordinator.getActiveSessionCount()).toBe(0);
});
it('gives every rebuilt subscription at least the first-frame budget before resubscribing again', () => {
const {fire, scheduledDelays} = acquireWatchingTile(FIRST_FRAME_TIMEOUT_FAILURE);
fire();
fire();
expect(scheduledDelays().slice(1)).toEqual([20_000, 20_000]);
});
it('keeps polling without recovering while a receivable track has no reported failure', () => {
const {coordinator, fire, recover} = acquireWatchingTile(null);
fire();
fire();
fire();
expect(recover).not.toHaveBeenCalled();
expect(coordinator.getActiveSessionCount()).toBe(1);
});
it('closes the session without recovering for a failure other than a first-frame timeout', () => {
const {coordinator, fire, recover} = acquireWatchingTile(REMOTE_SUBSCRIPTION_FAILURE);
fire();
expect(recover).not.toHaveBeenCalled();
expect(coordinator.getActiveSessionCount()).toBe(0);
});
});
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
selectVoiceMediaGraphFailure,
selectVoiceMediaGraphViewerStreamKeys,
selectVoiceMediaGraphWatchGeneration,
type VoiceMediaGraphEvent,
@@ -8,7 +9,10 @@ import {
} from '@app/features/voice/engine/VoiceMediaGraph';
import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import {getScreenShareWatchFailureForPublicationOperation} from '@app/features/voice/state/ScreenShareWatchFailures';
import {
getScreenShareWatchFailureForPublicationOperation,
ScreenShareWatchErrorCode,
} from '@app/features/voice/state/ScreenShareWatchFailures';
import {
refreshScreenSharePublicationSubscription,
resubscribeScreenSharePublication,
@@ -58,6 +62,7 @@ interface ScreenShareVideoSubscriptionRecoverySession {
key: string;
leaseCount: number;
attempt: number;
firstFrameRecoveries: number;
timeoutId: number | null;
graphGeneration: number | null;
options: ScreenShareVideoSubscriptionRecoveryLeaseOptions;
@@ -67,6 +72,8 @@ const SCREEN_SHARE_VIDEO_SUBSCRIPTION_RETRY_INITIAL_DELAY_MS = 2500;
const SCREEN_SHARE_VIDEO_SUBSCRIPTION_RETRY_MAX_DELAY_MS = 30_000;
const SCREEN_SHARE_VIDEO_SUBSCRIPTION_HEALTH_CHECK_DELAY_MS = 2500;
const SCREEN_SHARE_VIDEO_SUBSCRIPTION_REFRESH_ATTEMPTS = 0;
const SCREEN_SHARE_VIDEO_SUBSCRIPTION_FIRST_FRAME_RECOVERY_ATTEMPTS = 3;
const SCREEN_SHARE_VIDEO_SUBSCRIPTION_FIRST_FRAME_RECOVERY_DELAY_MS = 20_000;
const defaultScheduler: ScreenShareVideoSubscriptionRecoveryScheduler = {
setTimeout: (callback, delayMs) => window.setTimeout(callback, delayMs),
@@ -106,6 +113,16 @@ function hasReceivableTrack(publication: ScreenShareVideoSubscriptionRecoveryPub
return isTrackReceivable(track);
}
export function isScreenShareVideoSubscriptionRecoveryWanted(
snapshot: VoiceMediaGraphSnapshot,
streamKey: string | null | undefined,
): boolean {
if (!streamKey) return false;
if (!selectVoiceMediaGraphViewerStreamKeys(snapshot).includes(streamKey)) return false;
const failure = selectVoiceMediaGraphFailure(snapshot, {streamKey});
return failure == null || failure.code === ScreenShareWatchErrorCode.FirstFrameTimeout;
}
export function getScreenShareVideoSubscriptionRecoveryKey({
trackSid,
streamKey,
@@ -152,6 +169,7 @@ export class ScreenShareVideoSubscriptionRecoveryCoordinator {
key: options.key,
leaseCount: 0,
attempt: 0,
firstFrameRecoveries: 0,
timeoutId: null,
graphGeneration: null,
options,
@@ -225,6 +243,13 @@ export class ScreenShareVideoSubscriptionRecoveryCoordinator {
return selectVoiceMediaGraphWatchGeneration(snapshot, streamKey) === session.graphGeneration;
}
private hasFirstFrameTimeoutFailure(session: ScreenShareVideoSubscriptionRecoverySession): boolean {
const streamKey = session.options.streamKey ?? null;
if (!streamKey) return false;
const failure = selectVoiceMediaGraphFailure(this.graph.getGraphSnapshot(), {streamKey});
return failure?.code === ScreenShareWatchErrorCode.FirstFrameTimeout;
}
private run(key: string): void {
const session = this.sessions.get(key);
if (!session) return;
@@ -234,11 +259,20 @@ export class ScreenShareVideoSubscriptionRecoveryCoordinator {
this.closeSession(session);
return;
}
if (hasReceivableTrack(options.publication)) {
const firstFrameStalled = this.hasFirstFrameTimeoutFailure(session);
if (
firstFrameStalled &&
session.firstFrameRecoveries >= SCREEN_SHARE_VIDEO_SUBSCRIPTION_FIRST_FRAME_RECOVERY_ATTEMPTS
) {
this.closeSession(session);
return;
}
if (!firstFrameStalled && hasReceivableTrack(options.publication)) {
session.attempt = 0;
this.schedule(session, SCREEN_SHARE_VIDEO_SUBSCRIPTION_HEALTH_CHECK_DELAY_MS);
return;
}
if (firstFrameStalled) session.firstFrameRecoveries += 1;
session.attempt += 1;
const mode = selectScreenShareVideoSubscriptionRecoveryMode(session.attempt);
const delayMs = getScreenShareVideoSubscriptionRetryDelayMs(session.attempt);
@@ -250,7 +284,12 @@ export class ScreenShareVideoSubscriptionRecoveryCoordinator {
mode,
});
this.recoverPublication(session, mode);
this.schedule(session, getScreenShareVideoSubscriptionRetryDelayMs(session.attempt + 1));
this.schedule(
session,
firstFrameStalled
? SCREEN_SHARE_VIDEO_SUBSCRIPTION_FIRST_FRAME_RECOVERY_DELAY_MS
: getScreenShareVideoSubscriptionRetryDelayMs(session.attempt + 1),
);
}
private reportCommandFailed(
@@ -13,6 +13,7 @@ import {
} from '@app/features/voice/components/voice_participant_tile/previewEncoding';
import {
getScreenShareVideoSubscriptionRecoveryKey,
isScreenShareVideoSubscriptionRecoveryWanted,
screenShareVideoSubscriptionRecoveryCoordinator,
} from '@app/features/voice/components/voice_participant_tile/ScreenShareVideoSubscriptionRecovery';
import {
@@ -27,7 +28,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
import {useStoreVersion} from '@app/features/voice/engine/Store';
import {
selectVoiceMediaGraphHasFailureForStreamKey,
selectVoiceMediaGraphViewerStreamKeys,
type VoiceMediaGraphSnapshot,
} from '@app/features/voice/engine/VoiceMediaGraph';
@@ -538,14 +538,8 @@ export function useScreenshareWatchSubscription(opts: {
publication: pub,
streamKey,
participantIdentity,
isStillWanted: () => {
const currentStreamKey = streamKeyRef.current;
if (currentStreamKey == null) return false;
const graph = getGraphSnapshotRef.current();
if (!selectVoiceMediaGraphViewerStreamKeys(graph).includes(currentStreamKey)) return false;
if (selectVoiceMediaGraphHasFailureForStreamKey(graph, currentStreamKey)) return false;
return true;
},
isStillWanted: () =>
isScreenShareVideoSubscriptionRecoveryWanted(getGraphSnapshotRef.current(), streamKeyRef.current),
onRetry: (retry) => {
logger.warn('Retrying stalled screen share video subscription', retry);
},
@@ -27,6 +27,7 @@ function createStatsData(): StatsForNerdsData {
subscriberTransport: null,
},
localVideo: null,
localVideoLayers: [],
localAudio: null,
localScreenShare: null,
localScreenShareAudio: null,
@@ -68,20 +69,8 @@ function createStatsData(): StatsForNerdsData {
openH264Enabled: true,
},
screenShareAudioCapture: {
pump: {
active: false,
captureId: null,
sampleRate: null,
channels: null,
usesNativeSink: false,
publishStrategy: 'none',
publishedFormatKey: null,
eagerPublish: null,
eagerPublishError: null,
droppedPushFrames: 0,
pendingPushFrames: 0,
},
nativeCapture: {},
publications: [],
},
appInfo: {
appVersion: 'dev',
@@ -111,6 +100,50 @@ function createStatsData(): StatsForNerdsData {
};
}
function createNativeCaptureRecord(): Record<string, unknown> {
return {
armedCapture: null,
activeBridge: {captureId: 'native-audio:live'},
supersededBridge: null,
lastStartedCapture: null,
lastArmFailure: null,
bridgeStats: {
active: true,
bridgeMode: 'generator',
captureId: 'native-audio:live',
startedAt: 1772000000000,
lastFrameAt: 1772000011240,
lastFrameTimestampUs: 11240000,
framesReceived: 1124,
framesDropped: 0,
lateFrameCount: 0,
rebufferCount: 0,
maxFrameArrivalGapMs: 32,
maxFrameTimestampGapMs: 10,
maxPendingFrames: 3,
maxBufferedDurationMs: 30,
lastFramePeak: 0.42,
lastFrameRms: 0.11,
maxFramePeak: 0.71,
maxFrameRms: 0.19,
nonSilentFrameCount: 1118,
prebufferTargetMs: 60,
frameDurationMs: 10,
endReason: null,
endDetail: null,
endedAt: null,
},
endedBridgeCaptures: [],
lifecycleFaults: [],
};
}
function screenShareAudioCaptureSection(html: string): string {
const match = /<section[^>]*><h3[^>]*>screenShareAudioCapture \(native\)<\/h3>[\s\S]*?<\/section>/.exec(html);
expect(match).not.toBeNull();
return match?.[0] ?? '';
}
describe('renderVoiceDebugStatsHtml', () => {
it('renders pure SVG sparklines for key popout metrics', () => {
const html = renderVoiceDebugStatsHtml(createStatsData(), '2026-06-10T18:00:00.000Z');
@@ -124,4 +157,57 @@ describe('renderVoiceDebugStatsHtml', () => {
expect(html).toContain('heapUsedMB sparkline');
expect(html).toContain('mainProcessCpuPercent sparkline');
});
it('does not render a screen-share audio pump section while the native capture is live', () => {
const data = createStatsData();
data.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
expect(html).toContain('screenShareAudioCapture (native)');
expect(html).toContain('framesReceived');
expect(html).not.toContain('screenShareAudioCapture (pump)');
expect(html).not.toContain('publishStrategy');
expect(html).not.toContain('droppedPushFrames');
expect(html).not.toContain('usesNativeSink');
});
it('renders the screen-share audio capture section identically for Chromium and Firefox dumps', () => {
const chromium = createStatsData();
chromium.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
chromium.system.platform = 'Win32';
chromium.system.userAgent =
'(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
const firefox = createStatsData();
firefox.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
firefox.system.platform = 'Win32';
firefox.system.userAgent = '(Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0';
firefox.appInfo.electronVersion = null;
firefox.appInfo.chromiumVersion = null;
firefox.appInfo.hardwareAccelerationEnabled = null;
const chromiumHtml = renderVoiceDebugStatsHtml(chromium, '2026-06-10T18:00:00.000Z');
const firefoxHtml = renderVoiceDebugStatsHtml(firefox, '2026-06-10T18:00:00.000Z');
expect(screenShareAudioCaptureSection(chromiumHtml)).toBe(screenShareAudioCaptureSection(firefoxHtml));
expect(chromiumHtml).not.toContain('screenShareAudioCapture (pump)');
expect(firefoxHtml).not.toContain('screenShareAudioCapture (pump)');
});
it('renders every camera simulcast layer instead of only the projected localVideo row', () => {
const data = createStatsData();
data.localVideoLayers = [
{direction: 'send', kind: 'video', ssrc: 1001, rid: 'q', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
{direction: 'send', kind: 'video', ssrc: 1002, rid: 'h', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
{direction: 'send', kind: 'video', ssrc: 1003, rid: 'f', active: true, bitrateKbps: 1133, bitrateWindowMs: 2001},
];
data.localVideo = data.localVideoLayers[0];
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
expect(html).toContain('track:localVideoLayers[0]');
expect(html).toContain('track:localVideoLayers[1]');
expect(html).toContain('track:localVideoLayers[2]');
expect(html).toContain('bitrateWindowMs');
});
});
@@ -233,6 +233,10 @@ function TrackTables({data}: TrackTablesProps) {
const tracks: Array<[string, VoiceEngineV2PerTrackStats | null]> = [
['track:localAudio', data.localAudio],
['track:localVideo', data.localVideo],
...data.localVideoLayers.map((layer, index): [string, VoiceEngineV2PerTrackStats | null] => [
`track:localVideoLayers[${index}]`,
layer,
]),
['track:localScreenShare', data.localScreenShare],
['track:localScreenShareAudio', data.localScreenShareAudio],
['track:remoteAudio', data.remoteAudio],
@@ -372,11 +376,6 @@ function VoiceDebugStats({data, generatedAtIso}: VoiceDebugStatsProps) {
record={asStatsRecord(data.screenShareSettings)}
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--7"
/>
<StatsTable
title="screenShareAudioCapture (pump)"
record={asStatsRecord(data.screenShareAudioCapture.pump)}
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--8"
/>
<StatsTable
title="screenShareAudioCapture (native)"
record={asStatsRecord(data.screenShareAudioCapture.nativeCapture)}
@@ -749,6 +749,16 @@ class MediaEngineFacade extends Store {
await voiceEngineV2AppMediaExecutionAdapter.refreshCameraCapture();
}
refreshScreenShareCodecNegotiationFromSettings(): void {
void this.refreshScreenShareCodecNegotiationFromCurrentEngine().catch((error) => {
logger.warn('Failed to refresh screen share codec negotiation from settings', {error});
});
}
private async refreshScreenShareCodecNegotiationFromCurrentEngine(): Promise<void> {
await ScreenShareCodecNegotiation.publishLocalCapabilities(this.room, 'manual');
}
private reconcileLocalAudioStateInBackground(reason: string): void {
const previous = this.localAudioReconcileCoalescer;
const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason});
@@ -121,7 +121,7 @@ describe('MediaEngineFacadeStateMachine', () => {
expect(snapshot.context.pendingServerDisconnectConnectionId).toBeNull();
});
it('disconnects immediately when the server removes the current local voice state', () => {
it('disconnects immediately when the server removes the current local voice state without a live transport', () => {
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
@@ -131,6 +131,28 @@ describe('MediaEngineFacadeStateMachine', () => {
connected: true,
connecting: false,
}),
).toBe(false);
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: false,
connecting: true,
}),
).toBe(true);
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: false,
connecting: false,
}),
).toBe(true);
expect(
@@ -156,6 +178,50 @@ describe('MediaEngineFacadeStateMachine', () => {
).toBe(false);
});
it('keeps the transport during a server move until the replacement connection arrives', () => {
let snapshot = createMediaEngineFacadeSnapshot();
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: true,
connecting: false,
}),
).toBe(false);
expect(
shouldCancelMediaEngineReconnectForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: true,
connecting: false,
}),
).toBe(false);
snapshot = transitionMediaEngineFacadeSnapshot(snapshot, {
type: 'serverDisconnect.schedule',
connectionId: 'connection-1',
});
expect(
shouldRunMediaEngineDeferredDisconnect(snapshot, {
connectionId: 'connection-1',
currentConnectionId: 'connection-2',
connected: true,
currentVoiceStateChannelId: null,
}),
).toBe(false);
expect(
shouldRunMediaEngineDeferredDisconnect(snapshot, {
connectionId: 'connection-1',
currentConnectionId: 'connection-1',
connected: true,
currentVoiceStateChannelId: null,
}),
).toBe(true);
});
it('cancels reconnect when local voice-state removal arrives after transport disconnect', () => {
expect(
shouldCancelMediaEngineReconnectForServerVoiceStateRemoval({
@@ -527,7 +527,8 @@ export function shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval
input: MediaEngineFacadeServerVoiceStateRemovalInput,
): boolean {
if (!isCurrentServerVoiceStateRemoval(input)) return false;
return input.connected || input.connecting || input.currentChannelId != null;
if (input.connected) return false;
return input.connecting || input.currentChannelId != null;
}
export function shouldCancelMediaEngineReconnectForServerVoiceStateRemoval(
@@ -1,13 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {
FluxerCodecAdvertisement,
FluxerSelectProtocolMessage,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {beforeEach, describe, expect, it, vi} from 'vitest';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
let av1OptIn = false;
let hevcOptIn = false;
let preferredScreenShareCodec = 'auto';
let gpuReport: HardwareEncodeReport | null = null;
let firefoxBrowser = false;
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
@@ -26,7 +30,7 @@ vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
isFirefoxBrowser: () => false,
isFirefoxBrowser: () => firefoxBrowser,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
@@ -48,6 +52,8 @@ vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
}));
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
clearScreenShareDecodeFailures: () => undefined,
getScreenShareDecodeFailures: () => new Set(),
getVideoDecoderExclusionsSync: () => [],
loadVideoDecoderExclusions: async () => [],
}));
@@ -216,6 +222,92 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
});
});
describe('screen-share codec negotiation on Gecko', () => {
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = null;
firefoxBrowser = true;
resetCachedCodecCapabilities();
});
afterEach(() => {
firefoxBrowser = false;
resetCachedCodecCapabilities();
});
it('ranks VP8 ahead of H.264 because Gecko always software-encodes H.264', () => {
expect(getScreenShareCodecPreferenceOrder()).toEqual(['vp8', 'h264']);
});
it('negotiates VP8 when both ends can encode and decode VP8 and H.264', () => {
const local = buildLocalCodecAdvertisements();
const remote = [buildLocalCodecAdvertisements()];
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp8');
});
it('still lets an explicit H.264 pin win over the Gecko order', () => {
expect(getScreenShareCodecPreferenceOrder('h264')[0]).toBe('h264');
});
});
describe('advertising local capabilities over the negotiation topic', () => {
let room: FakeRoom;
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
resetCachedCodecCapabilities();
room = new FakeRoom();
ScreenShareCodecNegotiation.setSelectionChangeListener(null);
});
async function publishAndCaptureSelectProtocol(): Promise<{
payload: Uint8Array;
message: FluxerSelectProtocolMessage;
}> {
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
try {
ScreenShareCodecNegotiation.bind(room as never);
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
} finally {
Reflect.deleteProperty(globalThis, 'window');
}
const payloads = (room.localParticipant.publishData.mock.calls as Array<Array<unknown>>).map(
(call) => call[0] as Uint8Array,
);
const payload = payloads.findLast(
(candidate) => (JSON.parse(new TextDecoder().decode(candidate)) as {op: number}).op === 1,
);
if (!payload) throw new Error('no select-protocol message was published');
return {payload, message: JSON.parse(new TextDecoder().decode(payload)) as FluxerSelectProtocolMessage};
}
it('advertises no experiments because nothing on either side reads them', async () => {
const {message} = await publishAndCaptureSelectProtocol();
expect(message.op).toBe(1);
expect(message.d.experiments).toEqual([]);
ScreenShareCodecNegotiation.dispose();
});
it('keeps the empty experiments list acceptable to the receive-side validator', async () => {
const {payload} = await publishAndCaptureSelectProtocol();
ScreenShareCodecNegotiation.dispose();
const peerRoom = new FakeRoom();
peerRoom.remoteParticipants.set('peer', {identity: 'peer'});
ScreenShareCodecNegotiation.bind(peerRoom as never);
const listener = vi.fn();
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
peerRoom.emit(RoomEvent.DataReceived, payload, {identity: 'peer'}, undefined, SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC);
await new Promise((resolve) => setTimeout(resolve, 0));
expect(listener).toHaveBeenCalled();
ScreenShareCodecNegotiation.dispose();
});
});
describe('reacting to a mid-session codec selection change', () => {
let room: FakeRoom;
@@ -267,6 +359,47 @@ describe('reacting to a mid-session codec selection change', () => {
ScreenShareCodecNegotiation.dispose();
});
async function withWindowDefined(run: () => Promise<void>): Promise<void> {
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
try {
await run();
} finally {
Reflect.deleteProperty(globalThis, 'window');
}
}
it('stops reusing a live AV1 selection once the AV1 opt-in is turned off mid-share', async () => {
av1OptIn = true;
resetCachedCodecCapabilities();
await establishBaseline();
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
av1OptIn = false;
resetCachedCodecCapabilities();
expect(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')).not.toBe('av1');
expect(ScreenShareCodecNegotiation.selectNativeScreenShareCodec('auto')).not.toBe('av1');
expect(getScreenShareCodecPreferenceOrder()).toContain(ScreenShareCodecNegotiation.selectScreenShareCodec('auto'));
ScreenShareCodecNegotiation.dispose();
});
it('renegotiates the live share off AV1 when the opt-in toggle refreshes the local capabilities', async () => {
av1OptIn = true;
resetCachedCodecCapabilities();
await establishBaseline();
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
const listener = vi.fn();
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
av1OptIn = false;
resetCachedCodecCapabilities();
await withWindowDefined(async () => {
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
});
const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
expect(negotiated).not.toBe('av1');
expect(listener).toHaveBeenCalledTimes(1);
expect(listener).toHaveBeenCalledWith(room, negotiated, 'manual');
ScreenShareCodecNegotiation.dispose();
});
it('does not notify the listener when the recomputed selection is unchanged', async () => {
await establishBaseline();
room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {isFirefoxBrowser} from '@app/features/ui/utils/NativeUtils';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
type CodecCapabilityReport,
@@ -17,6 +18,8 @@ import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabili
import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
import {loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
import {
clearScreenShareDecodeFailures,
getScreenShareDecodeFailures,
getVideoDecoderExclusionsSync,
loadVideoDecoderExclusions,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
@@ -39,6 +42,7 @@ const RTP_PAYLOAD_TYPE_MAX = 255;
const CODEC_PRIORITY_MAX = 65_535;
const CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'vp9', 'h264', 'vp8', 'h265'];
const GECKO_SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['vp8', 'h264'];
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8'];
const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8';
const VIDEO_CODEC_NAMES: Record<VideoCodec, FluxerVideoCodecName> = {
@@ -163,6 +167,9 @@ function hasReceiverCapability(codec: VideoCodec): boolean | null {
function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
const exclusions = new Set(getVideoDecoderExclusionsSync() ?? []);
for (const codec of getScreenShareDecodeFailures()) {
exclusions.add(codec);
}
const result: Record<VideoCodec, boolean> = {
av1: false,
h265: false,
@@ -172,7 +179,7 @@ function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
};
for (const codec of CODEC_PREFERENCE) {
const advertised = hasReceiverCapability(codec);
result[codec] = advertised === null ? result[codec] : advertised && !exclusions.has(codec);
result[codec] = (advertised === null ? result[codec] : advertised) && !exclusions.has(codec);
}
return result;
}
@@ -181,8 +188,11 @@ export function getScreenShareCodecPreferenceOrder(
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
): ReadonlyArray<VideoCodec> {
const encoderMode = resolveEffectiveScreenShareEncoderMode(VoiceSettings.getScreenShareEncoderMode());
const automaticOrder =
encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder();
const automaticOrder = isFirefoxBrowser()
? GECKO_SOFTWARE_CODEC_PREFERENCE
: encoderMode === 'software'
? SOFTWARE_CODEC_PREFERENCE
: getHardwareFirstScreenShareCodecPreferenceOrder();
const order =
preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder;
return order.filter((codec) => isVideoCodecAllowedForPublish(codec));
@@ -537,6 +547,7 @@ class ScreenShareCodecNegotiation {
}
private canUseSelectedCodecForCurrentParticipants(codec: VideoCodec): boolean {
if (!isVideoCodecAllowedForPublish(codec)) return false;
if (!this.canLocalEncode(codec)) return false;
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs();
if (unknownParticipants > 0) return false;
@@ -631,6 +642,7 @@ class ScreenShareCodecNegotiation {
this.remoteCodecsByIdentity.clear();
this.mediaSessionId = createId('media');
this.negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
clearScreenShareDecodeFailures();
}
async publishLocalCapabilities(
@@ -665,7 +677,7 @@ class ScreenShareCodecNegotiation {
},
codecs: this.localCodecs,
rtc_connection_id: this.rtcConnectionId,
experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'],
experiments: [],
},
};
await this.publishMessage(room, message);
@@ -0,0 +1,166 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {beforeEach, describe, expect, it, vi} from 'vitest';
const gpuReport: HardwareEncodeReport = {
av1: 'hardware',
h265: 'hardware',
h264: 'hardware',
vp9: 'software',
vp8: 'software',
};
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
getScreenShareAv1OptIn: () => false,
getScreenShareHevcOptIn: () => false,
getPreferredScreenShareCodec: () => 'auto',
getScreenShareEncoderMode: () => 'auto',
},
}));
vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
getCachedDesktopTroubleshootingSettings: () => null,
}));
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
isFirefoxBrowser: () => false,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
getGpuEncoderReportSync: () => gpuReport,
loadGpuEncoderReport: async () => gpuReport,
}));
vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
getNativeHardwareEncoderCapabilitiesSync: () => null,
hasNativeHardwareEncoder: () => false,
resetNativeHardwareEncoderCapabilities: () => undefined,
loadNativeHardwareEncoderCapabilities: async () => null,
}));
vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
getOpenH264StatusSync: () => null,
resetOpenH264Status: () => undefined,
loadOpenH264Status: async () => null,
}));
const VIDEO_CAPABILITIES = {
codecs: [
{mimeType: 'video/VP8'},
{mimeType: 'video/VP9'},
{mimeType: 'video/H264'},
{mimeType: 'video/H265'},
{mimeType: 'video/AV1'},
],
};
Object.defineProperty(globalThis, 'RTCRtpSender', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
Object.defineProperty(globalThis, 'RTCRtpReceiver', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
const {
default: ScreenShareCodecNegotiation,
buildLocalCodecAdvertisements,
computeNegotiatedVideoCodec,
} = await import('./ScreenShareCodecNegotiation');
const {findStalledVideoDecoder} = await import('@app/features/voice/utils/ScreenShareCodecDiagnostics');
const {getVideoDecoderExclusionsSync, markScreenShareDecodeFailure, resetVideoDecoderExclusions} = await import(
'@app/features/voice/utils/VideoDecoderCapabilities'
);
const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
function stalledScreenShareStats(mimeType: string): RTCStatsReport {
const entries: Array<Record<string, unknown>> = [
{id: 'codec-1', type: 'codec', mimeType},
{
id: 'inbound-1',
type: 'inbound-rtp',
kind: 'video',
codecId: 'codec-1',
packetsReceived: 4200,
bytesReceived: 3_500_000,
framesReceived: 180,
framesDecoded: 0,
},
];
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
}
function decodeAdvertisedFor(name: 'H264' | 'VP8' | 'VP9'): boolean | undefined {
return buildLocalCodecAdvertisements().find((codec) => codec.name === name)?.decode;
}
function publisherThatEncodes(): Array<FluxerCodecAdvertisement> {
return [
{name: 'H264', type: 'video', payload_type: 102, priority: 1, encode: true, decode: true},
{name: 'VP9', type: 'video', payload_type: 109, priority: 2, encode: true, decode: true},
];
}
describe('a stalled H.264 screen share decode', () => {
beforeEach(() => {
resetVideoDecoderExclusions();
resetCachedCodecCapabilities();
ScreenShareCodecNegotiation.dispose();
});
it('is reported as an h264 decode stall by the stats diagnostic', () => {
expect(findStalledVideoDecoder(stalledScreenShareStats('video/H264'))?.codec).toBe('h264');
});
it('withdraws the local H.264 decode advertisement so the publisher renegotiates away from it', () => {
const stall = findStalledVideoDecoder(stalledScreenShareStats('video/H264'));
if (!stall) throw new Error('expected the stats diagnostic to report a stalled decoder');
expect(stall.codec).toBe('h264');
expect(decodeAdvertisedFor('H264')).toBe(true);
expect(
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
).toBe('h264');
expect(markScreenShareDecodeFailure(stall.codec, 'screen-share-decode-stalled')).toBe(true);
expect(decodeAdvertisedFor('H264')).toBe(false);
expect(
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
).toBe('vp9');
});
it('never removes h264 or vp8 from the SDP-level subscriber exclusions', () => {
markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled');
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('h264');
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('vp8');
});
it('keeps a non-baseline runtime stall out of the SDP-level subscriber exclusions too', () => {
expect(markScreenShareDecodeFailure('av1', 'screen-share-decode-stalled')).toBe(true);
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('av1');
});
it('restores the H.264 decode advertisement when the room is torn down', () => {
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
expect(decodeAdvertisedFor('H264')).toBe(false);
ScreenShareCodecNegotiation.dispose();
expect(decodeAdvertisedFor('H264')).toBe(true);
});
it('stops after one codec change instead of cycling through the remaining codecs', () => {
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(false);
expect(markScreenShareDecodeFailure('vp8', 'screen-share-decode-stalled')).toBe(false);
expect(decodeAdvertisedFor('VP8')).toBe(true);
});
});
@@ -1,7 +1,21 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
getLocalSpeakingThresholdRms,
LOCAL_MAX_RMS,
LOCAL_MIN_RMS,
SPEAKING_LOCAL_RELEASE_MS,
} from '@app/features/voice/engine/VoiceSpeakingThreshold';
import {describe, expect, it} from 'vitest';
import {type LocalSpeakingOverrideInput, resolveLocalSpeakingOverrideState} from './VoiceLocalSpeakingGate';
import {
type LocalSpeakingOverrideInput,
resolveLocalSpeakingOverrideState,
resolveVoiceActivityGateState,
resolveVoiceActivityThresholdRms,
updateVoiceActivityNoiseFloorRms,
type VoiceActivityGateInput,
type VoiceActivityGateState,
} from './VoiceLocalSpeakingGate';
const UNMUTED_OPEN_MIC: LocalSpeakingOverrideInput = {
pushToTalkActive: false,
@@ -70,3 +84,131 @@ describe('resolveLocalSpeakingOverrideState', () => {
).toBe(false);
});
});
const GATE_TICK_MS = 50;
const KEYBOARD_NOISE_RMS = 0.012;
const SPEECH_RMS = 0.06;
function runGate(samples: ReadonlyArray<number>, thresholdRms: number): Array<boolean> {
let open = false;
let silenceStartedAtMs: number | null = null;
return samples.map((rms, index) => {
const input: VoiceActivityGateInput = {
rms,
thresholdRms,
nowMs: index * GATE_TICK_MS,
silenceStartedAtMs,
gateOpen: open,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
};
const next = resolveVoiceActivityGateState(input);
open = next.open;
silenceStartedAtMs = next.silenceStartedAtMs;
return open;
});
}
describe('resolveVoiceActivityGateState', () => {
it('keeps keyboard noise off the wire when the activity threshold is raised', () => {
const opened = runGate(new Array(20).fill(KEYBOARD_NOISE_RMS), getLocalSpeakingThresholdRms(100));
expect(opened.some((open) => open)).toBe(false);
});
it('still transmits the same noise when the activity threshold is lowered', () => {
const opened = runGate(new Array(20).fill(KEYBOARD_NOISE_RMS), getLocalSpeakingThresholdRms(0));
expect(opened.every((open) => open)).toBe(true);
});
it('opens on the first sample at or above the threshold and clears the silence clock', () => {
const expected: VoiceActivityGateState = {open: true, silenceStartedAtMs: null};
expect(
resolveVoiceActivityGateState({
rms: SPEECH_RMS,
thresholdRms: getLocalSpeakingThresholdRms(100),
nowMs: 1_000,
silenceStartedAtMs: 800,
gateOpen: false,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}),
).toEqual(expected);
});
it('holds the gate open for the release window before closing', () => {
const threshold = getLocalSpeakingThresholdRms(100);
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000,
silenceStartedAtMs: null,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}),
).toEqual({open: true, silenceStartedAtMs: 1_000});
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000 + SPEAKING_LOCAL_RELEASE_MS - 1,
silenceStartedAtMs: 1_000,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}).open,
).toBe(true);
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000 + SPEAKING_LOCAL_RELEASE_MS,
silenceStartedAtMs: 1_000,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}).open,
).toBe(false);
});
it('closes after speech stops and reopens when speech returns', () => {
const threshold = getLocalSpeakingThresholdRms(100);
const opened = runGate(
[SPEECH_RMS, SPEECH_RMS, ...new Array(6).fill(KEYBOARD_NOISE_RMS), SPEECH_RMS, KEYBOARD_NOISE_RMS],
threshold,
);
expect(opened).toEqual([true, true, true, true, true, true, false, false, true, true]);
});
});
describe('resolveVoiceActivityThresholdRms', () => {
it('follows the activity threshold slider when auto sensitivity is off', () => {
expect(resolveVoiceActivityThresholdRms({autoSensitivity: false, vadThreshold: 100, noiseFloorRms: 0})).toBe(
getLocalSpeakingThresholdRms(100),
);
expect(resolveVoiceActivityThresholdRms({autoSensitivity: false, vadThreshold: 0, noiseFloorRms: 0})).toBe(
getLocalSpeakingThresholdRms(0),
);
});
it('ignores the slider and tracks the measured noise floor when auto sensitivity is on', () => {
expect(
resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 0, noiseFloorRms: 0.005}),
).toBeCloseTo(0.015, 6);
});
it('clamps the auto threshold to the local speaking range', () => {
expect(resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 50, noiseFloorRms: 0})).toBe(
LOCAL_MIN_RMS,
);
expect(resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 50, noiseFloorRms: 1})).toBe(
LOCAL_MAX_RMS,
);
});
});
describe('updateVoiceActivityNoiseFloorRms', () => {
it('converges on the level measured while the gate is closed', () => {
let noiseFloorRms = 0;
for (let i = 0; i < 200; i++) {
noiseFloorRms = updateVoiceActivityNoiseFloorRms(noiseFloorRms, KEYBOARD_NOISE_RMS);
}
expect(noiseFloorRms).toBeCloseTo(KEYBOARD_NOISE_RMS, 4);
});
});
@@ -1,5 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
getLocalSpeakingThresholdRms,
LOCAL_MAX_RMS,
LOCAL_MIN_RMS,
} from '@app/features/voice/engine/VoiceSpeakingThreshold';
const VOICE_ACTIVITY_NOISE_FLOOR_HEADROOM = 3;
const VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING = 0.05;
export interface LocalSpeakingOverrideInput {
pushToTalkActive: boolean;
pushToMuteActive: boolean;
@@ -18,3 +27,45 @@ export function resolveLocalSpeakingOverrideState(input: LocalSpeakingOverrideIn
if (effectivelyMuted) return false;
return null;
}
export interface VoiceActivityGateInput {
rms: number;
thresholdRms: number;
nowMs: number;
silenceStartedAtMs: number | null;
gateOpen: boolean;
releaseDelayMs: number;
}
export interface VoiceActivityGateState {
open: boolean;
silenceStartedAtMs: number | null;
}
export function resolveVoiceActivityGateState(input: VoiceActivityGateInput): VoiceActivityGateState {
if (input.rms >= input.thresholdRms) {
return {open: true, silenceStartedAtMs: null};
}
const silenceStartedAtMs = input.silenceStartedAtMs ?? input.nowMs;
return {
open: input.gateOpen && input.nowMs - silenceStartedAtMs < input.releaseDelayMs,
silenceStartedAtMs,
};
}
export function resolveVoiceActivityThresholdRms(input: {
autoSensitivity: boolean;
vadThreshold: number;
noiseFloorRms: number;
}): number {
if (!input.autoSensitivity) {
return getLocalSpeakingThresholdRms(input.vadThreshold);
}
const tracked = input.noiseFloorRms * VOICE_ACTIVITY_NOISE_FLOOR_HEADROOM;
if (!Number.isFinite(tracked)) return LOCAL_MIN_RMS;
return Math.min(LOCAL_MAX_RMS, Math.max(LOCAL_MIN_RMS, tracked));
}
export function updateVoiceActivityNoiseFloorRms(noiseFloorRms: number, rms: number): number {
return noiseFloorRms * (1 - VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING) + rms * VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING;
}
@@ -18,7 +18,7 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import {scheduleScreenShareDecoderVerification} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
import {markVideoDecoderRuntimeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
import type {
LocalParticipant,
@@ -266,7 +266,7 @@ export function bindRoomEvents(
screenShareDecoderVerificationTimers.delete(trackSid);
},
(failure) => {
if (!markVideoDecoderRuntimeFailure(failure.codec, 'screen-share-decode-stalled')) return;
if (!markScreenShareDecodeFailure(failure.codec, 'screen-share-decode-stalled')) return;
void ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual').catch((error) => {
logger.warn('Failed to publish updated codec capabilities after decode stall', {
error,
@@ -1,8 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const LOCAL_MIN_RMS = 0.003;
export const LOCAL_MIN_RMS = 0.003;
const LOCAL_DEFAULT_RMS = 0.008;
const LOCAL_MAX_RMS = 0.04;
export const LOCAL_MAX_RMS = 0.04;
const REMOTE_MIN_RMS = 0.002;
const REMOTE_DEFAULT_RMS = 0.006;
const REMOTE_MAX_RMS = 0.035;
@@ -3,6 +3,7 @@
import assert from 'node:assert/strict';
import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {isVoiceActivityGateEnabled} from '@app/features/voice/utils/VoiceInputProcessor';
import {
getActiveInputDeviceLabel,
resolveVoiceProcessingFromStateForDeviceLabel,
@@ -13,6 +14,8 @@ export interface VoiceEngineV2AppAudioSettingsSnapshot {
readonly inputVolume: number;
readonly outputVolume: number;
readonly vadThreshold: number;
readonly vadAutoSensitivity: boolean;
readonly voiceActivityGate: boolean;
readonly requestedInputDeviceId: string;
readonly effectiveInputDeviceId: string;
readonly activeInputDeviceLabel: string | null;
@@ -39,6 +42,8 @@ function assertAudioSettingsSnapshot(snapshot: VoiceEngineV2AppAudioSettingsSnap
assert.equal(typeof snapshot.inputVolume, 'number', `${name}.inputVolume must be a number`);
assert.equal(typeof snapshot.outputVolume, 'number', `${name}.outputVolume must be a number`);
assert.equal(typeof snapshot.vadThreshold, 'number', `${name}.vadThreshold must be a number`);
assert.equal(typeof snapshot.vadAutoSensitivity, 'boolean', `${name}.vadAutoSensitivity must be a boolean`);
assert.equal(typeof snapshot.voiceActivityGate, 'boolean', `${name}.voiceActivityGate must be a boolean`);
assert.equal(typeof snapshot.requestedInputDeviceId, 'string', `${name}.requestedInputDeviceId must be a string`);
assert.equal(typeof snapshot.effectiveInputDeviceId, 'string', `${name}.effectiveInputDeviceId must be a string`);
assert.equal(typeof snapshot.echoCancellation, 'boolean', `${name}.echoCancellation must be a boolean`);
@@ -54,6 +59,8 @@ export function createVoiceEngineV2AppAudioSettingsSnapshot(): VoiceEngineV2AppA
inputVolume: VoiceSettings.getInputVolume(),
outputVolume: VoiceSettings.getOutputVolume(),
vadThreshold: VoiceSettings.getVadThreshold(),
vadAutoSensitivity: VoiceSettings.getVadAutoSensitivity(),
voiceActivityGate: isVoiceActivityGateEnabled(),
requestedInputDeviceId: VoiceSettings.getInputDeviceId(),
effectiveInputDeviceId: resolveEffectiveInputDeviceId(),
activeInputDeviceLabel,
@@ -90,5 +97,7 @@ export function hasVoiceEngineV2InputProcessorSettingsChanged(
assertAudioSettingsSnapshot(current, 'current');
if (previous.deepFilter !== current.deepFilter) return true;
if (previous.deepFilterNoiseReductionLevel !== current.deepFilterNoiseReductionLevel) return true;
if (previous.voiceActivityGate !== current.voiceActivityGate) return true;
if (previous.vadAutoSensitivity !== current.vadAutoSensitivity) return true;
return false;
}
@@ -806,7 +806,12 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
assertObjectLike<VoiceEngineV2MicrophoneOptions>(options, 'enableMicrophone.options');
if (this.microphoneEnablePromise) {
await this.microphoneEnablePromise;
return;
if (this.hasLiveMicrophonePublication(room) || this.microphoneEnablePromise !== null) {
return;
}
logger.warn('Coalesced microphone enable left no live publication for this room; enabling again', {
channelId,
});
}
this.microphoneEnablePromise = this.enableMicrophoneNow(room, channelId, options);
try {
@@ -1524,6 +1529,9 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
this.syncVoiceState({self_mute: targetMute});
this.updateMediaAudioControls();
this.syncLocalSpeakingOverride(room);
void this.refreshLocalVoiceInputProcessor(room).catch((error) => {
logger.warn('Failed to refresh voice input processor after transmit mode change', {error});
});
}
getMuteReason(voiceState: VoiceState | null, guildId?: string | null, channelId?: string | null): VoiceMuteReason {
@@ -0,0 +1,146 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {installVoiceMenuTestBootstrap} from '@app/features/ui/action_menu/items/__fixtures__/VoiceMenuTestBootstrap';
import type {Room} from 'livekit-client';
import {expect, test, vi} from 'vitest';
vi.mock('@lingui/core/macro', () => {
const descriptor = (value: unknown): unknown => (typeof value === 'string' ? {message: value} : value);
return {msg: descriptor, t: descriptor, plural: () => '', select: () => '', selectOrdinal: () => ''};
});
vi.mock('@lingui/react/macro', () => ({
Trans: () => null,
useLingui: () => ({i18n: {_: (descriptor: {message?: string}) => descriptor.message ?? '', locale: 'en'}}),
}));
installVoiceMenuTestBootstrap();
await import('@app/features/voice/engine/MediaEngineFacade');
const {VoiceEngineV2AppMediaExecutionAdapter} = await import(
'@app/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter'
);
interface FakeRoom {
audioTrackPublications: Map<string, unknown>;
localParticipant: {audioTrackPublications: Map<string, unknown>};
}
function createFakeRoom(): FakeRoom {
const audioTrackPublications = new Map<string, unknown>();
return {audioTrackPublications, localParticipant: {audioTrackPublications}};
}
function publishMicrophone(room: FakeRoom): void {
room.audioTrackPublications.set('mic', {
source: 'microphone',
track: {mediaStreamTrack: {readyState: 'live'}},
});
}
function unpublishMicrophone(room: FakeRoom): void {
room.audioTrackPublications.delete('mic');
}
function asRoom(room: FakeRoom): Room {
return room as unknown as Room;
}
function createDeferred(): {promise: Promise<void>; resolve: () => void} {
let resolve = (): void => undefined;
const promise = new Promise<void>((res) => {
resolve = () => res();
});
return {promise, resolve};
}
function createAdapter(onEnable: (room: FakeRoom, channelId: string | null) => Promise<void>): {
adapter: InstanceType<typeof VoiceEngineV2AppMediaExecutionAdapter>;
calls: Array<{room: FakeRoom; channelId: string | null}>;
} {
const adapter = new VoiceEngineV2AppMediaExecutionAdapter();
const calls: Array<{room: FakeRoom; channelId: string | null}> = [];
(
adapter as unknown as {enableMicrophoneNow: (room: Room, channelId: string | null) => Promise<void>}
).enableMicrophoneNow = async (room: Room, channelId: string | null): Promise<void> => {
const fakeRoom = room as unknown as FakeRoom;
calls.push({room: fakeRoom, channelId});
await onEnable(fakeRoom, channelId);
};
return {adapter, calls};
}
test('an enable for a second room is not satisfied by an enable in flight for the first room', async () => {
const roomA = createFakeRoom();
const roomB = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (room) => {
await gate.promise;
publishMicrophone(room);
});
const first = adapter.enableMicrophone(asRoom(roomA), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(roomB), 'channel-b');
gate.resolve();
await Promise.all([first, second]);
expect(calls.map((call) => call.channelId)).toEqual(['channel-a', 'channel-b']);
expect(roomA.audioTrackPublications.size).toBe(1);
expect(roomB.audioTrackPublications.size).toBe(1);
});
test('an enable is not satisfied by an in-flight enable whose publication was already unpublished', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (target) => {
publishMicrophone(target);
await gate.promise;
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
unpublishMicrophone(room);
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await Promise.all([first, second]);
expect(calls).toHaveLength(2);
expect(room.audioTrackPublications.size).toBe(1);
});
test('a coalesced enable that already has a live publication does not enable again', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (target) => {
publishMicrophone(target);
await gate.promise;
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await Promise.all([first, second]);
expect(calls).toHaveLength(1);
expect(room.audioTrackPublications.size).toBe(1);
});
test('a rejection from the in-flight enable still propagates to the coalesced caller', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async () => {
await gate.promise;
throw new Error('enable failed');
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await expect(first).rejects.toThrow('enable failed');
await expect(second).rejects.toThrow('enable failed');
expect(calls).toHaveLength(1);
});
@@ -1,31 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export interface ScreenShareAudioPumpDiagnostics {
active: boolean;
captureId: string | null;
sampleRate: number | null;
channels: number | null;
usesNativeSink: boolean;
publishStrategy: 'eager' | 'lazy' | 'none';
publishedFormatKey: string | null;
eagerPublish: 'succeeded' | 'failed' | 'skipped' | null;
eagerPublishError: string | null;
droppedPushFrames: number;
pendingPushFrames: number;
}
export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics {
return {
active: false,
captureId: null,
sampleRate: null,
channels: null,
usesNativeSink: false,
publishStrategy: 'none',
publishedFormatKey: null,
eagerPublish: null,
eagerPublishError: null,
droppedPushFrames: 0,
pendingPushFrames: 0,
};
}
@@ -0,0 +1,337 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
type PerTrackStats,
VoiceEngineV2AppStatsHostAdapter,
type VoiceEngineV2AppStatsHostAdapterScheduler,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
import {classifyVoiceEngineV2TrackStats} from '@fluxer/voice_engine_v2';
import type {Room} from 'livekit-client';
import {describe, expect, it} from 'vitest';
interface FakeScheduler extends VoiceEngineV2AppStatsHostAdapterScheduler {
runStatsTick(): void;
}
function createScheduler(): FakeScheduler {
const handlers = new Map<number, () => void>();
return {
setInterval(handler, intervalMs) {
handlers.set(intervalMs, handler);
return intervalMs;
},
clearInterval(handle) {
handlers.delete(handle as number);
},
runStatsTick() {
handlers.get(2000)?.();
},
};
}
function chromiumCameraSimulcastReports(bytesSentByRid: Record<string, number>): Map<string, unknown> {
return new Map<string, unknown>([
[
'SV1',
{
type: 'media-source',
id: 'SV1',
kind: 'video',
trackIdentifier: 'cam-track',
width: 1280,
height: 720,
framesPerSecond: 30,
frames: 1801,
},
],
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
[
'OT01V1',
{
type: 'outbound-rtp',
id: 'OT01V1',
kind: 'video',
ssrc: 1001,
rid: 'q',
mid: '2',
active: false,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.q,
framesEncoded: 1,
targetBitrate: 0,
qualityLimitationReason: 'none',
},
],
[
'OT01V2',
{
type: 'outbound-rtp',
id: 'OT01V2',
kind: 'video',
ssrc: 1002,
rid: 'h',
mid: '2',
active: false,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.h,
framesEncoded: 1,
targetBitrate: 0,
qualityLimitationReason: 'none',
},
],
[
'OT01V3',
{
type: 'outbound-rtp',
id: 'OT01V3',
kind: 'video',
ssrc: 1003,
rid: 'f',
mid: '2',
active: true,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.f,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
targetBitrate: 1133000,
qualityLimitationReason: 'none',
},
],
]);
}
async function collectChromiumCameraRows(
samples: ReadonlyArray<{now: number; bytesSentByRid: Record<string, number>}>,
): Promise<Array<PerTrackStats>> {
const scheduler = createScheduler();
let sampleIndex = 0;
let now = samples[0].now;
const publisher = {
getStats() {
return Promise.resolve(chromiumCameraSimulcastReports(samples[sampleIndex].bytesSentByRid));
},
};
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => now, scheduler});
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
adapter.startStatsTracking();
for (sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
now = samples[sampleIndex].now;
scheduler.runStatsTick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
const rows = adapter.perTrackStats;
adapter.cleanup();
return rows;
}
describe('VoiceEngineV2AppStatsHostAdapter chromium outbound rows', () => {
it('collects the outbound-rtp active flag for every simulcast layer', async () => {
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
expect(rows.map((row) => row.rid)).toEqual(['q', 'h', 'f']);
expect(rows.map((row) => row.active)).toEqual([false, false, true]);
});
it('omits the bitrate window on a first observation so a zero bitrate is not read as measured', async () => {
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([undefined, undefined, undefined]);
});
it('reports the bitrate window once a delta was measured so a zero bitrate is falsifiable', async () => {
const rows = await collectChromiumCameraRows([
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
{now: 3001, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
]);
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
});
it('keeps every pre-existing chromium row field byte-identical', async () => {
const rows = await collectChromiumCameraRows([
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
{now: 3000, bytesSentByRid: {q: 0, h: 0, f: 5_500_000}},
]);
const {active, bitrateWindowMs, ...legacy} = rows[2];
expect(legacy).toEqual({
direction: 'send',
kind: 'video',
ssrc: 1003,
rid: 'f',
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
codec: 'video/H264',
payloadType: 108,
bitrateKbps: 2000,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
sourceFrameWidth: 1280,
sourceFrameHeight: 720,
sourceFramesPerSecond: 30,
sourceFrames: 1801,
targetBitrateKbps: 1133,
qualityLimitationReason: 'none',
encoderAcceleration: 'unknown',
});
});
});
function firefoxCameraAndScreenShareReports(): Map<string, unknown> {
return new Map<string, unknown>([
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
['C2', {type: 'codec', id: 'C2', mimeType: 'audio/opus', payloadType: 109}],
[
'outbound_rtp_audio_1',
{
type: 'outbound-rtp',
id: 'outbound_rtp_audio_1',
kind: 'audio',
ssrc: 724915693,
mid: '1',
codecId: 'C2',
bytesSent: 100_000,
},
],
[
'outbound_rtp_video_3',
{
type: 'outbound-rtp',
id: 'outbound_rtp_video_3',
kind: 'video',
ssrc: 4089824766,
mid: '3',
codecId: 'C1',
bytesSent: 5_000_000,
framesEncoded: 900,
},
],
[
'outbound_rtp_video_4',
{
type: 'outbound-rtp',
id: 'outbound_rtp_video_4',
kind: 'video',
ssrc: 1477030229,
mid: '4',
codecId: 'C1',
bytesSent: 2_000_000,
framesEncoded: 600,
},
],
[
'inbound_rtp_audio_2',
{
type: 'inbound-rtp',
id: 'inbound_rtp_audio_2',
kind: 'audio',
ssrc: 1661789229,
mid: '2',
codecId: 'C2',
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
bytesReceived: 50_000,
},
],
]);
}
async function collectFirefoxRows(
transceivers: ReadonlyArray<{mid: string | null; sender?: {track?: {id: string} | null} | null}> | null,
): Promise<Array<PerTrackStats>> {
const scheduler = createScheduler();
const publisher = {
getStats() {
return Promise.resolve(firefoxCameraAndScreenShareReports());
},
...(transceivers === null ? {} : {getTransceivers: () => transceivers}),
};
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
adapter.startStatsTracking();
scheduler.runStatsTick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
const rows = adapter.perTrackStats;
adapter.cleanup();
return rows;
}
describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
it('backfills the outbound trackIdentifier from the transceiver mid when the report carries neither identity member', async () => {
const rows = await collectFirefoxRows([
{mid: '1', sender: {track: {id: '{mic-track}'}}},
{mid: '2', sender: {track: null}},
{mid: '3', sender: {track: {id: '{camera-track}'}}},
{mid: '4', sender: {track: {id: '{screen-track}'}}},
]);
expect(rows.map((row) => [row.direction, row.kind, row.mid, row.trackIdentifier])).toEqual([
['send', 'audio', '1', '{mic-track}'],
['send', 'video', '3', '{camera-track}'],
['send', 'video', '4', '{screen-track}'],
['recv', 'audio', '2', '{8a106956-d202-4916-8e19-0a1672c64b73}'],
]);
});
it('leaves an outbound row unidentified when no transceiver carries its mid', async () => {
const rows = await collectFirefoxRows([{mid: '3', sender: {track: {id: '{camera-track}'}}}]);
expect(rows.map((row) => row.trackIdentifier)).toEqual([
undefined,
'{camera-track}',
undefined,
'{8a106956-d202-4916-8e19-0a1672c64b73}',
]);
});
it('keeps the inbound trackIdentifier when the source exposes no transceivers at all', async () => {
const rows = await collectFirefoxRows(null);
expect(rows.map((row) => row.trackIdentifier)).toEqual([
undefined,
undefined,
undefined,
'{8a106956-d202-4916-8e19-0a1672c64b73}',
]);
});
it('lets the classifier tell a firefox camera apart from a simultaneous screen share', async () => {
const rows = await collectFirefoxRows([
{mid: '1', sender: {track: {id: '{mic-track}'}}},
{mid: '3', sender: {track: {id: '{camera-track}'}}},
{mid: '4', sender: {track: {id: '{screen-track}'}}},
]);
const classification = classifyVoiceEngineV2TrackStats({
tracks: rows.map((row) => ({
direction: row.direction,
kind: row.kind,
rid: row.rid,
trackIdentifier: row.trackIdentifier,
bitrateKbps: row.bitrateKbps,
})),
publications: {
localCameraTrackId: '{camera-track}',
localMicrophoneTrackId: '{mic-track}',
localScreenShareTrackId: '{screen-track}',
localScreenShareAudioTrackId: null,
remoteMicrophoneTrackIds: [],
remoteScreenShareTrackIds: [],
remoteScreenShareAudioTrackIds: [],
},
});
expect(rows[classification.localVideoTrackIndex!].mid).toBe('3');
expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
});
});
@@ -54,12 +54,14 @@ export interface PerTrackStats {
kind: 'audio' | 'video' | 'unknown';
ssrc?: number;
rid?: string;
active?: boolean;
mid?: string;
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
payloadType?: number;
bitrateKbps: number;
bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
jitterMs?: number;
@@ -160,6 +162,12 @@ type RoomWithEngine = Room & {
interface StatsSource {
getStats(): Promise<StatsReportMap>;
getTransceivers?(): ReadonlyArray<StatsTransceiver>;
}
interface StatsTransceiver {
mid: string | null;
sender?: {track?: {id: string} | null} | null;
}
interface StatsReportMap {
@@ -186,6 +194,7 @@ interface RTCStatsEntry {
payloadType?: number;
ssrc?: number;
rid?: string;
active?: boolean;
mid?: string;
trackId?: string;
trackIdentifier?: string;
@@ -248,6 +257,11 @@ const LATENCY_UPDATE_INTERVAL_MS = 2000;
const STATS_CLOCK_INTERVAL_MS = 1000;
const STATS_UPDATE_INTERVAL_MS = 2000;
interface BitrateSample {
bitrateKbps: number;
windowMs?: number;
}
function normalizeTrackKind(kind: string | undefined): PerTrackStats['kind'] {
if (kind === 'audio' || kind === 'video') return kind;
return 'unknown';
@@ -264,28 +278,32 @@ function getReportKind(report: RTCStatsEntry, reportsById: Map<string, RTCStatsE
return normalizeTrackKind(track?.kind ?? track?.mediaType ?? mediaSource?.kind ?? mediaSource?.mediaType);
}
function getBitrateKbps(
function getBitrateSample(
reportId: string,
currentBytes: number | undefined,
now: number,
rtpCounters: Map<string, VoiceStatsRtpCounter>,
): number {
): BitrateSample {
if (typeof currentBytes !== 'number' || !Number.isFinite(currentBytes)) {
return 0;
return {bitrateKbps: 0};
}
const previous = rtpCounters.get(reportId);
let bitrateKbps = 0;
let windowMs: number | undefined;
if (previous?.bytes !== undefined) {
const dt = (now - previous.timestamp) / 1000;
const db = currentBytes - previous.bytes;
if (dt > 0 && db >= 0) bitrateKbps = (db * 8) / 1000 / dt;
if (dt > 0 && db >= 0) {
bitrateKbps = (db * 8) / 1000 / dt;
windowMs = Math.round(now - previous.timestamp);
}
}
rtpCounters.set(reportId, {
...previous,
bytes: currentBytes,
timestamp: now,
});
return bitrateKbps;
return {bitrateKbps, windowMs};
}
function hasUsableLossDeltas(lostDelta: number, receivedDelta: number): boolean {
@@ -344,6 +362,7 @@ function hasUsableJitterBuffer(report: RTCStatsEntry): boolean {
function buildOutboundTrackExtras(report: RTCStatsEntry): Partial<PerTrackStats> {
return {
active: report.active,
retransmittedPacketsSent: report.retransmittedPacketsSent,
retransmittedBytesSent: report.retransmittedBytesSent,
keyFramesEncoded: report.keyFramesEncoded,
@@ -379,11 +398,12 @@ function buildPerTrackStat(args: {
now: number;
rtpCounters: Map<string, VoiceStatsRtpCounter>;
reportsById: Map<string, RTCStatsEntry>;
midToSenderTrackId: Map<string, string>;
}): PerTrackStats {
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById} = args;
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId} = args;
const id = `${sourceId}:${report.type}:${report.id}`;
const currentBytes = isOutbound ? report.bytesSent : report.bytesReceived;
const bitrateKbps = getBitrateKbps(id, currentBytes, now, rtpCounters);
const bitrate = getBitrateSample(id, currentBytes, now, rtpCounters);
const codec = report.codecId ? reportsById.get(report.codecId) : undefined;
const mediaSource = report.mediaSourceId ? reportsById.get(report.mediaSourceId) : undefined;
const packetsLossPercent = isInbound ? getPacketLossPercent(id, report, now, rtpCounters) : undefined;
@@ -402,11 +422,15 @@ function buildPerTrackStat(args: {
ssrc: report.ssrc,
rid: report.rid,
mid: report.mid,
trackIdentifier: report.trackIdentifier ?? mediaSource?.trackIdentifier,
trackIdentifier:
report.trackIdentifier ??
mediaSource?.trackIdentifier ??
(isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
mediaSourceId: report.mediaSourceId,
codec: codec?.mimeType,
payloadType: codec?.payloadType,
bitrateKbps: Math.round(bitrateKbps),
bitrateKbps: Math.round(bitrate.bitrateKbps),
bitrateWindowMs: bitrate.windowMs,
packetsLost: report.packetsLost,
packetsLossPercent: packetsLossPercent !== undefined ? Math.round(packetsLossPercent * 10) / 10 : undefined,
jitterMs: typeof report.jitter === 'number' ? Math.round(report.jitter * 1000 * 10) / 10 : undefined,
@@ -485,6 +509,11 @@ async function collectFromStatsSource(
transport: TransportInfo | null;
}> {
const reports = await source.getStats();
const midToSenderTrackId = new Map<string, string>();
for (const transceiver of source.getTransceivers?.() ?? []) {
const senderTrackId = transceiver.sender?.track?.id;
if (transceiver.mid && senderTrackId) midToSenderTrackId.set(transceiver.mid, senderTrackId);
}
const reportsById = new Map<string, RTCStatsEntry>();
for (const report of reports.values()) {
const statsEntry = report as RTCStatsEntry;
@@ -511,7 +540,9 @@ async function collectFromStatsSource(
if (!isOutbound && !isInbound) continue;
const id = `${sourceId}:${report.type}:${report.id}`;
activeCounterIds.add(id);
tracks.push(buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById}));
tracks.push(
buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId}),
);
}
if (!activePair && transportReport?.selectedCandidatePairId) {
const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
@@ -0,0 +1,61 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import {describe, expect, test, vi} from 'vitest';
vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
function makeStats(entries: ReadonlyArray<Record<string, unknown>>): RTCStatsReport {
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
}
const CODEC_REPORT = {type: 'codec', id: 'codec-h264', mimeType: 'video/H264'};
const SOURCE_REPORT = {type: 'media-source', id: 'source-1', kind: 'video', frames: 240, framesPerSecond: 30};
function outboundReport(overrides: Record<string, unknown>): Record<string, unknown> {
return {
type: 'outbound-rtp',
id: 'outbound-1',
kind: 'video',
codecId: 'codec-h264',
mediaSourceId: 'source-1',
framesEncoded: 0,
framesSent: 0,
...overrides,
};
}
describe('findStalledVideoEncoder', () => {
test('reports a stall when the encoding is active and the source is producing frames', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: true})]);
expect(findStalledVideoEncoder(stats, 'h264')).toEqual({
codec: 'h264',
framesEncoded: 0,
framesSent: 0,
sourceFrames: 240,
sourceFramesPerSecond: 30,
});
});
test('ignores an encoding the SFU deactivated even though the source is producing frames', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: false})]);
expect(findStalledVideoEncoder(stats, 'h264')).toBeNull();
});
test('still reports a stall when active is absent, as on Firefox', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({})]);
expect(findStalledVideoEncoder(stats, 'h264')?.codec).toBe('h264');
});
test('skips a deactivated layer and reports the still-active sibling instead', () => {
const stats = makeStats([
CODEC_REPORT,
SOURCE_REPORT,
{type: 'media-source', id: 'source-low', kind: 'video', frames: 90, framesPerSecond: 15},
outboundReport({id: 'outbound-low', mediaSourceId: 'source-low', active: false}),
outboundReport({id: 'outbound-high', active: true}),
]);
expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240);
});
});
@@ -469,6 +469,7 @@ interface OutboundVideoStatsEntry {
mediaType?: string;
codecId?: string;
mediaSourceId?: string;
active?: boolean;
framesEncoded?: number;
framesSent?: number;
encoderImplementation?: string;
@@ -592,6 +593,7 @@ export function findStalledVideoEncoder(stats: RTCStatsReport, codec?: VideoCode
}
for (const report of reports) {
if (getStatsKind(report, reportsById) !== 'video') continue;
if (report.active === false) continue;
const mimeType = report.codecId ? reportsById.get(report.codecId)?.mimeType : undefined;
if (report.codecId && !codecMatchesTarget(mimeType, codec)) continue;
const resolvedCodec = codec ?? getVideoCodecFromMimeType(mimeType);
@@ -0,0 +1,208 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {PiPContent} from '@app/features/ui/state/PiP';
import type {VoiceMediaGraphSnapshot} from '@app/features/voice/engine/VoiceMediaGraph';
import {type Room, Track} from 'livekit-client';
import {beforeEach, describe, expect, test, vi} from 'vitest';
const viewerStreamKeys = {current: [] as Array<string>};
const pipContent = {current: null as PiPContent | null};
const selfStream = {current: false};
const mediaEngine = {
room: null as Room | null,
participants: {} as Record<string, {identity: string; isScreenShareEnabled: boolean}>,
connectionVoiceStates: {} as Record<string, {channel_id: string}>,
channelId: null as string | null,
guildId: null as string | null,
connectionId: null as string | null,
subscribe: vi.fn(),
};
vi.mock('@app/features/platform/utils/AppLogger', () => ({
Logger: class {
debug = vi.fn();
info = vi.fn();
warn = vi.fn();
error = vi.fn();
},
}));
vi.mock('@app/features/platform/utils/DeferUntilModulesLoaded', () => ({
deferUntilModulesLoaded: vi.fn(),
}));
vi.mock('@app/features/navigation/state/SelectedChannel', () => ({
default: {currentChannelId: null},
}));
vi.mock('@app/features/ui/commands/PiPCommands', () => ({
openPiP: vi.fn(),
closePiP: vi.fn(),
}));
vi.mock('@app/features/ui/state/MobileLayout', () => ({
default: {isMobileLayout: () => false},
}));
vi.mock('@app/features/ui/state/PiP', () => ({
default: {
getContent: () => pipContent.current,
getSessionDisable: () => false,
},
}));
vi.mock('@app/features/user/state/Users', () => ({
default: {currentUser: {id: '1'}},
}));
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
default: mediaEngine,
}));
vi.mock('@app/features/voice/engine/VoiceMediaGraphStore', () => ({
voiceMediaGraphStore: {
getGraphSnapshot: () =>
({watchIntent: {viewerStreamKeys: viewerStreamKeys.current}}) as unknown as VoiceMediaGraphSnapshot,
subscribe: vi.fn(),
},
}));
vi.mock('@app/features/voice/state/LocalVoiceState', () => ({
default: {
getSelfStream: () => selfStream.current,
subscribe: vi.fn(),
},
}));
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {disablePictureInPicturePopoutScreenShare: false},
}));
const {detectScreenSharePiPContent} = await import('@app/features/voice/state/ScreenSharePiPController');
function makeParticipant(identity: string, sharing: boolean) {
const publications = new Map<string, {source: Track.Source; isMuted: boolean; trackSid: string}>();
publications.set('camera', {source: Track.Source.Camera, isMuted: false, trackSid: `${identity}-camera`});
if (sharing) {
publications.set('screen', {source: Track.Source.ScreenShare, isMuted: false, trackSid: `${identity}-screen`});
}
return {identity, trackPublications: publications};
}
function setRoom(remoteIdentities: Array<string>) {
mediaEngine.room = {
localParticipant: makeParticipant('user_1_connection-self', false),
remoteParticipants: new Map(remoteIdentities.map((identity) => [identity, makeParticipant(identity, true)])),
} as unknown as Room;
}
beforeEach(() => {
viewerStreamKeys.current = [];
pipContent.current = null;
selfStream.current = false;
mediaEngine.room = null;
mediaEngine.participants = {};
mediaEngine.connectionVoiceStates = {};
mediaEngine.channelId = 'channel-1';
mediaEngine.guildId = 'guild-1';
mediaEngine.connectionId = 'connection-self';
});
describe('detectScreenSharePiPContent', () => {
test('ignores a remote screen share the user is not watching', () => {
setRoom(['user_2_connection-2']);
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toBeNull();
});
test('returns a remote screen share the user is watching', () => {
setRoom(['user_2_connection-2']);
viewerStreamKeys.current = ['guild-1:channel-1:connection-2'];
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toEqual({
participantIdentity: 'user_2_connection-2',
userId: '2',
connectionId: 'connection-2',
source: 'livekit',
});
});
test('returns the watched share when several participants are sharing', () => {
setRoom(['user_2_connection-2', 'user_3_connection-3']);
viewerStreamKeys.current = ['guild-1:channel-1:connection-3'];
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toEqual({
participantIdentity: 'user_3_connection-3',
userId: '3',
connectionId: 'connection-3',
source: 'livekit',
});
});
test('ignores an unwatched participant snapshot screen share', () => {
mediaEngine.participants = {
'user_2_connection-2': {identity: 'user_2_connection-2', isScreenShareEnabled: true},
};
mediaEngine.connectionVoiceStates = {'connection-2': {channel_id: 'channel-1'}};
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toBeNull();
});
test('returns a watched participant snapshot screen share', () => {
mediaEngine.participants = {
'user_2_connection-2': {identity: 'user_2_connection-2', isScreenShareEnabled: true},
};
mediaEngine.connectionVoiceStates = {'connection-2': {channel_id: 'channel-1'}};
viewerStreamKeys.current = ['guild-1:channel-1:connection-2'];
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toEqual({
participantIdentity: 'user_2_connection-2',
userId: '2',
connectionId: 'connection-2',
source: 'participant-snapshot',
});
});
test('ignores an unwatched remote screen share in a DM call', () => {
setRoom(['user_2_connection-2']);
mediaEngine.guildId = null;
expect(detectScreenSharePiPContent('channel-1', null)).toBeNull();
});
test('returns a watched remote screen share in a DM call', () => {
setRoom(['user_2_connection-2']);
mediaEngine.guildId = null;
viewerStreamKeys.current = ['dm:channel-1:connection-2'];
expect(detectScreenSharePiPContent('channel-1', null)).toEqual({
participantIdentity: 'user_2_connection-2',
userId: '2',
connectionId: 'connection-2',
source: 'livekit',
});
});
test('keeps returning the local share while watch intent is empty', () => {
selfStream.current = true;
setRoom([]);
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toEqual({
participantIdentity: 'user_1_connection-self',
userId: '1',
connectionId: 'connection-self',
source: 'local-self-state',
});
});
test('keeps an open PiP alive for a watched stream that stopped publishing', () => {
pipContent.current = {
type: 'stream',
participantIdentity: 'user_2_connection-2',
channelId: 'channel-1',
guildId: 'guild-1',
connectionId: 'connection-2',
userId: '2',
};
viewerStreamKeys.current = ['guild-1:channel-1:connection-2'];
expect(detectScreenSharePiPContent('channel-1', 'guild-1')).toEqual({
participantIdentity: 'user_2_connection-2',
userId: '2',
connectionId: 'connection-2',
source: 'watched-stream',
});
});
});
@@ -54,7 +54,16 @@ function buildScreenShare(
return {...screenShare, source};
}
function pickLiveKitScreenShare(room: Room, channelId: string): ScreenSharePiPScreenShare | null {
function isWatchedScreenShare(guildId: string | null, channelId: string, connectionId: string): boolean {
const streamKey = getStreamKey(guildId, channelId, connectionId);
return selectVoiceMediaGraphViewerStreamKeys(voiceMediaGraphStore.getGraphSnapshot()).includes(streamKey);
}
function pickLiveKitScreenShare(
room: Room,
channelId: string,
guildId: string | null,
): ScreenSharePiPScreenShare | null {
const participants = [room.localParticipant, ...Array.from(room.remoteParticipants.values())];
for (const participant of participants) {
for (const publication of participant.trackPublications.values()) {
@@ -62,6 +71,7 @@ function pickLiveKitScreenShare(room: Room, channelId: string): ScreenSharePiPSc
if (publication.isMuted) continue;
const parsed = parseVoiceParticipantIdentity(participant.identity);
if (!parsed.userId || !parsed.connectionId) continue;
if (!isWatchedScreenShare(guildId, channelId, parsed.connectionId)) continue;
return buildScreenShare(
{
participantIdentity: participant.identity,
@@ -72,11 +82,13 @@ function pickLiveKitScreenShare(room: Room, channelId: string): ScreenSharePiPSc
);
}
}
void channelId;
return null;
}
function pickRemoteScreenShareFromSnapshots(channelId: string): ScreenSharePiPScreenShare | null {
function pickRemoteScreenShareFromSnapshots(
channelId: string,
guildId: string | null,
): ScreenSharePiPScreenShare | null {
const snapshots = MediaEngine.participants;
const connectionVoiceStates = MediaEngine.connectionVoiceStates;
for (const participantIdentity in snapshots) {
@@ -87,6 +99,7 @@ function pickRemoteScreenShareFromSnapshots(channelId: string): ScreenSharePiPSc
if (!parsed.userId || !parsed.connectionId) continue;
const voiceState = connectionVoiceStates[parsed.connectionId];
if (voiceState && voiceState.channel_id !== channelId) continue;
if (!isWatchedScreenShare(guildId, channelId, parsed.connectionId)) continue;
return buildScreenShare(
{
participantIdentity: snapshot.identity,
@@ -116,16 +129,16 @@ function pickLocalSelfShare(channelId: string): ScreenSharePiPScreenShare | null
);
}
function detectActiveScreenShare(channelId: string | null): ScreenSharePiPScreenShare | null {
function detectActiveScreenShare(channelId: string | null, guildId: string | null): ScreenSharePiPScreenShare | null {
if (!channelId) return null;
const local = pickLocalSelfShare(channelId);
if (local) return local;
const room = MediaEngine.room;
if (room) {
const fromRoom = pickLiveKitScreenShare(room, channelId);
const fromRoom = pickLiveKitScreenShare(room, channelId, guildId);
if (fromRoom) return fromRoom;
}
const fromSnapshots = pickRemoteScreenShareFromSnapshots(channelId);
const fromSnapshots = pickRemoteScreenShareFromSnapshots(channelId, guildId);
if (fromSnapshots) return fromSnapshots;
return null;
}
@@ -150,16 +163,16 @@ function pickWatchedScreenShareFromOpenPiP(
);
}
function detectScreenSharePiPContent(
export function detectScreenSharePiPContent(
channelId: string | null,
guildId: string | null,
): ScreenSharePiPScreenShare | null {
return detectActiveScreenShare(channelId) ?? pickWatchedScreenShareFromOpenPiP(channelId, guildId);
return detectActiveScreenShare(channelId, guildId) ?? pickWatchedScreenShareFromOpenPiP(channelId, guildId);
}
function getClosedReason(conditions: ScreenSharePiPConditions): string | null {
if (!conditions.connectedChannelId) return 'not-connected-to-voice';
if (!conditions.screenShare) return 'no-active-screen-share';
if (!conditions.screenShare) return 'no-watched-screen-share';
if (conditions.isMobile) return 'mobile-layout';
if (conditions.disabledBySetting) return 'disabled-by-setting';
if (conditions.disabledBySession) return 'disabled-for-session';
@@ -272,6 +285,7 @@ class ScreenSharePiPController {
);
MediaEngine.subscribe(() => this.recompute('media-engine-store'));
LocalVoiceState.subscribe(() => this.recompute('local-voice-state'));
voiceMediaGraphStore.subscribe(() => this.recompute('voice-media-graph'));
}
private readReactiveInputs(): ReactiveInputs {
@@ -122,6 +122,7 @@ type VoiceSettingsUpdate = Partial<{
screenShareAudioSourceMode: 'none' | 'system' | 'specific';
screenShareAudioIncludeSources: Array<Record<string, string>>;
screenShareAudioExcludeSources: Array<Record<string, string>>;
screenShareDeviceAudioUsesMicrophone: boolean;
openH264Enabled: boolean;
lastScreenShareSource: LastScreenShareSource | null;
}>;
@@ -434,6 +435,7 @@ class VoiceSettings {
screenShareAudioSourceMode: 'none' | 'system' | 'specific' = 'system';
screenShareAudioIncludeSources: Array<Record<string, string>> = [];
screenShareAudioExcludeSources: Array<Record<string, string>> = [];
screenShareDeviceAudioUsesMicrophone = false;
openH264Enabled = true;
lastScreenShareSource: LastScreenShareSource | null = null;
prioritizeSpeakingParticipants = false;
@@ -510,6 +512,7 @@ class VoiceSettings {
getScreenShareAudioSourceMode: false,
getScreenShareAudioIncludeSources: false,
getScreenShareAudioExcludeSources: false,
getScreenShareDeviceAudioUsesMicrophone: false,
getOpenH264Enabled: false,
getLastScreenShareSource: false,
getPrioritizeSpeakingParticipants: false,
@@ -623,6 +626,7 @@ class VoiceSettings {
'screenShareAudioSourceMode',
'screenShareAudioIncludeSources',
'screenShareAudioExcludeSources',
'screenShareDeviceAudioUsesMicrophone',
'openH264Enabled',
'lastScreenShareSource',
'prioritizeSpeakingParticipants',
@@ -1008,6 +1012,10 @@ class VoiceSettings {
return this.screenShareAudioExcludeSources;
}
getScreenShareDeviceAudioUsesMicrophone(): boolean {
return this.screenShareDeviceAudioUsesMicrophone;
}
getEffectiveScreenShareAudioSourceMode(): 'none' | 'system' | 'specific' {
return this.getScreenShareAudioSourceMode();
}
@@ -1126,6 +1134,8 @@ class VoiceSettings {
this.screenShareAudioIncludeSources = validated.screenShareAudioIncludeSources;
if (validated.screenShareAudioExcludeSources !== undefined)
this.screenShareAudioExcludeSources = validated.screenShareAudioExcludeSources;
if (validated.screenShareDeviceAudioUsesMicrophone !== undefined)
this.screenShareDeviceAudioUsesMicrophone = validated.screenShareDeviceAudioUsesMicrophone;
if (validated.openH264Enabled !== undefined) this.openH264Enabled = validated.openH264Enabled;
if (validated.lastScreenShareSource !== undefined) this.lastScreenShareSource = validated.lastScreenShareSource;
this.notifyListeners();
@@ -1257,6 +1267,8 @@ class VoiceSettings {
validateSourceList(data.screenShareAudioIncludeSources) ?? this.screenShareAudioIncludeSources,
screenShareAudioExcludeSources:
validateSourceList(data.screenShareAudioExcludeSources) ?? this.screenShareAudioExcludeSources,
screenShareDeviceAudioUsesMicrophone:
data.screenShareDeviceAudioUsesMicrophone ?? this.screenShareDeviceAudioUsesMicrophone,
openH264Enabled: data.openH264Enabled ?? this.openH264Enabled,
lastScreenShareSource:
data.lastScreenShareSource === undefined
@@ -0,0 +1,43 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import type {
NativeAudioBridgeEndedCapture,
NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
import {describe, expect, it} from 'vitest';
describe('getNativeAudioCaptureDiagnosticState', () => {
it('keeps the six fields real dumps already carry and appends the retained histories', () => {
expect(Object.keys(getNativeAudioCaptureDiagnosticState())).toEqual([
'armedCapture',
'activeBridge',
'supersededBridge',
'lastStartedCapture',
'lastArmFailure',
'bridgeStats',
'endedBridgeCaptures',
'lifecycleFaults',
]);
});
it('emits the superseded capture next to the live one instead of only the replacement', () => {
startBridgeStats('generator', 'native-audio:diag-failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
recordBridgeFrame('native-audio:diag-failed', {timestampUs: 0, durationUs: 10_000, peak: 0.31, rms: 0.08});
startBridgeStats('generator', 'native-audio:diag-live', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
const state = getNativeAudioCaptureDiagnosticState();
expect((state.bridgeStats as NativeAudioBridgeStats).captureId).toBe('native-audio:diag-live');
const ended = state.endedBridgeCaptures as Array<NativeAudioBridgeEndedCapture>;
const retained = ended.find((capture) => capture.captureId === 'native-audio:diag-failed');
expect(retained?.framesReceived).toBe(1);
expect(retained?.nonSilentFrameCount).toBe(1);
expect(retained?.endReason).toBe('superseded');
});
it('exposes a native-audio fault history that outlives the lifecycle unbind', () => {
expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true);
});
});
@@ -4,7 +4,10 @@ import {Logger} from '@app/features/platform/utils/AppLogger';
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge';
import {getNativeAudioErrorDetail} from '@app/features/voice/utils/NativeAudioFailureUtils';
import {getBridgeStats as getNativeAudioBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import {
getEndedBridgeCaptures,
getBridgeStats as getNativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import {createGeneratorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge';
import {createScriptProcessorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge';
import {
@@ -67,8 +70,18 @@ interface NativeAudioStartedCaptureDiagnostic {
includeSelfWindowAudio?: boolean;
}
interface NativeAudioLifecycleFaultDiagnostic {
captureId: string;
sourceId: string;
message: string;
atMs: number;
}
const MAX_RETAINED_LIFECYCLE_FAULTS = 8;
let lastStartedCapture: NativeAudioStartedCaptureDiagnostic | null = null;
let sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null;
let lifecycleFaults: Array<NativeAudioLifecycleFaultDiagnostic> = [];
const lifecycleBoundCaptureIds = new Set<string>();
export {getNativeAudioBridgeStats};
@@ -93,9 +106,19 @@ function bindNativeAudioCaptureLifecycle(captureId: string): void {
}
}
function recordLifecycleFault(captureId: string, message: string): void {
lifecycleFaults.push({captureId, sourceId: `native-audio-tap:${captureId}`, message, atMs: Date.now()});
if (lifecycleFaults.length > MAX_RETAINED_LIFECYCLE_FAULTS) {
lifecycleFaults = lifecycleFaults.slice(-MAX_RETAINED_LIFECYCLE_FAULTS);
}
}
function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void {
if (!lifecycleBoundCaptureIds.has(captureId)) return;
const bridge = sourceLifecycleBridge;
if (faulted) {
recordLifecycleFault(captureId, 'native-audio-tap-track-ended');
}
if (bridge) {
if (faulted) {
bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'});
@@ -167,6 +190,8 @@ export function getNativeAudioCaptureDiagnosticState(): Record<string, unknown>
lastStartedCapture: started,
lastArmFailure: getLastNativeAudioArmFailure(),
bridgeStats: getNativeAudioBridgeStats(),
endedBridgeCaptures: getEndedBridgeCaptures(),
lifecycleFaults: lifecycleFaults.map((fault) => ({...fault})),
};
}
@@ -1499,5 +1524,6 @@ export function resetNativeAudioCaptureBridgeForTests(): void {
activeBridge = null;
supersededBridge = null;
sourceLifecycleBridge = null;
lifecycleFaults = [];
lifecycleBoundCaptureIds.clear();
}
@@ -143,6 +143,30 @@ describe('resolveScreenShareAudioSummary', () => {
});
});
it('names the microphone on a device share that picked it over the stored application selection', () => {
expect(
summary({
shareContext: 'device',
microphoneLabel: 'Elgato 4K X Analog Stereo',
sourceMode: 'specific',
includeSources: [{'application.name': 'mpv'}],
usesDeviceMicrophone: true,
}),
).toEqual({
kind: 'message',
descriptor: MICROPHONE_WITH_DEVICE_DESCRIPTOR,
values: {deviceLabel: 'Elgato 4K X Analog Stereo'},
});
expect(
summary({
shareContext: 'display',
sourceMode: 'specific',
includeSources: [{'application.name': 'mpv'}],
usesDeviceMicrophone: true,
}),
).toEqual({kind: 'sourceName', name: 'mpv'});
});
it('keeps a window share on the shared window whatever the stored display selection says', () => {
for (const sourceMode of ['none', 'system', 'specific'] as const) {
expect(summary({shareContext: 'app', sourceMode, includeSources: [{'application.name': 'mpv'}]})).toEqual({
@@ -54,6 +54,7 @@ export interface ScreenShareAudioSummaryInput {
microphoneLabel?: string | null;
displayShareEnvironment?: DisplayShareEnvironment;
windowAudioScope?: WindowShareAudioScope;
usesDeviceMicrophone?: boolean;
}
export type ScreenShareAudioSummary =
@@ -83,7 +84,9 @@ export function resolveScreenShareAudioSummary(input: ScreenShareAudioSummaryInp
const selected = filterRoutableLinuxAudioSources(input.includeSources);
const routesSelectedSources = input.sourceMode === 'specific' && selected.length > 0;
if (input.shareContext === 'device') {
return routesSelectedSources ? summariseSelectedSources(selected) : summariseMicrophone(input.microphoneLabel);
return routesSelectedSources && input.usesDeviceMicrophone !== true
? summariseSelectedSources(selected)
: summariseMicrophone(input.microphoneLabel);
}
if (supportsWindowShareAudioScope(input)) {
const route = selectAppShareAudioRoute({
@@ -9,6 +9,7 @@ const voiceSettings = {
includeSources: [] as Array<Record<string, string>>,
excludeSources: [] as Array<Record<string, string>>,
audioDeviceId: 'default',
deviceUsesMicrophone: false,
};
const activeShareContext = {current: null as 'app' | 'device' | 'display' | null};
@@ -149,6 +150,7 @@ vi.mock('@app/features/voice/state/VoiceSettings', () => ({
getScreenShareAudioSourceMode: () => voiceSettings.audioSourceMode,
getScreenShareAudioIncludeSources: () => voiceSettings.includeSources,
getScreenShareAudioExcludeSources: () => voiceSettings.excludeSources,
getScreenShareDeviceAudioUsesMicrophone: () => voiceSettings.deviceUsesMicrophone,
getEffectiveScreenShareAudioSourceMode: () => voiceSettings.audioSourceMode,
getEffectiveScreenShareAudioIncludeSources: () => voiceSettings.includeSources,
getEffectiveScreenShareAudioExcludeSources: () => voiceSettings.excludeSources,
@@ -178,6 +180,7 @@ beforeEach(() => {
voiceSettings.includeSources = [];
voiceSettings.excludeSources = [];
voiceSettings.audioDeviceId = 'default';
voiceSettings.deviceUsesMicrophone = false;
activeShareContext.current = null;
activeShareSourceId.current = null;
activeShareOwnWindow.current = false;
@@ -362,6 +365,37 @@ describe('sharing a video device', () => {
expect(ensureDeviceScreenShareMicPublication).toHaveBeenCalledTimes(1);
});
test('keeps the microphone the device picker chose while the shared application selection stands', async () => {
platform.current = 'linux';
voiceSettings.audioSourceMode = 'specific';
voiceSettings.includeSources = [{'application.name': 'mpv'}];
voiceSettings.deviceUsesMicrophone = true;
expect(await startConfiguredDeviceScreenShare('camera-1')).toBe(true);
expect(deviceShareAudioDeviceId(startDeviceScreenShare.mock.calls[0])).toBe('mic-1');
expect(ensureLinuxScreenShareAudioPublication).not.toHaveBeenCalled();
expect(await reconfigureActiveDeviceShareAudio()).toBe(true);
expect(ensureDeviceScreenShareMicPublication).toHaveBeenCalledWith('mic-1');
expect(ensureLinuxScreenShareAudioPublication).not.toHaveBeenCalled();
expect(voiceSettings.audioSourceMode).toBe('specific');
expect(voiceSettings.includeSources).toEqual([{'application.name': 'mpv'}]);
});
test('routes the selected applications again once the device picker leaves the microphone', async () => {
platform.current = 'linux';
voiceSettings.audioSourceMode = 'specific';
voiceSettings.includeSources = [{'application.name': 'mpv'}];
voiceSettings.deviceUsesMicrophone = false;
expect(await startConfiguredDeviceScreenShare('camera-1')).toBe(true);
expect(deviceShareAudioDeviceId(startDeviceScreenShare.mock.calls[0])).toBeUndefined();
expect(ensureLinuxScreenShareAudioPublication).toHaveBeenCalledTimes(1);
});
test('keeps the paired capture card input when the live share rebinds its microphone', async () => {
platform.current = 'linux';
activeShareVideoDeviceId.current = 'camera-1';
@@ -418,6 +452,20 @@ describe('sharing a whole display', () => {
});
});
test('keeps the stored include list while a device share of its own is on the microphone', async () => {
platform.current = 'linux';
voiceSettings.audioSourceMode = 'specific';
voiceSettings.includeSources = [{'application.name': 'mpv'}];
voiceSettings.deviceUsesMicrophone = true;
expect(await startConfiguredDisplayScreenShare('screen:1')).toBe(true);
expect(armNativeAudioForLinuxRouting.mock.calls[0][0]).toMatchObject({
include: [{'application.name': 'mpv'}],
ignoreInputMedia: true,
});
});
test('arms native system audio on Windows and macOS', async () => {
for (const current of ['win32', 'darwin']) {
vi.clearAllMocks();
@@ -156,6 +156,7 @@ async function getManualAudioSourceSelectionInput(shareContext: StreamSettingsSh
nativeAudioAvailability: platform === 'linux' ? await getNativeAudioAvailabilityCached() : null,
audioSourceMode: VoiceSettings.getScreenShareAudioSourceMode(),
selectedSourceCount: countRoutableAudioSources(),
usesDeviceMicrophone: VoiceSettings.getScreenShareDeviceAudioUsesMicrophone(),
};
}
@@ -15,7 +15,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
@@ -36,6 +35,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {NativeAudioApplication, VirtmicNode} from '@app/types/electron.d';
@@ -519,8 +519,8 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
@@ -20,6 +20,7 @@ export interface ManualAudioSourceSelectionInput {
nativeAudioAvailability?: NativeAudioAvailability | null;
audioSourceMode?: ScreenShareAudioSourceMode;
selectedSourceCount?: number;
usesDeviceMicrophone?: boolean;
}
export interface WindowShareAudioScopeInput {
@@ -74,6 +75,7 @@ export function canSelectManualAudioSources(
}
export function routesManualAudioSources(input: ManualAudioSourceSelectionInput): boolean {
if (input.shareContext === 'device' && input.usesDeviceMicrophone === true) return false;
return (
canSelectManualAudioSources(input) && input.audioSourceMode === 'specific' && (input.selectedSourceCount ?? 0) > 0
);
@@ -45,9 +45,11 @@ const CODEC_PROBES: ReadonlyArray<CodecProbe> = [
},
];
const SCREEN_SHARE_DECODE_FAILURES_MAX = 1;
let cachedExclusions: Array<VideoCodec> | null = null;
let pendingProbe: Promise<Array<VideoCodec>> | null = null;
const runtimeExcludedCodecs = new Set<VideoCodec>();
const screenShareDecodeFailures = new Set<VideoCodec>();
interface VideoDecoderLike {
isConfigSupported(config: WebCodecsConfig): Promise<{supported: boolean}>;
@@ -80,15 +82,6 @@ function isBaselineWebRtcDecodeCodec(codec: VideoCodec): boolean {
return codec === 'h264' || codec === 'vp8';
}
function mergeRuntimeExclusions(exclusions: Array<VideoCodec> | null): Array<VideoCodec> | null {
if (runtimeExcludedCodecs.size === 0) return exclusions;
const merged = new Set(exclusions ?? []);
for (const codec of runtimeExcludedCodecs) {
merged.add(codec);
}
return [...merged];
}
function getPlatformPolicyExclusions(): Array<VideoCodec> | null {
if ((globalThis as Record<string, unknown>).RTCRtpReceiver === undefined) return null;
const excluded: Array<VideoCodec> = [];
@@ -145,17 +138,17 @@ async function probeAllCodecs(): Promise<Array<VideoCodec>> {
}
export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
if (cachedExclusions) return Promise.resolve(mergeRuntimeExclusions(cachedExclusions) ?? []);
if (cachedExclusions) return Promise.resolve(cachedExclusions);
if (pendingProbe) return pendingProbe;
pendingProbe = probeAllCodecs()
.then((result) => {
cachedExclusions = result;
return mergeRuntimeExclusions(result) ?? [];
return result;
})
.catch((error) => {
logger.warn('Video decoder probe failed entirely, not excluding any codecs', {error});
cachedExclusions = [];
return mergeRuntimeExclusions([]) ?? [];
return [];
})
.finally(() => {
pendingProbe = null;
@@ -164,21 +157,29 @@ export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
}
export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
return mergeRuntimeExclusions(cachedExclusions ?? getPlatformPolicyExclusions());
return cachedExclusions ?? getPlatformPolicyExclusions();
}
export function markVideoDecoderRuntimeFailure(codec: VideoCodec, reason: string): boolean {
if (isBaselineWebRtcDecodeCodec(codec)) return false;
if (runtimeExcludedCodecs.has(codec)) return false;
runtimeExcludedCodecs.add(codec);
logger.warn('Excluding codec from subscriber decode after runtime failure', {codec, reason});
export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
if (screenShareDecodeFailures.has(codec)) return false;
if (screenShareDecodeFailures.size >= SCREEN_SHARE_DECODE_FAILURES_MAX) return false;
screenShareDecodeFailures.add(codec);
logger.warn('Withdrawing decode advertisement for a codec after a screen share decode stall', {codec, reason});
return true;
}
export function getScreenShareDecodeFailures(): ReadonlySet<VideoCodec> {
return screenShareDecodeFailures;
}
export function clearScreenShareDecodeFailures(): void {
screenShareDecodeFailures.clear();
}
export function resetVideoDecoderExclusions(): void {
cachedExclusions = null;
pendingProbe = null;
runtimeExcludedCodecs.clear();
screenShareDecodeFailures.clear();
}
if (typeof window !== 'undefined') {
@@ -1,6 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import Keybind from '@app/features/input/state/InputKeybind';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {
resolveVoiceActivityGateState,
resolveVoiceActivityThresholdRms,
updateVoiceActivityNoiseFloorRms,
} from '@app/features/voice/engine/VoiceLocalSpeakingGate';
import {SPEAKING_LOCAL_RELEASE_MS} from '@app/features/voice/engine/VoiceSpeakingThreshold';
import {computeSpeakingDetectorRms} from '@app/features/voice/engine/v2/VoiceEngineV2AppMicrophoneTransaction';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {buildDeepFilterAudioChain, type DeepFilterAudioChain} from '@app/features/voice/utils/DeepFilterNoiseProcessor';
import {
@@ -12,6 +20,10 @@ import {inputVoiceVolumePercentToGain} from '@app/features/voice/utils/VoiceVolu
import type {AudioProcessorOptions, LocalAudioTrack, Track, TrackProcessor} from 'livekit-client';
const logger = new Logger('VoiceInputProcessor');
const GATE_ANALYSER_FFT_SIZE = 256;
const GATE_TICK_INTERVAL_MS = 50;
const GATE_ATTACK_TIME_CONSTANT = 0.005;
const GATE_RELEASE_TIME_CONSTANT = 0.02;
class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> {
name = 'fluxer-voice-input-processor';
@@ -20,17 +32,26 @@ class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio, Audio
private gainNode: GainNode | null = null;
private passthroughDestination: MediaStreamAudioDestinationNode | null = null;
private deepFilterChain: DeepFilterAudioChain | null = null;
private gateNode: GainNode | null = null;
private gateAnalyserNode: AnalyserNode | null = null;
private gateSamples: Uint8Array<ArrayBuffer> | null = null;
private gateTimerId: number | null = null;
private gateOpen = false;
private gateSilenceStartedAtMs: number | null = null;
private gateNoiseFloorRms = 0;
constructor(
private inputVolumePercent: number,
private deepFilterEnabled: boolean,
private deepFilterNoiseReductionLevel: number,
private gateEnabled: boolean,
) {}
matchesMode(deepFilterEnabled: boolean, deepFilterNoiseReductionLevel: number): boolean {
matchesMode(deepFilterEnabled: boolean, deepFilterNoiseReductionLevel: number, gateEnabled: boolean): boolean {
return (
this.deepFilterEnabled === deepFilterEnabled &&
this.deepFilterNoiseReductionLevel === deepFilterNoiseReductionLevel
this.deepFilterNoiseReductionLevel === deepFilterNoiseReductionLevel &&
this.gateEnabled === gateEnabled
);
}
@@ -60,18 +81,19 @@ class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio, Audio
this.gainNode = opts.audioContext.createGain();
this.gainNode.gain.value = inputVoiceVolumePercentToGain(this.inputVolumePercent);
this.sourceNode.connect(this.gainNode);
const chainTail = this.gateEnabled ? this.startVoiceActivityGate(opts.audioContext) : this.gainNode;
if (this.deepFilterEnabled) {
const chain = await buildDeepFilterAudioChain({
audioContext: opts.audioContext,
noiseReductionLevel: this.deepFilterNoiseReductionLevel,
});
this.deepFilterChain = chain;
this.gainNode.connect(chain.inputDestination);
chainTail.connect(chain.inputDestination);
this.processedTrack = chain.processedTrack;
return;
}
this.passthroughDestination = opts.audioContext.createMediaStreamDestination();
this.gainNode.connect(this.passthroughDestination);
chainTail.connect(this.passthroughDestination);
const passthroughTrack = this.passthroughDestination.stream.getAudioTracks()[0];
if (!passthroughTrack) {
throw new Error('Voice input processor produced no passthrough output track');
@@ -83,9 +105,70 @@ class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio, Audio
}
}
private startVoiceActivityGate(audioContext: BaseAudioContext): AudioNode {
const gainNode = this.gainNode;
if (!gainNode || !this.sourceNode) {
throw new Error('Voice input processor gate requires a built input chain');
}
const gateNode = audioContext.createGain();
gateNode.gain.value = 0;
gainNode.connect(gateNode);
const analyserNode = audioContext.createAnalyser();
analyserNode.fftSize = GATE_ANALYSER_FFT_SIZE;
this.sourceNode.connect(analyserNode);
this.gateNode = gateNode;
this.gateAnalyserNode = analyserNode;
this.gateSamples = new Uint8Array(analyserNode.fftSize);
this.gateOpen = false;
this.gateSilenceStartedAtMs = null;
this.gateNoiseFloorRms = 0;
this.gateTimerId = window.setInterval(() => {
this.tickVoiceActivityGate(audioContext);
}, GATE_TICK_INTERVAL_MS);
return gateNode;
}
private tickVoiceActivityGate(audioContext: BaseAudioContext): void {
const analyserNode = this.gateAnalyserNode;
const samples = this.gateSamples;
const gateNode = this.gateNode;
if (!analyserNode || !samples || !gateNode) return;
analyserNode.getByteTimeDomainData(samples);
const rms = computeSpeakingDetectorRms(samples);
const next = resolveVoiceActivityGateState({
rms,
thresholdRms: resolveVoiceActivityThresholdRms({
autoSensitivity: VoiceSettings.getVadAutoSensitivity(),
vadThreshold: VoiceSettings.getVadThreshold(),
noiseFloorRms: this.gateNoiseFloorRms,
}),
nowMs: performance.now(),
silenceStartedAtMs: this.gateSilenceStartedAtMs,
gateOpen: this.gateOpen,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
});
this.gateSilenceStartedAtMs = next.silenceStartedAtMs;
if (!next.open) {
this.gateNoiseFloorRms = updateVoiceActivityNoiseFloorRms(this.gateNoiseFloorRms, rms);
}
if (next.open === this.gateOpen) return;
this.gateOpen = next.open;
gateNode.gain.setTargetAtTime(
next.open ? 1 : 0,
audioContext.currentTime,
next.open ? GATE_ATTACK_TIME_CONSTANT : GATE_RELEASE_TIME_CONSTANT,
);
}
private async teardown(): Promise<void> {
if (this.gateTimerId !== null) {
window.clearInterval(this.gateTimerId);
this.gateTimerId = null;
}
this.sourceNode?.disconnect();
this.gainNode?.disconnect();
this.gateNode?.disconnect();
this.gateAnalyserNode?.disconnect();
this.passthroughDestination?.disconnect();
if (this.deepFilterChain) {
try {
@@ -103,6 +186,12 @@ class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio, Audio
}
this.sourceNode = null;
this.gainNode = null;
this.gateNode = null;
this.gateAnalyserNode = null;
this.gateSamples = null;
this.gateOpen = false;
this.gateSilenceStartedAtMs = null;
this.gateNoiseFloorRms = 0;
this.passthroughDestination = null;
this.deepFilterChain = null;
this.processedTrack = undefined;
@@ -117,8 +206,16 @@ function resolveActiveVoiceProcessing(): ResolvedVoiceProcessing {
return resolveVoiceProcessingFromStateForDeviceLabel(VoiceSettings, label);
}
export function isVoiceActivityGateEnabled(): boolean {
if (Keybind.transmitMode !== 'voice_activity') return false;
if (Keybind.isPushToMuteEffective()) return false;
return resolveActiveVoiceProcessing().mode !== 'studio';
}
function shouldUseVoiceInputProcessor(): boolean {
return resolveActiveVoiceProcessing().deepFilter || VoiceSettings.getInputVolume() !== 100;
return (
resolveActiveVoiceProcessing().deepFilter || VoiceSettings.getInputVolume() !== 100 || isVoiceActivityGateEnabled()
);
}
export async function syncVoiceInputProcessor(track: LocalAudioTrack | null): Promise<void> {
@@ -130,16 +227,25 @@ export async function syncVoiceInputProcessor(track: LocalAudioTrack | null): Pr
const deepFilterEnabled = profile.deepFilter;
const deepFilterNoiseReductionLevel = profile.deepFilterNoiseReductionLevel;
const inputVolumePercent = VoiceSettings.getInputVolume();
const gateEnabled = isVoiceActivityGateEnabled();
if (!shouldUseVoiceInputProcessor()) {
await removeVoiceInputProcessor(track);
return;
}
if (activeTrack === track && activeProcessor?.matchesMode(deepFilterEnabled, deepFilterNoiseReductionLevel)) {
if (
activeTrack === track &&
activeProcessor?.matchesMode(deepFilterEnabled, deepFilterNoiseReductionLevel, gateEnabled)
) {
activeProcessor.updateInputVolumePercent(inputVolumePercent);
return;
}
await removeVoiceInputProcessor();
const processor = new VoiceInputTrackProcessor(inputVolumePercent, deepFilterEnabled, deepFilterNoiseReductionLevel);
const processor = new VoiceInputTrackProcessor(
inputVolumePercent,
deepFilterEnabled,
deepFilterNoiseReductionLevel,
gateEnabled,
);
try {
await track.setProcessor(processor);
} catch (error) {
@@ -157,7 +263,7 @@ export async function syncVoiceInputProcessor(track: LocalAudioTrack | null): Pr
}
activeTrack = track;
activeProcessor = processor;
logger.debug('Applied voice input processor', {inputVolumePercent, deepFilterEnabled});
logger.debug('Applied voice input processor', {inputVolumePercent, deepFilterEnabled, gateEnabled});
}
export function updateVoiceInputGain(track: LocalAudioTrack | null): void {
@@ -2,6 +2,7 @@
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type ParticipantPublicationLookup,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {
@@ -86,6 +87,64 @@ function nativeStats(overrides: Partial<VoiceEngineV2Stats>): VoiceEngineV2Stats
return {rttMs: null, outbound: [], inbound: [], ...overrides};
}
function chromiumCameraSimulcastTracks(): Array<VoiceEngineV2PerTrackStats> {
return [
{
direction: 'send',
kind: 'video',
ssrc: 1001,
rid: 'q',
active: false,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
targetBitrateKbps: 0,
},
{
direction: 'send',
kind: 'video',
ssrc: 1002,
rid: 'h',
active: false,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
targetBitrateKbps: 0,
},
{
direction: 'send',
kind: 'video',
ssrc: 1003,
rid: 'f',
active: true,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
targetBitrateKbps: 1133,
},
{
direction: 'send',
kind: 'video',
ssrc: 2001,
mid: '3',
trackIdentifier: 'screen-track',
mediaSourceId: 'SV2',
bitrateKbps: 3500,
},
];
}
describe('buildVoiceStatsForNerdsPresentation', () => {
it('classifies browser per-track stats once for overlay and copy consumers', () => {
const localParticipant = participant({
@@ -180,6 +239,203 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.remoteAudio).toBeNull();
});
it('classifies a firefox screen share whose outbound stats omit the track identifier', () => {
const localParticipant = participant({
microphone: 'b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36',
screen_share: '2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f',
});
const remoteParticipant = participant({microphone: '8a106956-d202-4916-8e19-0a1672c64b73'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{
direction: 'recv',
kind: 'audio',
mid: '2',
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
bitrateKbps: 46,
},
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: [remoteParticipant],
});
expect(presentation.localScreenShare?.mid).toBe('3');
expect(presentation.localVideo).toBeNull();
});
it('classifies a firefox inbound screen share reported with a braced track identifier', () => {
const remoteParticipant = participant({screen_share: '34171e0d-174d-4d56-bb71-76b4fba330c3'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{
direction: 'recv',
kind: 'video',
mid: '3',
trackIdentifier: '{34171e0d-174d-4d56-bb71-76b4fba330c3}',
bitrateKbps: 3585,
},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant: null,
remoteParticipants: [remoteParticipant],
});
expect(presentation.remoteScreenShare?.mid).toBe('3');
expect(presentation.remoteVideo).toBeNull();
});
it('exposes every camera simulcast layer next to the projected localVideo row', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo?.rid).toBe('q');
expect(presentation.localVideoLayers).toHaveLength(3);
expect(presentation.localVideoLayers.map((layer) => layer.rid)).toEqual(['q', 'h', 'f']);
expect(presentation.localVideoLayers.map((layer) => layer.active)).toEqual([false, false, true]);
expect(presentation.localVideoLayers.map((layer) => layer.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
});
it('keeps the screen-share publication out of the camera layer view', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localScreenShare?.trackIdentifier).toBe('screen-track');
expect(presentation.localVideoLayers).not.toContain(perTrackStats[3]);
});
it('reports a single camera layer when firefox omits rid and media source ids', () => {
const localParticipant = participant({camera: 'cam-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo?.mid).toBe('3');
expect(presentation.localVideoLayers).toEqual([perTrackStats[1]]);
});
it('reports no camera layers rather than a false single layer when firefox simulcast is unidentifiable', () => {
const localParticipant = participant({camera: 'cam-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', rid: 'q', bitrateKbps: 0},
{direction: 'send', kind: 'video', mid: '3', rid: 'h', bitrateKbps: 0},
{direction: 'send', kind: 'video', mid: '3', rid: 'f', bitrateKbps: 1133},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo).toBeNull();
expect(presentation.localVideoLayers).toEqual([]);
});
it('keeps the chromium simulcast projection byte-identical', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo).toBe(perTrackStats[0]);
expect(presentation.localScreenShare).toBe(perTrackStats[3]);
expect(presentation.localAudio).toBeNull();
expect(presentation.remoteVideo).toBeNull();
expect(presentation.network.videoSendBitrateKbps).toBe(1200);
});
it('uses the canonical v2 native stats projection when native stats are available', () => {
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'video', trackIdentifier: 'browser-camera', bitrateKbps: 500},
@@ -309,3 +565,87 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.network.rttMs).toBeNull();
});
});
describe('collectScreenShareAudioPublicationDiagnostics', () => {
it('reports the mute and upstream state of every local screen-share audio publication', () => {
const displayPublication = {
trackSid: 'TR_display',
source: 'screen_share_audio',
isMuted: false,
audioTrack: {
isUpstreamPaused: true,
mediaStreamTrack: {
id: 'display-audio',
readyState: 'live',
muted: true,
enabled: true,
} as MediaStreamTrack,
},
};
const devicePublication = {
trackSid: 'TR_device',
source: 'screen_share_audio',
isMuted: false,
audioTrack: {
isUpstreamPaused: false,
mediaStreamTrack: {
id: 'device-audio',
readyState: 'live',
muted: false,
enabled: true,
} as MediaStreamTrack,
},
};
const micPublication = {
trackSid: 'TR_mic',
source: 'microphone',
isMuted: false,
audioTrack: {
isUpstreamPaused: false,
mediaStreamTrack: {
id: 'mic-audio',
readyState: 'live',
muted: false,
enabled: true,
} as MediaStreamTrack,
},
};
const localParticipant: ParticipantPublicationLookup = {
getTrackPublication: (source) => (String(source) === 'screen_share_audio' ? displayPublication : undefined),
audioTrackPublications: new Map([
['TR_display', displayPublication],
['TR_device', devicePublication],
['TR_mic', micPublication],
]),
};
const diagnostics = collectScreenShareAudioPublicationDiagnostics(localParticipant);
expect(diagnostics).toEqual([
{
trackSid: 'TR_display',
source: 'screen_share_audio',
isMuted: false,
isUpstreamPaused: true,
mediaStreamTrackId: 'display-audio',
mediaStreamTrackReadyState: 'live',
mediaStreamTrackMuted: true,
mediaStreamTrackEnabled: true,
},
{
trackSid: 'TR_device',
source: 'screen_share_audio',
isMuted: false,
isUpstreamPaused: false,
mediaStreamTrackId: 'device-audio',
mediaStreamTrackReadyState: 'live',
mediaStreamTrackMuted: false,
mediaStreamTrackEnabled: true,
},
]);
});
it('reports nothing when there is no local participant', () => {
expect(collectScreenShareAudioPublicationDiagnostics(null)).toEqual([]);
});
});
@@ -6,7 +6,6 @@ import {
type VoiceTrackPublicationSourceLike,
VoiceTrackSource,
} from '@app/features/voice/engine/VoiceTrackSource';
import type {ScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {
classifyVoiceEngineV2TrackStats,
selectVoiceEngineV2StatsPresentationProjection,
@@ -21,6 +20,17 @@ import {
} from '@fluxer/voice_engine_v2';
import type {Track} from 'livekit-client';
export interface ScreenShareAudioPublicationDiagnostic {
trackSid: string | null;
source: string | null;
isMuted: boolean | null;
isUpstreamPaused: boolean | null;
mediaStreamTrackId: string | null;
mediaStreamTrackReadyState: string | null;
mediaStreamTrackMuted: boolean | null;
mediaStreamTrackEnabled: boolean | null;
}
export interface StatsForNerdsData {
session: {
connectionId: string;
@@ -44,6 +54,7 @@ export interface StatsForNerdsData {
subscriberTransport: VoiceEngineV2TransportInfo | null;
};
localVideo: VoiceEngineV2PerTrackStats | null;
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -85,8 +96,8 @@ export interface StatsForNerdsData {
openH264Enabled: boolean;
};
screenShareAudioCapture: {
pump: ScreenShareAudioPumpDiagnostics;
nativeCapture: Record<string, unknown>;
publications: Array<ScreenShareAudioPublicationDiagnostic>;
};
appInfo: {
appVersion: string;
@@ -119,6 +130,7 @@ export interface VoiceStatsForNerdsPresentation {
session: StatsForNerdsData['session'];
network: StatsForNerdsData['network'];
localVideo: VoiceEngineV2PerTrackStats | null;
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -131,9 +143,12 @@ export interface VoiceStatsForNerdsPresentation {
interface PublicationTrackLike {
mediaStreamTrack?: MediaStreamTrack;
isUpstreamPaused?: boolean;
}
interface TrackPublicationLike extends VoiceTrackPublicationSourceLike {
trackSid?: string;
isMuted?: boolean;
audioTrack?: PublicationTrackLike | null;
videoTrack?: PublicationTrackLike | null;
track?: PublicationTrackLike | null;
@@ -234,6 +249,38 @@ function collectParticipantScreenShareAudioTrackIdentifiers(
return Array.from(identifiers);
}
function summarizeScreenShareAudioPublication(
publication: TrackPublicationLike,
): ScreenShareAudioPublicationDiagnostic {
const track = publication.audioTrack ?? publication.track ?? null;
const mediaStreamTrack = track?.mediaStreamTrack ?? null;
return {
trackSid: publication.trackSid ?? null,
source: typeof publication.source === 'string' ? publication.source : null,
isMuted: publication.isMuted ?? null,
isUpstreamPaused: track?.isUpstreamPaused ?? null,
mediaStreamTrackId: mediaStreamTrack?.id ?? null,
mediaStreamTrackReadyState: mediaStreamTrack?.readyState ?? null,
mediaStreamTrackMuted: mediaStreamTrack?.muted ?? null,
mediaStreamTrackEnabled: mediaStreamTrack?.enabled ?? null,
};
}
export function collectScreenShareAudioPublicationDiagnostics(
participant: ParticipantPublicationLookup | null | undefined,
): Array<ScreenShareAudioPublicationDiagnostic> {
const publications = new Set<TrackPublicationLike>();
const directPublication = participant?.getTrackPublication(SCREEN_SHARE_AUDIO_SOURCE);
if (directPublication) {
publications.add(directPublication);
}
for (const publication of participant?.audioTrackPublications?.values() ?? []) {
if (!isScreenShareAudioPublicationLike(publication)) continue;
publications.add(publication);
}
return Array.from(publications, summarizeScreenShareAudioPublication);
}
function collectRemoteScreenShareAudioTrackIdentifiers(
remoteParticipants: Iterable<ParticipantPublicationLookup> | null | undefined,
): Array<string> {
@@ -284,6 +331,25 @@ function getClassifiedTrack(
return perTrackStats[index] ?? null;
}
function getPublicationGroupKey(track: VoiceEngineV2PerTrackStats): string | null {
if (track.mediaSourceId !== undefined) return `source:${track.mediaSourceId}`;
if (track.trackIdentifier !== undefined) return `track:${track.trackIdentifier}`;
if (track.mid !== undefined) return `mid:${track.mid}`;
return null;
}
function selectLocalVideoLayers(
perTrackStats: ReadonlyArray<VoiceEngineV2PerTrackStats>,
localVideo: VoiceEngineV2PerTrackStats | null,
): Array<VoiceEngineV2PerTrackStats> {
if (!localVideo) return [];
const groupKey = getPublicationGroupKey(localVideo);
if (groupKey === null) return [localVideo];
return perTrackStats.filter(
(track) => track.direction === 'send' && track.kind === 'video' && getPublicationGroupKey(track) === groupKey,
);
}
function selectNativeStatsProjection(
nativeStats: VoiceEngineV2Stats | null,
): VoiceEngineV2StatsPresentationProjection | null {
@@ -366,6 +432,7 @@ export function buildVoiceStatsForNerdsPresentation(
subscriberTransport,
},
localVideo: nativeProjection ? nativeProjection.localVideo : localVideo,
localVideoLayers: selectLocalVideoLayers(perTrackStats, localVideo),
localAudio: nativeProjection ? nativeProjection.localAudio : localAudio,
localScreenShare: nativeProjection ? nativeProjection.localScreenShare : localScreenShare,
localScreenShareAudio: nativeProjection ? nativeProjection.localScreenShareAudio : localScreenShareAudio,
@@ -0,0 +1,141 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it, vi} from 'vitest';
import {
endBridgeStats,
getBridgeStats,
getEndedBridgeCaptures,
recordBridgeFrame,
startBridgeStats,
} from './bridgeStats';
function recordFrames(captureId: string, count: number, peak: number, rms: number): void {
for (let index = 0; index < count; index++) {
recordBridgeFrame(captureId, {timestampUs: index * 10_000, durationUs: 10_000, peak, rms});
}
}
function findEndedCapture(captureId: string) {
return getEndedBridgeCaptures().find((capture) => capture.captureId === captureId);
}
describe('bridgeStats', () => {
it('retains the counters of a capture that a mid-share restart supersedes', () => {
startBridgeStats('generator', 'native-audio:failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
recordFrames('native-audio:failed', 1124, 0.42, 0.11);
startBridgeStats('generator', 'native-audio:replacement', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
expect(getBridgeStats().captureId).toBe('native-audio:replacement');
expect(getBridgeStats().framesReceived).toBe(0);
const retained = findEndedCapture('native-audio:failed');
expect(retained?.bridgeMode).toBe('generator');
expect(retained?.framesReceived).toBe(1124);
expect(retained?.nonSilentFrameCount).toBe(1124);
expect(retained?.lastFramePeak).toBe(0.42);
expect(retained?.endReason).toBe('superseded');
});
it('records the end reason of a capture the newer capture already superseded', () => {
startBridgeStats('generator', 'native-audio:ends-late');
recordFrames('native-audio:ends-late', 3, 0.2, 0.05);
startBridgeStats('generator', 'native-audio:ends-late-replacement');
endBridgeStats('native-audio:ends-late', 'ended', 'source-disappeared');
const retained = findEndedCapture('native-audio:ends-late');
expect(retained?.endReason).toBe('ended');
expect(retained?.endDetail).toBe('source-disappeared');
expect(retained?.endedAt).not.toBeNull();
expect(getBridgeStats().captureId).toBe('native-audio:ends-late-replacement');
expect(getBridgeStats().active).toBe(true);
});
it('still ends the active capture in place', () => {
startBridgeStats('script-processor', 'native-audio:active-end');
endBridgeStats('native-audio:active-end', 'cleanup', 'caller-stopped');
const stats = getBridgeStats();
expect(stats.active).toBe(false);
expect(stats.endReason).toBe('cleanup');
expect(stats.endDetail).toBe('caller-stopped');
expect(stats.endedAt).not.toBeNull();
});
it('keeps the first end reason of the active capture', () => {
startBridgeStats('generator', 'native-audio:double-end');
endBridgeStats('native-audio:double-end', 'ended', 'source-disappeared');
endBridgeStats('native-audio:double-end', 'cleanup', 'caller-stopped');
expect(getBridgeStats().endReason).toBe('ended');
expect(getBridgeStats().endDetail).toBe('source-disappeared');
});
it('flags a capture whose frames keep arriving while every one of them is silence', () => {
const startMs = Date.UTC(2026, 8, 8, 12, 0, 0);
vi.useFakeTimers();
try {
vi.setSystemTime(startMs);
startBridgeStats('generator', 'native-audio:all-silent', {frameDurationUs: 10_000});
for (let index = 0; index < 400; index++) {
vi.setSystemTime(startMs + index * 100);
recordBridgeFrame('native-audio:all-silent', {
timestampUs: index * 10_000,
durationUs: 10_000,
peak: 0,
rms: 0,
});
}
const stats = getBridgeStats();
expect(stats.framesReceived).toBe(400);
expect(stats.nonSilentFrameCount).toBe(0);
expect(stats.lastNonSilentFrameAt).toBeNull();
expect(stats.silentFrameStreak).toBe(400);
expect(stats.maxSilentRunMs).toBe(39_900);
expect(stats.sustainedSilenceWarned).toBe(true);
} finally {
vi.useRealTimers();
}
});
it('does not flag a capture that keeps producing audible frames', () => {
const startMs = Date.UTC(2026, 8, 8, 13, 0, 0);
vi.useFakeTimers();
try {
vi.setSystemTime(startMs);
startBridgeStats('generator', 'native-audio:audible', {frameDurationUs: 10_000});
for (let index = 0; index < 400; index++) {
vi.setSystemTime(startMs + index * 100);
const audible = index % 4 === 0;
recordBridgeFrame('native-audio:audible', {
timestampUs: index * 10_000,
durationUs: 10_000,
peak: audible ? 0.3 : 0,
rms: audible ? 0.1 : 0,
});
}
const stats = getBridgeStats();
expect(stats.nonSilentFrameCount).toBe(100);
expect(stats.lastNonSilentFrameAt).toBe(startMs + 396 * 100);
expect(stats.silentFrameStreak).toBe(3);
expect(stats.maxSilentRunMs).toBe(300);
expect(stats.sustainedSilenceWarned).toBe(false);
} finally {
vi.useRealTimers();
}
});
it('bounds the retained capture history', () => {
for (let index = 0; index < 12; index++) {
startBridgeStats('generator', `native-audio:bounded-${index}`);
}
expect(getEndedBridgeCaptures().length).toBeLessThanOrEqual(8);
expect(findEndedCapture('native-audio:bounded-11')).toBeUndefined();
expect(findEndedCapture('native-audio:bounded-10')).toBeDefined();
});
});
@@ -1,22 +1,54 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {
initialBridgeStats,
type NativeAudioBridgeEndedCapture,
type NativeAudioBridgeFrameMetrics,
type NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
const logger = new Logger('NativeAudioBridgeStats');
const MAX_ENDED_BRIDGE_CAPTURES = 8;
const SUPERSEDED_END_REASON = 'superseded';
const SUSTAINED_SILENCE_WARN_MS = 30_000;
let bridgeStats: NativeAudioBridgeStats = {...initialBridgeStats};
let endedBridgeCaptures: Array<NativeAudioBridgeEndedCapture> = [];
export function getBridgeStats(): NativeAudioBridgeStats {
return {...bridgeStats};
}
export function getEndedBridgeCaptures(): Array<NativeAudioBridgeEndedCapture> {
return endedBridgeCaptures.map((capture) => ({...capture}));
}
function retireBridgeStats(): void {
if (bridgeStats.captureId === null) return;
endedBridgeCaptures.push({
captureId: bridgeStats.captureId,
bridgeMode: bridgeStats.bridgeMode,
startedAt: bridgeStats.startedAt,
endedAt: bridgeStats.endedAt ?? Date.now(),
endReason: bridgeStats.endReason ?? SUPERSEDED_END_REASON,
endDetail: bridgeStats.endDetail,
framesReceived: bridgeStats.framesReceived,
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
lastFramePeak: bridgeStats.lastFramePeak,
});
if (endedBridgeCaptures.length > MAX_ENDED_BRIDGE_CAPTURES) {
endedBridgeCaptures = endedBridgeCaptures.slice(-MAX_ENDED_BRIDGE_CAPTURES);
}
}
export function startBridgeStats(
mode: 'generator' | 'script-processor',
captureId: string,
options: {prebufferTargetUs?: number; frameDurationUs?: number} = {},
): void {
retireBridgeStats();
bridgeStats = {
...initialBridgeStats,
active: true,
@@ -57,6 +89,24 @@ export function recordBridgeFrame(captureId: string, metrics: NativeAudioBridgeF
bridgeStats.maxFrameRms = Math.max(bridgeStats.maxFrameRms, bridgeStats.lastFrameRms);
if (peak >= 0.0005 || rms >= 0.0001) {
bridgeStats.nonSilentFrameCount += 1;
bridgeStats.lastNonSilentFrameAt = now;
bridgeStats.silentFrameStreak = 0;
} else {
bridgeStats.silentFrameStreak += 1;
const silentSinceMs = bridgeStats.lastNonSilentFrameAt ?? bridgeStats.startedAt ?? now;
const silentRunMs = Math.max(0, now - silentSinceMs);
bridgeStats.maxSilentRunMs = Math.max(bridgeStats.maxSilentRunMs, silentRunMs);
if (!bridgeStats.sustainedSilenceWarned && silentRunMs >= SUSTAINED_SILENCE_WARN_MS) {
bridgeStats.sustainedSilenceWarned = true;
logger.warn('Native audio capture is delivering frames that contain only silence', {
captureId: bridgeStats.captureId,
bridgeMode: bridgeStats.bridgeMode,
framesReceived: bridgeStats.framesReceived,
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
silentFrameStreak: bridgeStats.silentFrameStreak,
silentRunMs,
});
}
}
}
}
@@ -95,9 +145,16 @@ export function recordBridgeQueue(captureId: string, pendingFrames: number, buff
}
export function endBridgeStats(captureId: string, reason: string | null, detail: string | null): void {
if (!bridgeStats.active || bridgeStats.captureId !== captureId) return;
bridgeStats.active = false;
bridgeStats.endReason = reason;
bridgeStats.endDetail = detail;
bridgeStats.endedAt = Date.now();
if (bridgeStats.active && bridgeStats.captureId === captureId) {
bridgeStats.active = false;
bridgeStats.endReason = reason;
bridgeStats.endDetail = detail;
bridgeStats.endedAt = Date.now();
return;
}
const ended = endedBridgeCaptures.find((capture) => capture.captureId === captureId);
if (!ended || ended.endReason !== SUPERSEDED_END_REASON) return;
ended.endReason = reason;
ended.endDetail = detail;
ended.endedAt = Date.now();
}
@@ -122,6 +122,10 @@ export interface NativeAudioBridgeStats {
maxFramePeak: number;
maxFrameRms: number;
nonSilentFrameCount: number;
lastNonSilentFrameAt: number | null;
silentFrameStreak: number;
maxSilentRunMs: number;
sustainedSilenceWarned: boolean;
prebufferTargetMs: number | null;
frameDurationMs: number | null;
endReason: string | null;
@@ -129,6 +133,18 @@ export interface NativeAudioBridgeStats {
endedAt: number | null;
}
export interface NativeAudioBridgeEndedCapture {
captureId: string;
bridgeMode: 'generator' | 'script-processor' | null;
startedAt: number | null;
endedAt: number | null;
endReason: string | null;
endDetail: string | null;
framesReceived: number;
nonSilentFrameCount: number;
lastFramePeak: number | null;
}
export interface NativeAudioBridgeFrameMetrics {
timestampUs?: number;
durationUs?: number;
@@ -156,6 +172,10 @@ export const initialBridgeStats: NativeAudioBridgeStats = {
maxFramePeak: 0,
maxFrameRms: 0,
nonSilentFrameCount: 0,
lastNonSilentFrameAt: null,
silentFrameStreak: 0,
maxSilentRunMs: 0,
sustainedSilenceWarned: false,
prebufferTargetMs: null,
frameDurationMs: null,
endReason: null,
+31 -1
View File
@@ -225,7 +225,7 @@ fn build_csp_directives(
extend_runtime_sources(&mut font, runtime_sources, true, true);
directives.push(format!("font-src {}", font.join(" ")));
let mut connect = vec!["'self'".to_owned(), "data:".to_owned()];
let mut connect = vec!["'self'".to_owned(), "blob:".to_owned(), "data:".to_owned()];
extend_from(&mut connect, &config.extra_connect_src, CONNECT_SOURCES);
extend_runtime_sources(&mut connect, runtime_sources, true, true);
extend_runtime_s3_sources(&mut connect, runtime_sources);
@@ -355,6 +355,36 @@ mod tests {
assert!(csp.contains("frame-ancestors 'none'"));
}
#[test]
fn build_csp_allows_blob_connections_for_camera_background_media() {
let config = default_csp_config();
let csp = build_csp(&config, "testnonce", &runtime_sources());
let connect = csp
.split("; ")
.find(|directive| directive.starts_with("connect-src "))
.expect("connect-src directive");
assert!(
connect.split(' ').any(|source| source == "blob:"),
"connect-src must allow blob: object URLs: {connect}"
);
}
#[test]
fn an_asset_header_allows_blob_connections_for_the_camera_effect_worker() {
let policy = CompiledCspPolicy::compile(default_csp_config(), &runtime_sources()).unwrap();
let asset = policy.asset_header();
let asset = asset.to_str().unwrap();
let connect = asset
.split("; ")
.find(|directive| directive.starts_with("connect-src "))
.expect("connect-src directive");
assert!(
connect.split(' ').any(|source| source == "blob:"),
"the camera-effect worker is served as /assets/*.worker.js and runs under the asset \
policy, so that policy must allow blob: object URLs: {connect}"
);
}
#[test]
fn build_csp_includes_nonce_in_script_src() {
let config = default_csp_config();
@@ -0,0 +1,171 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {readFileSync} from 'node:fs';
import {createRequire} from 'node:module';
import path from 'node:path';
import {describe, test} from 'node:test';
import {fileURLToPath} from 'node:url';
import vm from 'node:vm';
const require = createRequire(import.meta.url);
const esbuild = require('esbuild');
const sourcePath = fileURLToPath(new URL('./WindowsShortcuts.ts', import.meta.url));
const source = readFileSync(sourcePath, 'utf8');
const transformedSource = esbuild.transformSync(source, {
loader: 'ts',
format: 'cjs',
platform: 'node',
target: 'node20',
}).code;
const APPDATA = 'C:\\Users\\csh\\AppData\\Roaming';
const USERPROFILE = 'C:\\Users\\csh';
const PROGRAMS_DIR = path.win32.join(APPDATA, 'Microsoft', 'Windows', 'Start Menu', 'Programs');
const AUTHOR_DIR = path.win32.join(PROGRAMS_DIR, 'Fluxer Platform AB');
const ROOT_SHORTCUT = path.win32.join(PROGRAMS_DIR, 'Fluxer Canary.lnk');
const AUTHOR_SHORTCUT = path.win32.join(AUTHOR_DIR, 'Fluxer Canary.lnk');
const ROOT_APP_DIR = 'C:\\Users\\csh\\AppData\\Local\\fluxer_desktop_canary';
const CURRENT_DIR = path.win32.join(ROOT_APP_DIR, 'current');
const CURRENT_EXE = path.win32.join(CURRENT_DIR, 'Fluxer Canary.exe');
const STALE_EXE = path.win32.join(ROOT_APP_DIR, 'app-1.0.0', 'Fluxer Canary.exe');
const LEGACY_APP_USER_MODEL_ID = 'velopack.fluxer_desktop_canary';
function lnkBuffer(...values) {
return Buffer.concat([Buffer.from('L\0\0\0', 'utf8'), ...values.map((value) => Buffer.from(value, 'utf16le'))]);
}
function loadWindowsShortcuts(initialFiles) {
const files = new Map(initialFiles);
const dirs = new Set([PROGRAMS_DIR, ROOT_APP_DIR, CURRENT_DIR]);
for (const filePath of files.keys()) {
dirs.add(path.win32.dirname(filePath));
}
files.set(path.win32.join(ROOT_APP_DIR, 'Update.exe'), Buffer.from('exe'));
const fsOperations = [];
const fakeFs = {
existsSync: (target) => files.has(target) || dirs.has(target),
readFileSync: (target) => {
const contents = files.get(target);
if (!contents) throw new Error(`ENOENT ${target}`);
return contents;
},
mkdirSync: (target) => {
fsOperations.push(['mkdir', target]);
dirs.add(target);
},
renameSync: (from, to) => {
fsOperations.push(['rename', from, to]);
files.set(to, files.get(from));
files.delete(from);
},
rmSync: (target) => {
fsOperations.push(['rm', target]);
files.delete(target);
},
};
const createdShortcuts = [];
const winShell = {
loadError: null,
createShortcut: async (options) => {
createdShortcuts.push(options);
files.set(options.lnkPath, lnkBuffer(options.target, options.appUserModelId));
},
};
const module = {exports: {}};
const context = vm.createContext({
module,
exports: module.exports,
console,
Buffer,
process: {platform: 'win32', execPath: CURRENT_EXE, env: {APPDATA, USERPROFILE}},
require: (specifier) => {
if (specifier === 'node:fs') {
return {default: fakeFs, ...fakeFs};
}
if (specifier === 'node:path') {
return {default: path.win32, ...path.win32};
}
if (specifier === 'node:module') {
return {
createRequire: () => (moduleName) => {
if (moduleName === '@fluxer/win-shell') return winShell;
throw new Error(`unexpected require ${moduleName}`);
},
};
}
if (specifier === '@electron/common/DesktopIdentity') {
return {
DESKTOP_APP_NAME: 'Fluxer Canary',
WINDOWS_APP_USER_MODEL_ID: 'Fluxer.Fluxer.Canary',
WINDOWS_LEGACY_APP_USER_MODEL_IDS: [LEGACY_APP_USER_MODEL_ID],
WINDOWS_SHORTCUT_AUTHOR: 'Fluxer Platform AB',
WINDOWS_TOAST_ACTIVATOR_CLSID: '{9CEDB5C0-3552-43B0-A279-2232E0CDF74C}',
};
}
return require(specifier);
},
});
vm.runInContext(transformedSource, context, {filename: sourcePath});
return {...module.exports, createdShortcuts, dirs, files, fsOperations};
}
async function runRepair(harness) {
harness.repairWindowsShortcuts();
await new Promise((resolve) => setImmediate(resolve));
}
describe('Windows Start Menu shortcut repair', () => {
test('leaves a user-placed root shortcut alone and does not recreate the author folder', async () => {
const harness = loadWindowsShortcuts([[ROOT_SHORTCUT, lnkBuffer(CURRENT_EXE)]]);
await runRepair(harness);
assert.deepEqual(harness.fsOperations, []);
assert.equal(harness.files.has(ROOT_SHORTCUT), true);
assert.equal(harness.files.has(AUTHOR_SHORTCUT), false);
assert.equal(harness.dirs.has(AUTHOR_DIR), false);
});
test('does not delete a root shortcut when the author shortcut also exists', async () => {
const harness = loadWindowsShortcuts([
[ROOT_SHORTCUT, lnkBuffer(CURRENT_EXE)],
[AUTHOR_SHORTCUT, lnkBuffer(CURRENT_EXE)],
]);
await runRepair(harness);
assert.deepEqual(harness.fsOperations, []);
assert.equal(harness.files.has(ROOT_SHORTCUT), true);
assert.equal(harness.files.has(AUTHOR_SHORTCUT), true);
});
test('rewrites a stale root shortcut in place instead of relocating it', async () => {
const harness = loadWindowsShortcuts([[ROOT_SHORTCUT, lnkBuffer(STALE_EXE)]]);
await runRepair(harness);
assert.deepEqual(harness.fsOperations, []);
const rewritten = harness.createdShortcuts.filter((options) => options.lnkPath === ROOT_SHORTCUT);
assert.equal(rewritten.length, 1);
assert.equal(rewritten[0].target, CURRENT_EXE);
assert.equal(rewritten[0].appUserModelId, 'Fluxer.Fluxer.Canary');
assert.equal(harness.files.has(AUTHOR_SHORTCUT), false);
});
test('still rewrites the author shortcut carrying the legacy AppUserModelID', async () => {
const harness = loadWindowsShortcuts([[AUTHOR_SHORTCUT, lnkBuffer(CURRENT_EXE, LEGACY_APP_USER_MODEL_ID)]]);
await runRepair(harness);
const rewritten = harness.createdShortcuts.filter((options) => options.lnkPath === AUTHOR_SHORTCUT);
assert.equal(rewritten.length, 1);
assert.equal(rewritten[0].target, CURRENT_EXE);
assert.equal(rewritten[0].appUserModelId, 'Fluxer.Fluxer.Canary');
assert.equal(rewritten[0].toastActivatorClsid, '{9CEDB5C0-3552-43B0-A279-2232E0CDF74C}');
});
});
+1 -14
View File
@@ -146,20 +146,7 @@ async function repairWindowsShortcutsAsync(repairPaths: WindowsShortcutRepairPat
return;
}
const {createShortcut} = binding;
try {
if (fs.existsSync(repairPaths.rootShortcut)) {
const authorDir = path.dirname(repairPaths.authorShortcut);
fs.mkdirSync(authorDir, {recursive: true});
if (fs.existsSync(repairPaths.authorShortcut)) {
fs.rmSync(repairPaths.rootShortcut, {force: true});
} else {
fs.renameSync(repairPaths.rootShortcut, repairPaths.authorShortcut);
}
}
} catch (error) {
console.warn('[WindowsShortcuts] Failed to migrate root Start-Menu shortcut', error);
}
const shortcutPaths: Array<string> = [repairPaths.authorShortcut];
const shortcutPaths: Array<string> = [repairPaths.authorShortcut, repairPaths.rootShortcut];
const desktopDir = process.env.USERPROFILE ? path.join(process.env.USERPROFILE, 'Desktop') : '';
if (desktopDir) shortcutPaths.push(path.join(desktopDir, `${DESKTOP_APP_NAME}.lnk`));
if (process.env.APPDATA) {
@@ -85,8 +85,6 @@ Admission and dispatch tuning for the Gateway cluster.
| gateway_dispatch_relay_shards | integer | Dispatch relay shard count (1-10000, default 32) |
| gateway_dispatch_relay_max_queue | integer | Dispatch relay queue ceiling (0-1000000, default 50000) |
| voice_e2ee_scope | string | `guild_feature_only` or `platform_wide` (default `guild_feature_only`) |
| voice_reconciliation_v3_percentage | number | Percentage of voice states on the newer reconciliation path (0-100, default 100) |
| voice_reconciliation_v3_interval_ms | integer | Voice reconciliation interval (500-60000, default 2000) |
Every field is present on read. A deployment that has stored nothing reports the defaults above.

Some files were not shown because too many files have changed in this diff Show More