From 12397032e380690ab8e98c6fad76b2dd30dcd91a Mon Sep 17 00:00:00 2001 From: Hampus Date: Sat, 12 Sep 2026 15:38:32 +0200 Subject: [PATCH] feat(voice): add the recon service and remove the old worker (#2719) --- .github/labeller.yaml | 3 + .github/workflows/build-recon.yaml | 36 + Cargo.lock | 19 + Cargo.toml | 1 + config/env/development.env | 6 +- fluxer_api/src/api/Config.ts | 9 - fluxer_api/src/api/config/APIConfig.ts | 9 - .../api/infrastructure/VoiceRoomContext.ts | 25 - .../voice/VoiceReconciliationWorker.test.ts | 40 - .../api/voice/VoiceReconciliationWorker.ts | 1150 ------------ .../VoiceReconciliationWorkerStop.test.ts | 64 - .../src/api/worker/WorkerDependencies.ts | 30 +- fluxer_api/src/api/worker/WorkerMain.ts | 13 +- fluxer_recon/Cargo.toml | 24 + fluxer_recon/Dockerfile | 42 + fluxer_recon/src/actuate.rs | 1325 +++++++++++++ fluxer_recon/src/budget.rs | 1046 +++++++++++ fluxer_recon/src/census.rs | 308 +++ fluxer_recon/src/clock.rs | 125 ++ fluxer_recon/src/config.rs | 402 ++++ fluxer_recon/src/control.rs | 784 ++++++++ fluxer_recon/src/decide.rs | 873 +++++++++ fluxer_recon/src/discovery.rs | 1656 +++++++++++++++++ fluxer_recon/src/evidence.rs | 758 ++++++++ fluxer_recon/src/gateway/codes.rs | 164 ++ fluxer_recon/src/gateway/mod.rs | 316 ++++ fluxer_recon/src/gateway/nats.rs | 468 +++++ fluxer_recon/src/guards.rs | 572 ++++++ fluxer_recon/src/health.rs | 502 +++++ fluxer_recon/src/ids.rs | 252 +++ fluxer_recon/src/ledger.rs | 978 ++++++++++ fluxer_recon/src/lib.rs | 28 + fluxer_recon/src/livekit/auth.rs | 170 ++ fluxer_recon/src/livekit/mod.rs | 179 ++ fluxer_recon/src/livekit/twirp.rs | 482 +++++ fluxer_recon/src/main.rs | 155 ++ fluxer_recon/src/metrics.rs | 604 ++++++ fluxer_recon/src/names.rs | 79 + fluxer_recon/src/observe.rs | 380 ++++ fluxer_recon/src/runtime.rs | 705 +++++++ fluxer_recon/src/schedule.rs | 562 ++++++ fluxer_recon/src/service.rs | 1315 +++++++++++++ fluxer_recon/src/singleton.rs | 245 +++ fluxer_recon/src/suspicion.rs | 356 ++++ fluxer_recon/src/topology/mod.rs | 453 +++++ fluxer_recon/src/topology/postgres.rs | 96 + fluxer_recon/src/topology/scylla.rs | 97 + fluxer_recon/src/turn.rs | 597 ++++++ fluxer_recon/src/webhook.rs | 481 +++++ fluxer_svc/src/metrics.rs | 4 +- packages/config/src/MasterConfig.ts | 9 - .../src/config_loader/EnvironmentOverrides.ts | 28 - tools/ci/src/ci_workflow.rs | 5 + tools/ci/src/image_set.rs | 4 + 54 files changed, 17658 insertions(+), 1376 deletions(-) create mode 100644 .github/workflows/build-recon.yaml delete mode 100644 fluxer_api/src/api/voice/VoiceReconciliationWorker.test.ts delete mode 100644 fluxer_api/src/api/voice/VoiceReconciliationWorker.ts delete mode 100644 fluxer_api/src/api/voice/tests/VoiceReconciliationWorkerStop.test.ts create mode 100644 fluxer_recon/Cargo.toml create mode 100644 fluxer_recon/Dockerfile create mode 100644 fluxer_recon/src/actuate.rs create mode 100644 fluxer_recon/src/budget.rs create mode 100644 fluxer_recon/src/census.rs create mode 100644 fluxer_recon/src/clock.rs create mode 100644 fluxer_recon/src/config.rs create mode 100644 fluxer_recon/src/control.rs create mode 100644 fluxer_recon/src/decide.rs create mode 100644 fluxer_recon/src/discovery.rs create mode 100644 fluxer_recon/src/evidence.rs create mode 100644 fluxer_recon/src/gateway/codes.rs create mode 100644 fluxer_recon/src/gateway/mod.rs create mode 100644 fluxer_recon/src/gateway/nats.rs create mode 100644 fluxer_recon/src/guards.rs create mode 100644 fluxer_recon/src/health.rs create mode 100644 fluxer_recon/src/ids.rs create mode 100644 fluxer_recon/src/ledger.rs create mode 100644 fluxer_recon/src/lib.rs create mode 100644 fluxer_recon/src/livekit/auth.rs create mode 100644 fluxer_recon/src/livekit/mod.rs create mode 100644 fluxer_recon/src/livekit/twirp.rs create mode 100644 fluxer_recon/src/main.rs create mode 100644 fluxer_recon/src/metrics.rs create mode 100644 fluxer_recon/src/names.rs create mode 100644 fluxer_recon/src/observe.rs create mode 100644 fluxer_recon/src/runtime.rs create mode 100644 fluxer_recon/src/schedule.rs create mode 100644 fluxer_recon/src/service.rs create mode 100644 fluxer_recon/src/singleton.rs create mode 100644 fluxer_recon/src/suspicion.rs create mode 100644 fluxer_recon/src/topology/mod.rs create mode 100644 fluxer_recon/src/topology/postgres.rs create mode 100644 fluxer_recon/src/topology/scylla.rs create mode 100644 fluxer_recon/src/turn.rs create mode 100644 fluxer_recon/src/webhook.rs diff --git a/.github/labeller.yaml b/.github/labeller.yaml index 3eb7ad35e..3f84a94cd 100644 --- a/.github/labeller.yaml +++ b/.github/labeller.yaml @@ -33,6 +33,9 @@ f:media_proxy: f:messages: - changed-files: - any-glob-to-any-file: fluxer_messages/**/* +f:recon: + - changed-files: + - any-glob-to-any-file: fluxer_recon/**/* f:snowflakes: - changed-files: - any-glob-to-any-file: fluxer_snowflakes/**/* diff --git a/.github/workflows/build-recon.yaml b/.github/workflows/build-recon.yaml new file mode 100644 index 000000000..4ec9b31a2 --- /dev/null +++ b/.github/workflows/build-recon.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +name: build recon + +on: + workflow_dispatch: + inputs: + build-version: + description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation" + type: string + required: false + default: "" + +permissions: + actions: read + contents: write + packages: write + +jobs: + approve: + name: approve build release + permissions: {} + runs-on: ubuntu-24.04 + environment: builds + timeout-minutes: 5 + steps: + - name: approved + run: echo "Build release approved." + + image: + needs: approve + uses: ./.github/workflows/_build-image.yaml + secrets: inherit + with: + image: fluxer-recon + dockerfile: fluxer_recon/Dockerfile + build-version: ${{ inputs['build-version'] }} diff --git a/Cargo.lock b/Cargo.lock index c0dbd6753..1f659a2cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1866,6 +1866,25 @@ dependencies = [ "url", ] +[[package]] +name = "fluxer-recon" +version = "0.0.0" +dependencies = [ + "anyhow", + "axum", + "base64", + "fluxer-svc", + "hmac 0.13.0", + "reqwest", + "scylla", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "fluxer-snowflakes" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1998ffdff..1aa26fa69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "fluxer_users", "fluxer_unfurl", "packages/markdown_parser/rust", + "fluxer_recon", ] exclude = [ "fluxer_marketing", diff --git a/config/env/development.env b/config/env/development.env index 7a6eccbbd..4e9417934 100644 --- a/config/env/development.env +++ b/config/env/development.env @@ -65,10 +65,14 @@ FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686} +FLUXER_RECON_MODE=observing +FLUXER_RECON_EXPECTED_ROOMS=64 +FLUXER_RECON_WARMUP_SECONDS=15 +FLUXER_RECON_MAX_HOT_ROOMS=8 + FLUXER_API_PORT=8080 FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED=true FLUXER_API_WORKER_MODE=all_lanes -FLUXER_API_WORKER_ENABLE_VOICE_RECONCILIATION=true FLUXER_APP_DEV_PORT=3000 FLUXER_APP_PROXY_PORT=8773 FLUXER_STATIC_DIR=fluxer_app/dist diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index d62dbd30b..95d1fd511 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -531,15 +531,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { laneName: apiWorkerConfig?.lane, taskName: apiWorkerConfig?.task as WorkerTaskName | undefined, enableCronScheduler: apiWorkerConfig?.enable_cron_scheduler, - enableVoiceReconciliation: apiWorkerConfig?.enable_voice_reconciliation ?? true, - voiceReconciliation: { - intervalMs: apiWorkerConfig?.voice_reconciliation?.interval_ms, - staggerDelayMs: apiWorkerConfig?.voice_reconciliation?.stagger_delay_ms, - lockTtlSeconds: apiWorkerConfig?.voice_reconciliation?.lock_ttl_seconds, - cadenceTtlSeconds: apiWorkerConfig?.voice_reconciliation?.cadence_ttl_seconds, - gatewayOnlyGraceMs: apiWorkerConfig?.voice_reconciliation?.gateway_only_grace_ms, - liveKitOnlyGraceMs: apiWorkerConfig?.voice_reconciliation?.livekit_only_grace_ms, - }, laneConcurrencyOverrides: { realtime: apiWorkerConfig?.lane_concurrency_overrides?.realtime, unfurl: apiWorkerConfig?.lane_concurrency_overrides?.unfurl, diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 7dc431d89..ef7b1c5f4 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -377,15 +377,6 @@ export interface APIConfig { laneName?: APIWorkerLaneName; taskName?: WorkerTaskName; enableCronScheduler?: boolean; - enableVoiceReconciliation: boolean; - voiceReconciliation: { - intervalMs: number | undefined; - staggerDelayMs: number | undefined; - lockTtlSeconds: number | undefined; - cadenceTtlSeconds: number | undefined; - gatewayOnlyGraceMs: number | undefined; - liveKitOnlyGraceMs: number | undefined; - }; laneConcurrencyOverrides: { realtime?: number; unfurl?: number; diff --git a/fluxer_api/src/api/infrastructure/VoiceRoomContext.ts b/fluxer_api/src/api/infrastructure/VoiceRoomContext.ts index 3d3456506..967adba5b 100644 --- a/fluxer_api/src/api/infrastructure/VoiceRoomContext.ts +++ b/fluxer_api/src/api/infrastructure/VoiceRoomContext.ts @@ -134,28 +134,3 @@ export function parseParticipantMetadataWithRaw(metadata: string): { export function isDMRoom(context: VoiceRoomContext): context is DMRoomContext { return context.type === 'dm'; } - -const PARTICIPANT_IDENTITY_PREFIX = 'user_'; - -interface ParticipantIdentity { - readonly userId: UserID; - readonly connectionId: string; -} - -export function parseParticipantIdentity(identity: string): ParticipantIdentity | null { - if (!identity.startsWith(PARTICIPANT_IDENTITY_PREFIX)) { - return null; - } - const parts = identity.split('_'); - if (parts.length !== 3 || parts[0] !== 'user') { - return null; - } - try { - return { - userId: createUserID(BigInt(parts[1])), - connectionId: parts[2], - }; - } catch { - return null; - } -} diff --git a/fluxer_api/src/api/voice/VoiceReconciliationWorker.test.ts b/fluxer_api/src/api/voice/VoiceReconciliationWorker.test.ts deleted file mode 100644 index 214136e47..000000000 --- a/fluxer_api/src/api/voice/VoiceReconciliationWorker.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import {candidateTtlSecondsFor} from './VoiceReconciliationWorker'; - -const INTERVAL_MS = 15000; -const GATEWAY_ONLY_GRACE_MS = 10000; - -function ttlFor(observedSweepSpacingMs: number): number { - return candidateTtlSecondsFor({ - intervalMs: INTERVAL_MS, - observedSweepSpacingMs, - graceMs: GATEWAY_ONLY_GRACE_MS, - }); -} - -describe('candidateTtlSecondsFor', () => { - it('outlives the gap between two consecutive observations of the same key', () => { - for (const observedSweepSpacingMs of [0, 45_000, 136_000, 300_000, 596_000, 900_000]) { - expect(ttlFor(observedSweepSpacingMs) * 1000).toBeGreaterThan(observedSweepSpacingMs); - } - }); - - it('outlives a sweep gap far longer than the tick interval', () => { - expect(ttlFor(596_000) * 1000).toBeGreaterThan(596_000); - }); - - it('grows with the observed sweep spacing rather than the tick interval', () => { - expect(ttlFor(596_000)).toBeGreaterThan(ttlFor(136_000)); - expect(ttlFor(136_000)).toBeGreaterThan(ttlFor(0)); - }); - - it('keeps a floor that survives a single long sweep before any spacing is observed', () => { - expect(ttlFor(0)).toBeGreaterThanOrEqual(300); - }); - - it('stays bounded so a stale candidate cannot outlive its connection indefinitely', () => { - expect(ttlFor(Number.MAX_SAFE_INTEGER)).toBeLessThanOrEqual(3600); - }); -}); diff --git a/fluxer_api/src/api/voice/VoiceReconciliationWorker.ts b/fluxer_api/src/api/voice/VoiceReconciliationWorker.ts deleted file mode 100644 index b647bc9cd..000000000 --- a/fluxer_api/src/api/voice/VoiceReconciliationWorker.ts +++ /dev/null @@ -1,1150 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {randomUUID} from 'node:crypto'; -import {BadGatewayError} from '@fluxer/errors/src/domains/core/BadGatewayError'; -import {GatewayTimeoutError} from '@fluxer/errors/src/domains/core/GatewayTimeoutError'; -import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError'; -import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider'; -import type {ChannelID, GuildID, UserID} from '../BrandedTypes'; -import {createUserID} from '../BrandedTypes'; -import type {ILogger} from '../ILogger'; -import type {GatewayVoiceStateEntry, IGatewayService} from '../infrastructure/IGatewayService'; -import type {ILiveKitService, LiveKitRoomLocation} from '../infrastructure/ILiveKitService'; -import type {IVoiceRoomStore} from '../infrastructure/IVoiceRoomStore'; -import {parseParticipantIdentity, parseRoomName} from '../infrastructure/VoiceRoomContext'; - -interface GatewayPendingJoinEntry { - readonly connectionId: string; - readonly userId: string; - readonly tokenNonce: string; - readonly expiresAt: number; -} - -interface VoiceReconciliationWorkerOptions { - gatewayService: IGatewayService; - liveKitService: ILiveKitService; - voiceRoomStore: IVoiceRoomStore; - kvClient: IKVProvider; - logger: ILogger; - intervalMs?: number; - staggerDelayMs?: number; - lockTtlSeconds?: number; - cadenceTtlSeconds?: number; - gatewayOnlyGraceMs?: number; - liveKitOnlyGraceMs?: number; -} - -interface DiscoveredRoom { - readonly roomName: string; - readonly guildId?: GuildID; - readonly channelId: ChannelID; - readonly fromGateway: boolean; - readonly fromLiveKit: boolean; - readonly gatewayVoiceStateCount: number; -} - -interface RoomDiscovery { - readonly rooms: Array; - readonly liveKitLocationsByRoom: Map>; - readonly liveKitDiscoveryComplete: boolean; - readonly liveKitDiscoveryErrors: number; - readonly liveKitServersSearched: number; - readonly gatewayRoomsDiscovered: number; - readonly liveKitRoomsDiscovered: number; - readonly gatewayDiscoveryFailed: boolean; -} - -interface LiveKitParticipantEntry { - readonly identity: string; - readonly userId: UserID; - readonly connectionId: string; - readonly regionId: string; - readonly serverId: string; -} - -interface ResolvedLiveKitLocation extends LiveKitRoomLocation { - readonly authoritativeForGatewayState: boolean; -} - -interface LiveKitRoomSnapshot { - readonly participants: Array; - readonly completed: boolean; - readonly gatewayStateRemovalComplete: boolean; - readonly errors: number; - readonly searchedLocations: number; -} - -interface RoomReconciliationResult { - readonly roomName: string; - readonly livekitOnlyConfirmed: number; - readonly livekitOnlyRepaired: number; - readonly livekitOnlyDisconnected: number; - readonly livekitOnlyDeferred: number; - readonly gatewayOnlyRemoved: number; - readonly gatewayOnlyDeferred: number; - readonly gatewayOnlySkipped: number; - readonly consistent: number; - readonly transientSkip?: boolean; -} - -type LiveKitOnlyRepairResult = 'repaired' | 'not_repairable' | 'defer'; - -const DEFAULT_INTERVAL_MS = 15000; -const DEFAULT_STAGGER_DELAY_MS = 25; -const DEFAULT_LOCK_TTL_SECONDS = 180; -const DEFAULT_GATEWAY_ONLY_GRACE_MS = 10000; -const DEFAULT_LIVEKIT_ONLY_GRACE_MS = 60000; -const MIN_CANDIDATE_TTL_SECONDS = 300; -const MAX_CANDIDATE_TTL_SECONDS = 3600; -const CANDIDATE_TTL_SWEEP_MULTIPLIER = 3; -const LAST_SWEEP_KEY_TTL_SECONDS = 86400; -const ROOM_KEY_PREFIX = 'voice:room:server:'; -export function candidateTtlSecondsFor(input: { - intervalMs: number; - observedSweepSpacingMs: number; - graceMs: number; -}): number { - const spacingMs = Math.max(input.intervalMs, input.observedSweepSpacingMs); - const ttlSeconds = Math.ceil((spacingMs * CANDIDATE_TTL_SWEEP_MULTIPLIER + input.graceMs * 2) / 1000); - return Math.min(MAX_CANDIDATE_TTL_SECONDS, Math.max(MIN_CANDIDATE_TTL_SECONDS, ttlSeconds)); -} - -const VOICE_RECONCILIATION_LOCK_KEY = 'voice:reconcile:lock'; -const VOICE_RECONCILIATION_CADENCE_KEY = 'voice:reconcile:cadence'; -const VOICE_RECONCILIATION_LAST_SWEEP_KEY = 'voice:reconcile:last-sweep-at'; -const GATEWAY_ONLY_CANDIDATE_KEY_PREFIX = 'voice:reconcile:gateway-only:'; -const LIVEKIT_ONLY_CANDIDATE_KEY_PREFIX = 'voice:reconcile:livekit-only:'; - -export class VoiceReconciliationWorker { - private readonly gatewayService: IGatewayService; - private readonly liveKitService: ILiveKitService; - private readonly voiceRoomStore: IVoiceRoomStore; - private readonly kvClient: IKVProvider; - private readonly logger: ILogger; - private readonly intervalMs: number; - private readonly staggerDelayMs: number; - private readonly lockTtlSeconds: number; - private readonly cadenceTtlSeconds: number; - private readonly gatewayOnlyGraceMs: number; - private readonly liveKitOnlyGraceMs: number; - private observedSweepSpacingMs = 0; - private intervalHandle: NodeJS.Timeout | null = null; - private reconciling = false; - private reconciliationLockLost = false; - private activeReconciliation: Promise | null = null; - private stopping = false; - - constructor(options: VoiceReconciliationWorkerOptions) { - this.gatewayService = options.gatewayService; - this.liveKitService = options.liveKitService; - this.voiceRoomStore = options.voiceRoomStore; - this.kvClient = options.kvClient; - this.logger = options.logger.child({worker: 'VoiceReconciliationWorker'}); - this.intervalMs = options.intervalMs ?? DEFAULT_INTERVAL_MS; - this.staggerDelayMs = options.staggerDelayMs ?? DEFAULT_STAGGER_DELAY_MS; - this.lockTtlSeconds = - options.lockTtlSeconds ?? Math.max(DEFAULT_LOCK_TTL_SECONDS, Math.ceil((this.intervalMs * 3) / 1000)); - this.cadenceTtlSeconds = options.cadenceTtlSeconds ?? Math.max(1, Math.ceil((this.intervalMs * 3) / 1000)); - this.gatewayOnlyGraceMs = options.gatewayOnlyGraceMs ?? DEFAULT_GATEWAY_ONLY_GRACE_MS; - this.liveKitOnlyGraceMs = options.liveKitOnlyGraceMs ?? DEFAULT_LIVEKIT_ONLY_GRACE_MS; - } - - start(): void { - if (this.intervalHandle) { - this.logger.warn('VoiceReconciliationWorker is already running'); - return; - } - this.logger.info( - { - intervalMs: this.intervalMs, - gatewayOnlyGraceMs: this.gatewayOnlyGraceMs, - liveKitOnlyGraceMs: this.liveKitOnlyGraceMs, - gatewayOnlyCandidateTtlSeconds: this.candidateTtlSeconds(this.gatewayOnlyGraceMs), - }, - 'Starting VoiceReconciliationWorker', - ); - this.stopping = false; - void this.runReconciliation(); - this.intervalHandle = setInterval(() => { - void this.runReconciliation(); - }, this.intervalMs); - } - - async stop(): Promise { - this.stopping = true; - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - this.intervalHandle = null; - } - const activeReconciliation = this.activeReconciliation; - if (activeReconciliation !== null) { - await activeReconciliation; - } - this.logger.info('Stopped VoiceReconciliationWorker'); - } - - async reconcile(): Promise { - const startTime = Date.now(); - await this.recordSweepSpacing(startTime); - const discovery = await this.discoverActiveRooms(); - this.logger.info( - { - roomCount: discovery.rooms.length, - gatewayRoomsDiscovered: discovery.gatewayRoomsDiscovered, - liveKitRoomsDiscovered: discovery.liveKitRoomsDiscovered, - liveKitDiscoveryComplete: discovery.liveKitDiscoveryComplete, - liveKitDiscoveryErrors: discovery.liveKitDiscoveryErrors, - liveKitServersSearched: discovery.liveKitServersSearched, - gatewayDiscoveryFailed: discovery.gatewayDiscoveryFailed, - }, - 'Starting reconciliation sweep', - ); - let roomsChecked = 0; - let totalConfirmed = 0; - let totalRepaired = 0; - let totalDisconnected = 0; - let totalLiveKitOnlyDeferred = 0; - let totalGatewayRemoved = 0; - let totalGatewayDeferred = 0; - let totalGatewaySkipped = 0; - let totalConsistent = 0; - let totalErrors = 0; - let totalTransientSkips = 0; - for (const room of discovery.rooms) { - if (this.stopping) { - this.logger.info('Stopping reconciliation sweep because the worker is shutting down'); - break; - } - if (this.reconciliationLockLost) { - this.logger.warn('Stopping reconciliation sweep because the cluster lock was lost'); - break; - } - try { - const result = await this.reconcileRoom( - room, - discovery.liveKitLocationsByRoom.get(room.roomName) ?? [], - discovery.liveKitDiscoveryComplete, - ); - roomsChecked++; - totalConfirmed += result.livekitOnlyConfirmed; - totalRepaired += result.livekitOnlyRepaired; - totalDisconnected += result.livekitOnlyDisconnected; - totalLiveKitOnlyDeferred += result.livekitOnlyDeferred; - totalGatewayRemoved += result.gatewayOnlyRemoved; - totalGatewayDeferred += result.gatewayOnlyDeferred; - totalGatewaySkipped += result.gatewayOnlySkipped; - totalConsistent += result.consistent; - if (result.transientSkip) { - totalTransientSkips++; - } - } catch (error) { - totalErrors++; - this.logger.error({error, roomName: room.roomName}, 'Unexpected reconciliation failure; skipping room'); - } - if (this.staggerDelayMs > 0) { - await new Promise((resolve) => setTimeout(resolve, this.staggerDelayMs)); - } - } - const durationMs = Date.now() - startTime; - this.logger.info( - { - observedSweepSpacingMs: this.observedSweepSpacingMs, - gatewayOnlyCandidateTtlSeconds: this.candidateTtlSeconds(this.gatewayOnlyGraceMs), - roomsChecked, - totalConfirmed, - totalRepaired, - totalDisconnected, - totalLiveKitOnlyDeferred, - totalGatewayRemoved, - totalGatewayDeferred, - totalGatewaySkipped, - totalConsistent, - totalErrors, - totalTransientSkips, - durationMs, - }, - 'Reconciliation sweep complete', - ); - } - - private async runReconciliation(): Promise { - if (this.reconciling) { - this.logger.warn('Skipping reconciliation sweep; previous sweep still in progress'); - return; - } - this.reconciling = true; - const sweep = this.runReconciliationSweep(); - this.activeReconciliation = sweep; - try { - await sweep; - } finally { - this.activeReconciliation = null; - this.reconciling = false; - } - } - - private async runReconciliationSweep(): Promise { - let lockToken: string | null = null; - let lockRenewalHandle: NodeJS.Timeout | null = null; - try { - lockToken = await this.acquireReconciliationLock(); - if (lockToken === null) { - this.logger.debug('Skipping reconciliation sweep; lock held by another worker'); - return; - } - this.reconciliationLockLost = false; - lockRenewalHandle = this.startReconciliationLockRenewal(lockToken); - const shouldRun = await this.acquireCadenceLease(); - if (!shouldRun) { - this.logger.debug('Skipping reconciliation sweep; cadence lease held by another worker'); - return; - } - await this.reconcile(); - } catch (error) { - this.logger.error({error}, 'Reconciliation sweep failed unexpectedly'); - } finally { - if (lockRenewalHandle !== null) { - clearInterval(lockRenewalHandle); - } - if (lockToken !== null) { - await this.releaseReconciliationLock(lockToken); - } - this.reconciliationLockLost = false; - } - } - - private async acquireReconciliationLock(): Promise { - const token = randomUUID(); - try { - const acquired = await this.kvClient.acquireLock(VOICE_RECONCILIATION_LOCK_KEY, token, this.lockTtlSeconds); - if (!acquired) { - return null; - } - return token; - } catch (error) { - this.logger.error({error}, 'Failed to acquire voice reconciliation lock'); - return null; - } - } - - private candidateTtlSeconds(graceMs: number): number { - return candidateTtlSecondsFor({ - intervalMs: this.intervalMs, - observedSweepSpacingMs: this.observedSweepSpacingMs, - graceMs, - }); - } - - private async recordSweepSpacing(startedAt: number): Promise { - try { - const previous = await this.kvClient.get(VOICE_RECONCILIATION_LAST_SWEEP_KEY); - const previousAt = previous === null ? Number.NaN : Number(previous); - if (Number.isFinite(previousAt) && startedAt > previousAt) { - this.observedSweepSpacingMs = Math.max(this.observedSweepSpacingMs, startedAt - previousAt); - const gatewayOnlyCandidateTtlSeconds = this.candidateTtlSeconds(this.gatewayOnlyGraceMs); - if (this.observedSweepSpacingMs >= gatewayOnlyCandidateTtlSeconds * 1000) { - this.logger.warn( - { - observedSweepSpacingMs: this.observedSweepSpacingMs, - gatewayOnlyCandidateTtlSeconds, - }, - 'Reconciliation sweeps are further apart than the candidate TTL; divergent voice states will be deferred forever', - ); - } - } - await this.kvClient.setex(VOICE_RECONCILIATION_LAST_SWEEP_KEY, LAST_SWEEP_KEY_TTL_SECONDS, String(startedAt)); - } catch (error) { - this.logger.warn({error}, 'Failed to record reconciliation sweep spacing'); - } - } - - private async acquireCadenceLease(): Promise { - try { - return await this.kvClient.setnx(VOICE_RECONCILIATION_CADENCE_KEY, '1', this.cadenceTtlSeconds); - } catch (error) { - this.logger.error({error}, 'Failed to acquire voice reconciliation cadence lease'); - return false; - } - } - - private startReconciliationLockRenewal(token: string): NodeJS.Timeout { - const intervalMs = Math.max(1000, Math.floor((this.lockTtlSeconds * 1000) / 3)); - return setInterval(() => { - void this.renewReconciliationLock(token); - }, intervalMs); - } - - private async renewReconciliationLock(token: string): Promise { - try { - const renewed = await this.kvClient.extendLock(VOICE_RECONCILIATION_LOCK_KEY, token, this.lockTtlSeconds); - if (!renewed) { - this.reconciliationLockLost = true; - this.logger.warn('Voice reconciliation lock token no longer matched on renewal'); - } - } catch (error) { - this.logger.error({error}, 'Failed to renew voice reconciliation lock'); - } - } - - private async releaseReconciliationLock(token: string): Promise { - try { - const released = await this.kvClient.releaseLock(VOICE_RECONCILIATION_LOCK_KEY, token); - if (!released) { - this.logger.warn('Voice reconciliation lock token no longer matched on release'); - } - } catch (error) { - this.logger.error({error}, 'Failed to release voice reconciliation lock'); - } - } - - private async discoverActiveRooms(): Promise { - const roomsByName = new Map(); - const liveKitLocationsByRoom = new Map>(); - let gatewayRoomsDiscovered = 0; - let liveKitRoomsDiscovered = 0; - let gatewayDiscoveryFailed = false; - try { - const gatewayRooms = await this.gatewayService.getActiveVoiceRooms(); - for (const room of gatewayRooms.rooms) { - if (room.voiceStateCount <= 0) { - continue; - } - this.addDiscoveredRoom(roomsByName, { - roomName: buildRoomName(room.guildId, room.channelId), - guildId: room.guildId, - channelId: room.channelId, - fromGateway: true, - fromLiveKit: false, - gatewayVoiceStateCount: room.voiceStateCount, - }); - gatewayRoomsDiscovered++; - } - } catch (error) { - gatewayDiscoveryFailed = true; - this.logger.warn( - {error: errorMessage(error)}, - 'Failed to discover active voice rooms from gateway; using compatibility fallback', - ); - await this.discoverPinnedRoomsFallback(roomsByName); - } - let liveKitDiscoveryComplete = false; - let liveKitDiscoveryErrors = 0; - let liveKitServersSearched = 0; - try { - const liveKitRooms = await this.liveKitService.listActiveRooms(); - liveKitDiscoveryComplete = liveKitRooms.completed; - liveKitDiscoveryErrors = liveKitRooms.errors.length; - liveKitServersSearched = liveKitRooms.searchedServers; - for (const location of liveKitRooms.rooms) { - const parsed = parseRoomName(location.roomName); - if (!parsed) { - this.logger.warn({roomName: location.roomName}, 'Skipping LiveKit room with unrecognized voice room name'); - continue; - } - const guildId = parsed.type === 'guild' ? parsed.guildId : undefined; - this.addDiscoveredRoom(roomsByName, { - roomName: location.roomName, - guildId, - channelId: parsed.channelId, - fromGateway: false, - fromLiveKit: true, - gatewayVoiceStateCount: 0, - }); - const locations = liveKitLocationsByRoom.get(location.roomName) ?? []; - if (!locations.some((existing) => sameLiveKitLocation(existing, location))) { - locations.push(location); - liveKitLocationsByRoom.set(location.roomName, locations); - } - liveKitRoomsDiscovered++; - } - } catch (error) { - this.logger.warn({error: errorMessage(error)}, 'Failed to list active LiveKit rooms'); - } - return { - rooms: Array.from(roomsByName.values()).sort((left, right) => left.roomName.localeCompare(right.roomName)), - liveKitLocationsByRoom, - liveKitDiscoveryComplete, - liveKitDiscoveryErrors, - liveKitServersSearched, - gatewayRoomsDiscovered, - liveKitRoomsDiscovered, - gatewayDiscoveryFailed, - }; - } - - private addDiscoveredRoom(roomsByName: Map, room: DiscoveredRoom): void { - const existing = roomsByName.get(room.roomName); - if (!existing) { - roomsByName.set(room.roomName, room); - return; - } - roomsByName.set(room.roomName, { - ...existing, - fromGateway: existing.fromGateway || room.fromGateway, - fromLiveKit: existing.fromLiveKit || room.fromLiveKit, - gatewayVoiceStateCount: existing.gatewayVoiceStateCount + room.gatewayVoiceStateCount, - }); - } - - private async discoverPinnedRoomsFallback(roomsByName: Map): Promise { - try { - const rooms = await this.voiceRoomStore.listPinnedRooms(); - if (rooms.length > 0) { - for (const room of rooms) { - this.addDiscoveredRoom(roomsByName, { - roomName: buildRoomName(room.guildId, room.channelId), - guildId: room.guildId, - channelId: room.channelId, - fromGateway: false, - fromLiveKit: false, - gatewayVoiceStateCount: 0, - }); - } - return; - } - } catch (error) { - this.logger.warn({error}, 'Failed to discover voice rooms from voice room store; falling back to KV scan'); - } - const keys = await this.kvClient.scan(`${ROOM_KEY_PREFIX}*`, 1000); - for (const key of keys) { - const suffix = key.slice(ROOM_KEY_PREFIX.length); - const parsed = parsePinnedRoomKey(suffix); - if (!parsed) { - continue; - } - this.addDiscoveredRoom(roomsByName, { - roomName: buildRoomName(parsed.guildId, parsed.channelId), - guildId: parsed.guildId, - channelId: parsed.channelId, - fromGateway: false, - fromLiveKit: false, - gatewayVoiceStateCount: 0, - }); - } - } - - private async reconcileRoom( - room: DiscoveredRoom, - discoveredLiveKitLocations: Array, - liveKitDiscoveryComplete: boolean, - ): Promise { - let voiceStates: Array; - let pendingJoins: Array; - try { - const [voiceStatesResult, pendingJoinsResult] = await Promise.all([ - this.gatewayService.getVoiceStatesForChannel({guildId: room.guildId, channelId: room.channelId}), - this.gatewayService.getPendingJoinsForChannel({guildId: room.guildId, channelId: room.channelId}), - ]); - voiceStates = voiceStatesResult.voiceStates; - pendingJoins = pendingJoinsResult.pendingJoins; - } catch (error) { - if (VoiceReconciliationWorker.isTransientGatewayError(error)) { - this.logger.warn( - {roomName: room.roomName, error: errorMessage(error)}, - 'Skipping room this sweep — gateway RPC temporarily unavailable', - ); - return emptyRoomResult(room.roomName, {transientSkip: true}); - } - throw error; - } - const liveKitLocations = await this.resolveLiveKitLocations( - room, - discoveredLiveKitLocations, - voiceStates, - liveKitDiscoveryComplete, - ); - const liveKitSnapshot = await this.getLiveKitRoomSnapshot(room, liveKitLocations, liveKitDiscoveryComplete); - const liveKitConnectionIds = new Set(liveKitSnapshot.participants.map((participant) => participant.connectionId)); - const gatewayConnectionIds = new Set(voiceStates.map((voiceState) => voiceState.connectionId)); - const pendingJoinByConnectionId = new Map(); - for (const pendingJoin of pendingJoins) { - pendingJoinByConnectionId.set(pendingJoin.connectionId, pendingJoin); - } - let livekitOnlyConfirmed = 0; - let livekitOnlyRepaired = 0; - let livekitOnlyDisconnected = 0; - let livekitOnlyDeferred = 0; - for (const participant of liveKitSnapshot.participants) { - await this.clearGatewayOnlyCandidate(room.guildId, room.channelId, participant.connectionId); - if (gatewayConnectionIds.has(participant.connectionId)) { - await this.clearLiveKitOnlyCandidate(room.guildId, room.channelId, participant); - continue; - } - const pendingJoin = pendingJoinByConnectionId.get(participant.connectionId); - if (pendingJoin && pendingJoin.expiresAt > Date.now()) { - await this.confirmPendingJoin( - room.guildId, - room.channelId, - participant.connectionId, - pendingJoin, - room.roomName, - ); - await this.clearLiveKitOnlyCandidate(room.guildId, room.channelId, participant); - livekitOnlyConfirmed++; - } else { - const repairResult = await this.repairLiveKitOnlyParticipant(room, participant); - if (repairResult === 'repaired') { - await this.clearLiveKitOnlyCandidate(room.guildId, room.channelId, participant); - livekitOnlyRepaired++; - continue; - } - if (repairResult === 'defer') { - livekitOnlyDeferred++; - continue; - } - if (!liveKitSnapshot.completed) { - this.logger.warn( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }, - 'Deferring LiveKit-only participant because LiveKit snapshot was incomplete', - ); - livekitOnlyDeferred++; - continue; - } - const shouldDisconnect = await this.confirmLiveKitOnlyCandidate(room, participant); - if (!shouldDisconnect) { - this.logger.warn( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }, - 'Deferring LiveKit-only participant; gateway state may be temporarily incomplete', - ); - livekitOnlyDeferred++; - continue; - } - await this.disconnectLiveKitOnlyParticipant(room, participant); - livekitOnlyDisconnected++; - } - } - let consistent = 0; - let gatewayOnlyRemoved = 0; - let gatewayOnlyDeferred = 0; - let gatewayOnlySkipped = 0; - for (const voiceState of voiceStates) { - if (liveKitConnectionIds.has(voiceState.connectionId)) { - consistent++; - continue; - } - if (!liveKitSnapshot.gatewayStateRemovalComplete) { - gatewayOnlySkipped++; - continue; - } - const shouldRemove = await this.confirmGatewayOnlyCandidate(room.guildId, room.channelId, voiceState); - if (!shouldRemove) { - gatewayOnlyDeferred++; - continue; - } - await this.removeGhostState(room.guildId, voiceState, room.channelId, room.roomName); - gatewayOnlyRemoved++; - } - if ( - livekitOnlyConfirmed > 0 || - livekitOnlyRepaired > 0 || - livekitOnlyDisconnected > 0 || - livekitOnlyDeferred > 0 || - gatewayOnlyRemoved > 0 || - gatewayOnlyDeferred > 0 || - gatewayOnlySkipped > 0 - ) { - this.logger.info( - { - roomName: room.roomName, - livekitOnlyConfirmed, - livekitOnlyRepaired, - livekitOnlyDisconnected, - livekitOnlyDeferred, - gatewayOnlyRemoved, - gatewayOnlyDeferred, - gatewayOnlySkipped, - consistent, - liveKitLocations: liveKitSnapshot.searchedLocations, - liveKitErrors: liveKitSnapshot.errors, - gatewayStateRemovalComplete: liveKitSnapshot.gatewayStateRemovalComplete, - fromGateway: room.fromGateway, - fromLiveKit: room.fromLiveKit, - }, - 'Room reconciliation found divergence', - ); - } - return { - roomName: room.roomName, - livekitOnlyConfirmed, - livekitOnlyRepaired, - livekitOnlyDisconnected, - livekitOnlyDeferred, - gatewayOnlyRemoved, - gatewayOnlyDeferred, - gatewayOnlySkipped, - consistent, - }; - } - - private async resolveLiveKitLocations( - room: DiscoveredRoom, - discoveredLocations: Array, - voiceStates: Array, - liveKitDiscoveryComplete: boolean, - ): Promise> { - const locations: Array = discoveredLocations.map((location) => ({ - ...location, - authoritativeForGatewayState: false, - })); - if (liveKitDiscoveryComplete) { - return locations; - } - for (const voiceState of voiceStates) { - if (voiceState.regionId === undefined || voiceState.serverId === undefined) { - continue; - } - if (this.liveKitService.getServer(voiceState.regionId, voiceState.serverId) === null) { - continue; - } - addResolvedLiveKitLocation(locations, { - roomName: room.roomName, - regionId: voiceState.regionId, - serverId: voiceState.serverId, - authoritativeForGatewayState: true, - }); - } - try { - const pinned = await this.voiceRoomStore.getPinnedRoomServer(room.guildId, room.channelId); - if (pinned && this.liveKitService.getServer(pinned.regionId, pinned.serverId) !== null) { - addResolvedLiveKitLocation(locations, { - roomName: room.roomName, - regionId: pinned.regionId, - serverId: pinned.serverId, - authoritativeForGatewayState: true, - }); - } - } catch (error) { - this.logger.warn({error, roomName: room.roomName}, 'Failed to read pinned room server as fallback hint'); - } - return locations; - } - - private async getLiveKitRoomSnapshot( - room: DiscoveredRoom, - locations: Array, - liveKitDiscoveryComplete: boolean, - ): Promise { - const participants: Array = []; - let errors = 0; - for (const location of locations) { - const listResult = await this.liveKitService.listParticipants({ - guildId: room.guildId, - channelId: room.channelId, - regionId: location.regionId, - serverId: location.serverId, - }); - if (listResult.status === 'error') { - errors++; - this.logger.warn( - { - roomName: room.roomName, - regionId: location.regionId, - serverId: location.serverId, - errorCode: listResult.errorCode, - retryable: listResult.retryable, - }, - 'Skipping LiveKit location this sweep — listParticipants failed', - ); - continue; - } - for (const participant of listResult.participants) { - const parsed = parseParticipantIdentity(participant.identity); - if (!parsed) { - this.logger.warn( - {roomName: room.roomName, identity: participant.identity}, - 'Could not parse participant identity; skipping', - ); - continue; - } - participants.push({ - identity: participant.identity, - userId: parsed.userId, - connectionId: parsed.connectionId, - regionId: location.regionId, - serverId: location.serverId, - }); - } - } - const searchedAuthoritativeGatewayLocation = locations.some((location) => location.authoritativeForGatewayState); - const noRoomSpecificErrors = errors === 0; - return { - participants, - completed: liveKitDiscoveryComplete && noRoomSpecificErrors, - gatewayStateRemovalComplete: - noRoomSpecificErrors && (liveKitDiscoveryComplete || searchedAuthoritativeGatewayLocation), - errors, - searchedLocations: locations.length, - }; - } - - private async confirmPendingJoin( - guildId: GuildID | undefined, - channelId: ChannelID, - connectionId: string, - pendingJoin: GatewayPendingJoinEntry, - roomName: string, - ): Promise { - try { - const result = await this.gatewayService.confirmVoiceConnection({ - guildId, - channelId, - connectionId, - tokenNonce: pendingJoin.tokenNonce, - }); - if (!result.success) { - this.logger.warn( - {roomName, connectionId, error: result.error}, - 'Gateway rejected voice connection confirmation', - ); - } - } catch (error) { - if (VoiceReconciliationWorker.isTransientGatewayError(error)) { - this.logger.warn( - {roomName, connectionId, error: errorMessage(error)}, - 'Deferring pending-join confirmation — gateway temporarily unavailable', - ); - return; - } - this.logger.error({error, roomName, connectionId}, 'Failed to confirm pending voice connection'); - } - } - - private async repairLiveKitOnlyParticipant( - room: DiscoveredRoom, - participant: LiveKitParticipantEntry, - ): Promise { - if (room.guildId === undefined) { - return 'not_repairable'; - } - try { - const result = await this.gatewayService.repairVoiceStateFromCache({ - guildId: room.guildId, - channelId: room.channelId, - userId: participant.userId, - connectionId: participant.connectionId, - }); - if (result.success) { - this.logger.info( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - repaired: result.repaired ?? false, - }, - 'Repaired LiveKit-only participant from gateway cache', - ); - return 'repaired'; - } - this.logger.debug( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - error: result.error, - }, - 'LiveKit-only participant could not be repaired from gateway cache', - ); - if (VoiceReconciliationWorker.isDefinitiveVoiceRepairMiss(result.error)) { - return 'not_repairable'; - } - return 'defer'; - } catch (error) { - if (VoiceReconciliationWorker.isTransientGatewayError(error)) { - this.logger.warn( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - error: errorMessage(error), - }, - 'Deferring LiveKit-only repair because gateway is temporarily unavailable', - ); - return 'defer'; - } - this.logger.error( - {error, roomName: room.roomName, userId: participant.userId.toString(), connectionId: participant.connectionId}, - 'Failed to repair LiveKit-only participant from gateway cache', - ); - return 'defer'; - } - } - - private async disconnectLiveKitOnlyParticipant( - room: DiscoveredRoom, - participant: LiveKitParticipantEntry, - ): Promise { - this.logger.warn( - { - roomName: room.roomName, - userId: participant.userId.toString(), - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }, - 'Disconnecting confirmed orphan LiveKit participant', - ); - await this.liveKitService.disconnectParticipant({ - guildId: room.guildId, - channelId: room.channelId, - userId: participant.userId, - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }); - } - - private async removeGhostState( - guildId: GuildID | undefined, - voiceState: GatewayVoiceStateEntry, - channelId: ChannelID, - roomName: string, - ): Promise { - try { - this.logger.info( - {roomName, userId: voiceState.userId, connectionId: voiceState.connectionId}, - 'Removing confirmed ghost voice state from gateway', - ); - const result = await this.gatewayService.disconnectVoiceUserIfInChannel({ - guildId, - channelId, - userId: createUserID(BigInt(voiceState.userId)), - connectionId: voiceState.connectionId, - }); - if (result.ignored) { - this.logger.debug( - {roomName, userId: voiceState.userId, connectionId: voiceState.connectionId}, - 'Gateway ignored ghost state removal (user may have moved)', - ); - } - } catch (error) { - if (VoiceReconciliationWorker.isTransientGatewayError(error)) { - this.logger.warn( - {roomName, userId: voiceState.userId, connectionId: voiceState.connectionId, error: errorMessage(error)}, - 'Deferring ghost state removal — gateway temporarily unavailable', - ); - return; - } - this.logger.error( - {error, roomName, userId: voiceState.userId, connectionId: voiceState.connectionId}, - 'Failed to remove ghost voice state from gateway', - ); - } - } - - private async confirmGatewayOnlyCandidate( - guildId: GuildID | undefined, - channelId: ChannelID, - voiceState: GatewayVoiceStateEntry, - ): Promise { - if (this.gatewayOnlyGraceMs <= 0) { - return true; - } - const key = gatewayOnlyCandidateKey(guildId, channelId, voiceState.connectionId); - const now = Date.now(); - try { - const existing = await this.kvClient.get(key); - const firstSeen = existing === null ? Number.NaN : Number(existing); - if (Number.isFinite(firstSeen) && now - firstSeen >= this.gatewayOnlyGraceMs) { - await this.kvClient.del(key); - return true; - } - await this.kvClient.setex( - key, - this.candidateTtlSeconds(this.gatewayOnlyGraceMs), - Number.isFinite(firstSeen) ? String(firstSeen) : String(now), - ); - return false; - } catch (error) { - this.logger.warn( - {error, guildId, channelId, connectionId: voiceState.connectionId}, - 'Deferring gateway-only removal because candidate state could not be recorded', - ); - return false; - } - } - - private async confirmLiveKitOnlyCandidate( - room: DiscoveredRoom, - participant: LiveKitParticipantEntry, - ): Promise { - if (this.liveKitOnlyGraceMs <= 0) { - return true; - } - const key = liveKitOnlyCandidateKey(room.guildId, room.channelId, participant); - const now = Date.now(); - try { - const existing = await this.kvClient.get(key); - const firstSeen = existing === null ? Number.NaN : Number(existing); - if (Number.isFinite(firstSeen) && now - firstSeen >= this.liveKitOnlyGraceMs) { - await this.kvClient.del(key); - return true; - } - await this.kvClient.setex( - key, - this.candidateTtlSeconds(this.liveKitOnlyGraceMs), - Number.isFinite(firstSeen) ? String(firstSeen) : String(now), - ); - return false; - } catch (error) { - this.logger.warn( - { - error, - guildId: room.guildId, - channelId: room.channelId, - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }, - 'Deferring LiveKit-only removal because candidate state could not be recorded', - ); - return false; - } - } - - private async clearGatewayOnlyCandidate( - guildId: GuildID | undefined, - channelId: ChannelID, - connectionId: string, - ): Promise { - try { - await this.kvClient.del(gatewayOnlyCandidateKey(guildId, channelId, connectionId)); - } catch (error) { - this.logger.debug({error, guildId, channelId, connectionId}, 'Failed to clear gateway-only candidate marker'); - } - } - - private async clearLiveKitOnlyCandidate( - guildId: GuildID | undefined, - channelId: ChannelID, - participant: LiveKitParticipantEntry, - ): Promise { - try { - await this.kvClient.del(liveKitOnlyCandidateKey(guildId, channelId, participant)); - } catch (error) { - this.logger.debug( - { - error, - guildId, - channelId, - connectionId: participant.connectionId, - regionId: participant.regionId, - serverId: participant.serverId, - }, - 'Failed to clear LiveKit-only candidate marker', - ); - } - } - - private static isTransientGatewayError(error: unknown): boolean { - return ( - error instanceof ServiceUnavailableError || - error instanceof GatewayTimeoutError || - error instanceof BadGatewayError - ); - } - - private static isDefinitiveVoiceRepairMiss(error: string | undefined): boolean { - return ( - error === undefined || - error === 'connection_not_found' || - error === 'voice_state_mismatch' || - error === 'voice_invalid_state' - ); - } -} - -function emptyRoomResult(roomName: string, extra: Partial = {}): RoomReconciliationResult { - return { - roomName, - livekitOnlyConfirmed: 0, - livekitOnlyRepaired: 0, - livekitOnlyDisconnected: 0, - livekitOnlyDeferred: 0, - gatewayOnlyRemoved: 0, - gatewayOnlyDeferred: 0, - gatewayOnlySkipped: 0, - consistent: 0, - ...extra, - }; -} - -function parsePinnedRoomKey(suffix: string): {guildId?: GuildID; channelId: ChannelID} | null { - if (suffix.startsWith('guild:')) { - const parts = suffix.split(':'); - if (parts.length !== 3) { - return null; - } - try { - return {guildId: BigInt(parts[1]) as GuildID, channelId: BigInt(parts[2]) as ChannelID}; - } catch { - return null; - } - } - if (suffix.startsWith('dm:')) { - try { - return {channelId: BigInt(suffix.slice(3)) as ChannelID}; - } catch { - return null; - } - } - return null; -} - -function buildRoomName(guildId: GuildID | undefined, channelId: ChannelID): string { - if (guildId === undefined) { - return `dm_channel_${channelId.toString()}`; - } - return `guild_${guildId.toString()}_channel_${channelId.toString()}`; -} - -function sameLiveKitLocation(left: LiveKitRoomLocation, right: LiveKitRoomLocation): boolean { - return left.regionId === right.regionId && left.serverId === right.serverId && left.roomName === right.roomName; -} - -function addResolvedLiveKitLocation( - locations: Array, - location: ResolvedLiveKitLocation, -): void { - const index = locations.findIndex((existing) => sameLiveKitLocation(existing, location)); - if (index === -1) { - locations.push(location); - return; - } - if (location.authoritativeForGatewayState && !locations[index].authoritativeForGatewayState) { - locations[index] = {...locations[index], authoritativeForGatewayState: true}; - } -} - -function gatewayOnlyCandidateKey(guildId: GuildID | undefined, channelId: ChannelID, connectionId: string): string { - const scope = guildId === undefined ? 'dm' : `guild:${guildId.toString()}`; - return `${GATEWAY_ONLY_CANDIDATE_KEY_PREFIX}${scope}:channel:${channelId.toString()}:connection:${encodeURIComponent( - connectionId, - )}`; -} - -function liveKitOnlyCandidateKey( - guildId: GuildID | undefined, - channelId: ChannelID, - participant: LiveKitParticipantEntry, -): string { - const scope = guildId === undefined ? 'dm' : `guild:${guildId.toString()}`; - return `${LIVEKIT_ONLY_CANDIDATE_KEY_PREFIX}${scope}:channel:${channelId.toString()}:connection:${encodeURIComponent( - participant.connectionId, - )}:region:${encodeURIComponent(participant.regionId)}:server:${encodeURIComponent(participant.serverId)}`; -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} diff --git a/fluxer_api/src/api/voice/tests/VoiceReconciliationWorkerStop.test.ts b/fluxer_api/src/api/voice/tests/VoiceReconciliationWorkerStop.test.ts deleted file mode 100644 index 75aecbede..000000000 --- a/fluxer_api/src/api/voice/tests/VoiceReconciliationWorkerStop.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider'; -import {describe, expect, it, vi} from 'vitest'; -import type {ILogger} from '../../ILogger'; -import type {IGatewayService} from '../../infrastructure/IGatewayService'; -import type {ILiveKitService} from '../../infrastructure/ILiveKitService'; -import type {IVoiceRoomStore} from '../../infrastructure/IVoiceRoomStore'; -import {VoiceReconciliationWorker} from '../VoiceReconciliationWorker'; - -function createLogger(): ILogger { - const logger = { - trace: vi.fn(), - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - child: () => logger, - }; - return logger as unknown as ILogger; -} - -function createHarness() { - const releaseLock = vi.fn().mockResolvedValue(true); - const kvClient = { - acquireLock: vi.fn().mockResolvedValue(true), - extendLock: vi.fn().mockResolvedValue(true), - releaseLock, - setnx: vi.fn().mockResolvedValue(true), - get: vi.fn().mockResolvedValue(null), - setex: vi.fn().mockResolvedValue(undefined), - } as unknown as IKVProvider; - let finishDiscovery: () => void = () => {}; - const discovery = new Promise<{rooms: []}>((resolve) => { - finishDiscovery = () => resolve({rooms: []}); - }); - const getActiveVoiceRooms = vi.fn().mockReturnValue(discovery); - const worker = new VoiceReconciliationWorker({ - gatewayService: {getActiveVoiceRooms} as unknown as IGatewayService, - liveKitService: { - listActiveRooms: async () => ({rooms: [], errors: [], completed: true, searchedServers: 0}), - } as unknown as ILiveKitService, - voiceRoomStore: {listPinnedRooms: async () => []} as unknown as IVoiceRoomStore, - kvClient, - logger: createLogger(), - intervalMs: 60000, - staggerDelayMs: 0, - }); - return {worker, releaseLock, getActiveVoiceRooms, finishDiscovery: () => finishDiscovery()}; -} - -describe('VoiceReconciliationWorker stop', () => { - it('releases the reconciliation lock before stop resolves', async () => { - const {worker, releaseLock, getActiveVoiceRooms, finishDiscovery} = createHarness(); - - worker.start(); - await vi.waitFor(() => expect(getActiveVoiceRooms).toHaveBeenCalled()); - - setTimeout(finishDiscovery, 0); - await worker.stop(); - - expect(releaseLock).toHaveBeenCalledTimes(1); - }); -}); diff --git a/fluxer_api/src/api/worker/WorkerDependencies.ts b/fluxer_api/src/api/worker/WorkerDependencies.ts index a6ef6961e..b0b1d8c24 100644 --- a/fluxer_api/src/api/worker/WorkerDependencies.ts +++ b/fluxer_api/src/api/worker/WorkerDependencies.ts @@ -114,7 +114,6 @@ import type {UserContactChangeLogService} from '../user/services/UserContactChan import {UserDeletionEligibilityService} from '../user/services/UserDeletionEligibilityService'; import {UserHarvestRepository} from '../user/UserHarvestRepository'; import type {UserPermissionUtils} from '../utils/UserPermissionUtils'; -import {VoiceReconciliationWorker} from '../voice/VoiceReconciliationWorker'; import type {VoiceRepository} from '../voice/VoiceRepository'; import type {VoiceTopology} from '../voice/VoiceTopology'; import type {WorkerTaskName} from './WorkerLaneConfig'; @@ -165,7 +164,6 @@ export interface WorkerDependencies { voiceRoomStore: IVoiceRoomStore; liveKitService: ILiveKitService; voiceTopology: VoiceTopology | null; - voiceReconciliationWorker: VoiceReconciliationWorker | null; channelService: ChannelService; guildAuditLogService: GuildAuditLogService; contactChangeLogService: UserContactChangeLogService; @@ -231,25 +229,8 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS const voiceRoomStore = getVoiceRoomStoreInstance() ?? new InMemoryVoiceRoomStore(); const liveKitService = getLiveKitServiceInstance() ?? new DisabledLiveKitService(); const voiceAvailabilityService = getVoiceAvailabilityService(); - const voiceReconciliationEnabled = Config.worker.enableVoiceReconciliation; - const voiceReconciliationWorker = - Config.voice.enabled && voiceTopology !== null && voiceReconciliationEnabled - ? new VoiceReconciliationWorker({ - gatewayService, - liveKitService, - voiceRoomStore, - kvClient, - logger: Logger, - intervalMs: Config.worker.voiceReconciliation.intervalMs, - staggerDelayMs: Config.worker.voiceReconciliation.staggerDelayMs, - lockTtlSeconds: Config.worker.voiceReconciliation.lockTtlSeconds, - cadenceTtlSeconds: Config.worker.voiceReconciliation.cadenceTtlSeconds, - gatewayOnlyGraceMs: Config.worker.voiceReconciliation.gatewayOnlyGraceMs, - liveKitOnlyGraceMs: Config.worker.voiceReconciliation.liveKitOnlyGraceMs, - }) - : null; if (Config.voice.enabled && voiceTopology !== null) { - Logger.info({reconciliationEnabled: voiceReconciliationEnabled}, 'Voice services initialized'); + Logger.info('Voice services initialized'); } const inviteRepository = getInviteRepository(); const webhookRepository = getWebhookRepository(); @@ -337,7 +318,6 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS voiceRoomStore, liveKitService, voiceTopology, - voiceReconciliationWorker, channelService, guildService, donationRepository, @@ -348,11 +328,3 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS stripe, }; } - -export async function shutdownWorkerDependencies(deps: WorkerDependencies): Promise { - Logger.info('Shutting down worker dependencies...'); - if (deps.voiceReconciliationWorker !== null) { - await deps.voiceReconciliationWorker.stop(); - } - Logger.info('Worker dependencies shut down successfully'); -} diff --git a/fluxer_api/src/api/worker/WorkerMain.ts b/fluxer_api/src/api/worker/WorkerMain.ts index f6fba54dd..89a0b1620 100644 --- a/fluxer_api/src/api/worker/WorkerMain.ts +++ b/fluxer_api/src/api/worker/WorkerMain.ts @@ -22,7 +22,7 @@ import {initializeSearch, shutdownSearch} from '../SearchFactory'; import {CronScheduler} from './CronScheduler'; import {JetStreamWorkerQueue} from './JetStreamWorkerQueue'; import {clearWorkerDependencies, setWorkerDependencies} from './WorkerContext'; -import {initializeWorkerDependencies, shutdownWorkerDependencies, type WorkerDependencies} from './WorkerDependencies'; +import {initializeWorkerDependencies, type WorkerDependencies} from './WorkerDependencies'; import {WorkerHeartbeat} from './WorkerHeartbeat'; import { resolveCronSchedulerEnabled, @@ -118,11 +118,8 @@ export async function startWorkerMain(): Promise { await jsConnectionManager?.drain(); jsConnectionManager = null; }); - await cleanupStep('worker dependencies', async () => { - if (dependencies) { - await shutdownWorkerDependencies(dependencies); - dependencies = null; - } + await cleanupStep('worker dependencies', () => { + dependencies = null; clearWorkerDependencies(); setInjectedWorkerService(undefined); }); @@ -268,10 +265,6 @@ export async function startWorkerMain(): Promise { } else { Logger.info('Search initialisation skipped for worker lanes without search tasks'); } - if (dependencies.voiceReconciliationWorker !== null) { - dependencies.voiceReconciliationWorker.start(); - Logger.info('VoiceReconciliationWorker started'); - } if (cronSchedulerEnabled) { cron.start(); Logger.info('Cron scheduler started'); diff --git a/fluxer_recon/Cargo.toml b/fluxer_recon/Cargo.toml new file mode 100644 index 000000000..ce00b1c94 --- /dev/null +++ b/fluxer_recon/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "fluxer-recon" +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +fluxer-svc = { path = "../fluxer_svc" } +anyhow = "1.0.104" +thiserror = "2.0.18" +tokio = { version = "1.52.3", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +tracing = "0.1.44" +reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] } +hmac = "0.13.0" +sha2 = "0.11.0" +base64 = "0.22.1" +axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] } +scylla = { version = "1.6.0", optional = true } + +[features] +default = [] +scylla = ["dep:scylla", "fluxer-svc/scylla"] diff --git a/fluxer_recon/Dockerfile b/fluxer_recon/Dockerfile new file mode 100644 index 000000000..f6f208eb1 --- /dev/null +++ b/fluxer_recon/Dockerfile @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +FROM rust:1-bookworm AS builder + +WORKDIR /usr/src/app + +COPY . . + +RUN cargo build --release -p fluxer-recon --features scylla + +FROM debian:bookworm-slim + +ARG BUILD_VERSION="" +ARG SOURCE_SHA="" +ARG SOURCE_DATE="" + +LABEL org.opencontainers.image.title="fluxer-recon" +LABEL org.opencontainers.image.description="Fluxer voice reconciliation service" +LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" +LABEL org.opencontainers.image.vendor="Fluxer" +LABEL org.opencontainers.image.url="https://fluxer.app" +LABEL org.opencontainers.image.documentation="https://docs.fluxer.app" +LABEL org.opencontainers.image.source="https://github.com/fluxerapp/fluxer" +LABEL org.opencontainers.image.version="${BUILD_VERSION}" +LABEL org.opencontainers.image.revision="${SOURCE_SHA}" +LABEL org.opencontainers.image.created="${SOURCE_DATE}" +LABEL app.fluxer.build-version="${BUILD_VERSION}" + +WORKDIR /usr/local/bin + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/src/app/target/release/fluxer-recon /usr/local/bin/fluxer-recon + +ENV BUILD_VERSION="${BUILD_VERSION}" + +USER 65532:65532 + +EXPOSE 8090 + +CMD ["/usr/local/bin/fluxer-recon"] diff --git a/fluxer_recon/src/actuate.rs b/fluxer_recon/src/actuate.rs new file mode 100644 index 000000000..0482afade --- /dev/null +++ b/fluxer_recon/src/actuate.rs @@ -0,0 +1,1325 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::budget::{ + AuthorizationOutcome, AuthorizationRequest, BudgetClass, MutationAuthorization, +}; +use crate::clock::Clock; +use crate::config::{ConnectionIdGuard, ReconConfig}; +use crate::decide::{Decision, DecisionAction}; +use crate::evidence::{GatewayVoiceState, MediaSighting, PendingJoin}; +use crate::gateway::codes::{GatewayError, GatewayErrorCode}; +use crate::gateway::{DisconnectOutcome, GatewayApi, GatewayFault, Nonce, RepairOutcome}; +use crate::guards::{ + AbortReason, DmPosture, FleetSilence, GatewayPreflight, GatewayRemovalGuard, MediaCoverage, + MediaCustody, MediaRemovalGuard, PreflightVoiceState, PreflightWindow, UnknownMediaCause, + connection_id_is_honoured, fleet_silence_cause, gateway_removal_preflight, + media_coverage_cause, media_coverage_refusal, media_custody_cause, media_removal_preflight, +}; +use crate::health::ModeClamp; +use crate::ids::{ConnectionId, ConnectionKey, Location, Millis, RoomKey, TurnId, UserId}; +use crate::ledger::{ + ActionKind, ConnectionLedger, ConnectionState, Ledger, LocationIndex, RepairVerdict, RoomLedger, +}; +use crate::livekit::{LiveKitApi, LiveKitFault, ReadResult, RemoveOutcome}; +use crate::metrics::{MutationOutcome, ReconMetrics}; +use crate::names::{ParticipantIdentity, parse_participant_identity}; +use crate::runtime::{RuntimeState, Shared}; +use crate::turn::{Digest, Fresh, ReadSource, TurnError, TurnRecord, TurnScope, TurnSequencer}; + +pub const PREFLIGHT_GATEWAY_READS: u32 = 2; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Lane { + Constructive, + Destructive, +} + +impl Lane { + pub const fn label(self) -> &'static str { + match self { + Self::Constructive => "constructive", + Self::Destructive => "destructive", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Actuation { + Held, + Refused { + lane: Lane, + reason: AbortReason, + }, + Counterfactual { + lane: Lane, + reason: AbortReason, + }, + Issued { + lane: Lane, + call: &'static str, + outcome: &'static str, + }, + Failed { + lane: Lane, + call: &'static str, + fault: &'static str, + }, +} + +impl Actuation { + pub const fn called_out(&self) -> bool { + matches!(self, Self::Issued { .. } | Self::Failed { .. }) + } + + pub const fn mutated(&self) -> bool { + matches!(self, Self::Issued { .. }) + } + + pub const fn lane(&self) -> Option { + match self { + Self::Held => None, + Self::Refused { lane, .. } + | Self::Counterfactual { lane, .. } + | Self::Issued { lane, .. } + | Self::Failed { lane, .. } => Some(*lane), + } + } + + pub const fn refusal(&self) -> Option { + match self { + Self::Held | Self::Issued { .. } | Self::Failed { .. } => None, + Self::Refused { reason, .. } | Self::Counterfactual { reason, .. } => Some(*reason), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ActuationPolicy { + pub dm_posture: DmPosture, + pub connection_id_guard: ConnectionIdGuard, + pub preflight_max_age_ms: u64, + pub pending_join_skew_ms: u64, +} + +impl ActuationPolicy { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + dm_posture: DmPosture { + dm_gateway_eviction: config.dm_gateway_eviction, + dm_media_eviction: config.dm_media_eviction, + }, + connection_id_guard: config.gateway_connection_id_guard, + preflight_max_age_ms: config.preflight_max_age_ms, + pending_join_skew_ms: config.pending_join_skew_ms, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TurnFacts { + pub room: RoomKey, + pub authorizing_turn: TurnId, + pub media: Vec, +} + +impl TurnFacts { + pub fn media_connections(&self) -> Vec { + self.media + .iter() + .map(|sighting| sighting.connection.clone()) + .collect() + } + + pub fn media_siblings_of(&self, user_id: UserId, target: &ConnectionId) -> Vec { + self.media + .iter() + .filter(|sighting| sighting.user_id == user_id && &sighting.connection != target) + .map(|sighting| sighting.connection.clone()) + .collect() + } +} + +pub struct Ports<'a, G, L> { + pub gateway: &'a G, + pub livekit: &'a L, + pub clock: &'a dyn Clock, + pub shared: &'a Shared, + pub policy: ActuationPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreflightReads { + pub voice_states: Vec, + pub pending_joins: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PreflightError { + Gateway(GatewayFault), + Turn(TurnError), +} + +impl PreflightError { + pub const fn abort_reason(&self) -> AbortReason { + match self { + Self::Gateway(_) => AbortReason::UnknownGateway, + Self::Turn(_) => AbortReason::PreflightDisagreed, + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Gateway(fault) => fault.label(), + Self::Turn(_) => "turn_violation", + } + } +} + +fn digest_voice_states(states: &[GatewayVoiceState]) -> u64 { + let mut digest = Digest::new().text("preflight_voice_states"); + for state in states { + digest = digest + .text(state.connection.as_ref().map_or("", ConnectionId::as_str)) + .flag(state.connection.is_some()) + .number(state.user_id.get()) + .number(state.channel_id.get()); + } + digest.finish() +} + +fn digest_pending_joins(joins: &[PendingJoin]) -> u64 { + let mut digest = Digest::new().text("preflight_pending_joins"); + for join in joins { + digest = digest + .text(join.connection.as_str()) + .number(join.user_id.get()) + .number(join.expires_at.get()); + } + digest.finish() +} + +pub fn open_preflight( + state: &mut RuntimeState, + room: RoomKey, + now: Millis, +) -> Result { + if !state + .governor + .try_acquire_many(BudgetClass::GatewayRead, PREFLIGHT_GATEWAY_READS, now) + { + return Err(AbortReason::Budget); + } + Ok(state.sequencer.open(TurnScope::preflight(room), now)) +} + +pub async fn read_preflight( + gateway: &G, + room: RoomKey, + clock: &dyn Clock, + token: &mut crate::turn::TurnToken, +) -> Result, PreflightError> +where + G: GatewayApi, +{ + let states = gateway + .voice_states_for_channel(room) + .await + .map_err(PreflightError::Gateway)?; + let fresh_states = token.read( + ReadSource::GatewayVoiceStates, + clock.now(), + digest_voice_states(&states), + states, + ); + + let joins = match gateway.pending_joins_for_channel(room).await { + Ok(joins) => joins, + Err(fault) => { + let _ = token.discard(fresh_states); + return Err(PreflightError::Gateway(fault)); + } + }; + let fresh_joins = token.read( + ReadSource::GatewayPendingJoins, + clock.now(), + digest_pending_joins(&joins), + joins, + ); + + fresh_states + .zip(fresh_joins) + .map(|both| { + both.map(|(voice_states, pending_joins)| PreflightReads { + voice_states, + pending_joins, + }) + }) + .map_err(PreflightError::Turn) +} + +pub fn seal_preflight( + sequencer: &mut TurnSequencer, + token: crate::turn::TurnToken, + at: Millis, + reads: Fresh, + media_present: Vec, +) -> Result<(TurnRecord, GatewayPreflight), TurnError> { + let turn = token.turn(); + let taken_at = token.opened_at(); + let preflight = reads.map(|reads| GatewayPreflight { + turn, + taken_at, + voice_states: reads + .voice_states + .iter() + .map(|state| PreflightVoiceState { + user_id: state.user_id, + connection: state.connection.clone(), + }) + .collect(), + pending_joins: reads.pending_joins, + media_present, + }); + sequencer + .seal(token, at, preflight) + .map(crate::turn::Observed::into_parts) +} + +pub fn preflight_is_current( + sequencer: &TurnSequencer, + room: RoomKey, + record: &TurnRecord, + now: Millis, + max_age_ms: u64, +) -> Result<(), AbortReason> { + if !sequencer.is_latest_sealed(&room, record.turn()) { + return Err(AbortReason::PreflightDisagreed); + } + if !record.is_fresh(now, max_age_ms) { + return Err(AbortReason::PreflightStale); + } + Ok(()) +} + +pub fn media_sibling_refusal( + facts: &TurnFacts, + target: &ConnectionId, + user_id: UserId, + posture: ConnectionIdGuard, +) -> Option { + if connection_id_is_honoured(posture, facts.room) { + return None; + } + if facts.media_siblings_of(user_id, target).is_empty() { + None + } else { + Some(AbortReason::SiblingConnection) + } +} + +fn media_divergence_facts( + ledger: &Ledger, + key: &ConnectionKey, +) -> Option<(Millis, Millis, Option)> { + let entry = ledger.connection(key)?; + match &entry.state { + ConnectionState::MediaOnly { + since, + participant_joined_at, + repair_verdict, + .. + } => Some((*since, *participant_joined_at, *repair_verdict)), + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::GatewayOnly { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::ActionTaken { .. } + | ConnectionState::Wedged { .. } + | ConnectionState::Retired { .. } => None, + } +} + +fn mutations_are_paused(fault: &GatewayFault) -> bool { + matches!( + fault, + GatewayFault::NotOk(GatewayError::Known(GatewayErrorCode::EventMutationsPaused)) + ) +} + +fn note_mutation_fault(state: &mut RuntimeState, fault: &GatewayFault, now: Millis) { + if mutations_are_paused(fault) + && let Some(until) = state.mutations_pause.note_refused(now) + { + state + .governor + .clamps_mut() + .engage_until(ModeClamp::MutationsPaused, until); + tracing::warn!( + hold_ms = state.mutations_pause.hold_ms(), + windows = state.mutations_pause.windows(), + "the gateway is refusing mutations, holding the destructive lane for one backoff \ + window that expires on its own" + ); + } +} + +fn note_mutation_success(state: &mut RuntimeState) { + state.mutations_pause.clear(); + state + .governor + .clamps_mut() + .set(ModeClamp::MutationsPaused, false); +} + +fn refuse(metrics: &ReconMetrics, lane: Lane, reason: AbortReason) -> Actuation { + if matches!(lane, Lane::Destructive) { + metrics.record_eviction_aborted(reason); + } + Actuation::Refused { lane, reason } +} + +fn counterfactual(metrics: &ReconMetrics, lane: Lane, reason: AbortReason) -> Actuation { + if matches!(lane, Lane::Destructive) { + metrics.record_eviction_aborted(reason); + } + Actuation::Counterfactual { lane, reason } +} + +fn hold_gateway_slot(shared: &Shared) -> bool { + shared.with_state_mut(|state| state.governor.gateway_inflight().try_acquire()) +} + +fn release_gateway_slot(shared: &Shared) { + shared.with_state_mut(|state| state.governor.gateway_inflight().release()); +} + +fn hold_media_slot(shared: &Shared, location: &Location) -> bool { + shared.with_state_mut(|state| { + if !state.governor.livekit_inflight().try_acquire() { + return false; + } + if !state.governor.server_inflight(location).try_acquire() { + state.governor.livekit_inflight().release(); + return false; + } + true + }) +} + +fn release_media_slot(shared: &Shared, location: &Location) { + shared.with_state_mut(|state| { + state.governor.server_inflight(location).release(); + state.governor.livekit_inflight().release(); + }); +} + +fn constructive_gate( + state: &mut RuntimeState, + connection: &ConnectionKey, + now: Millis, +) -> Result<(), (bool, AbortReason)> { + match state.governor.authorize_constructive(connection, now) { + Ok(()) => Ok(()), + Err(AbortReason::Mode) => Err((true, AbortReason::Mode)), + Err(reason) => Err((false, reason)), + } +} + +pub async fn apply_decision( + ports: &Ports<'_, G, L>, + facts: &TurnFacts, + decision: &Decision, +) -> Actuation +where + G: GatewayApi, + L: LiveKitApi, +{ + let metrics = ports.shared.metrics().clone(); + + if let Some(reason) = decision.blocked_by { + return Actuation::Refused { + lane: if decision.is_destructive() { + Lane::Destructive + } else { + Lane::Constructive + }, + reason, + }; + } + + match &decision.action { + DecisionAction::Hold => Actuation::Held, + DecisionAction::ConfirmConnection { nonce } => { + issue_confirm(ports, &metrics, decision, nonce).await + } + DecisionAction::RepairState => issue_repair(ports, &metrics, decision).await, + DecisionAction::RemoveGatewayState => { + remove_gateway_state(ports, &metrics, facts, decision).await + } + DecisionAction::RemoveParticipant { location } => { + remove_participant(ports, &metrics, facts, decision, location).await + } + } +} + +async fn issue_confirm( + ports: &Ports<'_, G, L>, + metrics: &ReconMetrics, + decision: &Decision, + nonce: &Nonce, +) -> Actuation +where + G: GatewayApi, + L: LiveKitApi, +{ + let now = ports.clock.now(); + let key = decision.connection.clone(); + let room = key.room; + + if let Err((counterfactual_gate, reason)) = ports + .shared + .with_state_mut(|state| constructive_gate(state, &key, now)) + { + return if counterfactual_gate { + counterfactual(metrics, Lane::Constructive, reason) + } else { + refuse(metrics, Lane::Constructive, reason) + }; + } + + if !hold_gateway_slot(ports.shared) { + return refuse(metrics, Lane::Constructive, AbortReason::Budget); + } + let started = ports.clock.now(); + let result = ports + .gateway + .confirm_connection(room, &key.connection, nonce) + .await; + release_gateway_slot(ports.shared); + metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); + + let settled = ports.clock.now(); + ports.shared.with_state_mut(|state| { + state.ledger.record_confirm_issued(&key, settled); + match &result { + Err(fault) => note_mutation_fault(state, fault, settled), + Ok(_) => note_mutation_success(state), + } + }); + + match result { + Err(fault) => { + metrics.record_confirm("failed"); + metrics.record_gateway_rpc("confirm_connection", fault.label()); + Actuation::Failed { + lane: Lane::Constructive, + call: "confirm_connection", + fault: fault.label(), + } + } + Ok(outcome) => { + metrics.record_confirm(outcome.label()); + metrics.record_gateway_rpc("confirm_connection", outcome.label()); + Actuation::Issued { + lane: Lane::Constructive, + call: "confirm_connection", + outcome: outcome.label(), + } + } + } +} + +async fn issue_repair( + ports: &Ports<'_, G, L>, + metrics: &ReconMetrics, + decision: &Decision, +) -> Actuation +where + G: GatewayApi, + L: LiveKitApi, +{ + let now = ports.clock.now(); + let key = decision.connection.clone(); + let room = key.room; + + if let Err((counterfactual_gate, reason)) = ports + .shared + .with_state_mut(|state| constructive_gate(state, &key, now)) + { + return if counterfactual_gate { + counterfactual(metrics, Lane::Constructive, reason) + } else { + refuse(metrics, Lane::Constructive, reason) + }; + } + + if !hold_gateway_slot(ports.shared) { + return refuse(metrics, Lane::Constructive, AbortReason::Budget); + } + let started = ports.clock.now(); + let result = ports + .gateway + .repair_state_from_cache(room, decision.user_id, &key.connection) + .await; + release_gateway_slot(ports.shared); + metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); + + let settled = ports.clock.now(); + let verdict = result.as_ref().ok().map(RepairOutcome::verdict); + ports.shared.with_state_mut(|state| { + state.ledger.record_repair_issued(&key, settled); + match &result { + Err(fault) => note_mutation_fault(state, fault, settled), + Ok(_) => note_mutation_success(state), + } + if let Some(verdict) = verdict { + state.ledger.record_repair_verdict(&key, verdict); + } + }); + + match result { + Err(fault) => { + metrics.record_repair("failed"); + metrics.record_gateway_rpc("repair_state_from_cache", fault.label()); + Actuation::Failed { + lane: Lane::Constructive, + call: "repair_state_from_cache", + fault: fault.label(), + } + } + Ok(outcome) => { + metrics.record_repair_verdict(outcome.verdict()); + metrics.record_gateway_rpc("repair_state_from_cache", outcome.label()); + Actuation::Issued { + lane: Lane::Constructive, + call: "repair_state_from_cache", + outcome: outcome.label(), + } + } + } +} + +struct AuthorizedRemoval { + authorization: MutationAuthorization, +} + +enum RemovalGate { + Authorized(AuthorizedRemoval), + Denied { + counterfactual: bool, + reason: AbortReason, + cause: Option, + }, +} + +pub async fn sweep_media( + livekit: &L, + room: RoomKey, + fleet: Vec, + required: Vec, + budget: impl FnMut(&Location) -> bool, +) -> (Vec, MediaCoverage) +where + L: LiveKitApi, +{ + sweep_rosters(livekit, room, fleet, required, &[], budget).await +} + +pub async fn sweep_rosters( + livekit: &L, + room: RoomKey, + fleet: Vec, + required: Vec, + absent_ok: &[Location], + mut budget: impl FnMut(&Location) -> bool, +) -> (Vec, MediaCoverage) +where + L: LiveKitApi, +{ + let mut coverage = MediaCoverage::over(fleet, required); + let mut sightings: Vec = Vec::new(); + + for location in coverage.fleet().to_vec() { + if !budget(&location) { + continue; + } + let records = match livekit.list_participants(&location, room).await { + ReadResult::Read(records) => records, + ReadResult::Unreadable(LiveKitFault::NotFound) if absent_ok.contains(&location) => { + coverage.note_absent(&location); + continue; + } + ReadResult::Unreadable(_) => { + coverage.note_unreachable(&location); + continue; + } + }; + let mut seen: Vec = Vec::with_capacity(records.len()); + let mut readable = true; + for record in &records { + match parse_participant_identity(&record.identity) { + Err(_) => readable = false, + Ok(identity) => seen.push(MediaSighting { + connection: identity.connection_id, + user_id: identity.user_id, + }), + } + } + if readable { + coverage.note_read(&location); + sightings.extend(seen); + } + } + + sightings.sort_by(|left, right| left.connection.cmp(&right.connection)); + sightings.dedup_by(|left, right| left.connection == right.connection); + (sightings, coverage) +} + +fn fleet_locations(state: &RuntimeState, now: Millis) -> Vec { + state.topology.lens(now).live_locations() +} + +fn believed_footprint(state: &RuntimeState, room: RoomKey) -> Vec { + let mut locations = state.directory.locations_for(&room); + let homes = state + .ledger + .room(&room) + .map(RoomLedger::believed_homes) + .unwrap_or_default(); + for index in homes { + if let Some(location) = state.ledger.location(index) { + locations.push(location.clone()); + } + } + locations.sort(); + locations.dedup(); + locations +} + +fn absent_ok_locations(state: &RuntimeState, room: RoomKey, fleet: &[Location]) -> Vec { + fleet + .iter() + .filter(|location| state.directory.room_is_absent_from(location, &room)) + .cloned() + .collect() +} + +fn written_off_locations(state: &RuntimeState, fleet: &[Location]) -> Vec { + fleet + .iter() + .filter(|location| state.server_health.health(location).is_written_off()) + .cloned() + .collect() +} + +fn unprobeable_locations(state: &RuntimeState, fleet: &[Location], now: Millis) -> Vec { + fleet + .iter() + .filter(|location| !state.server_health.may_probe(location, now)) + .cloned() + .collect() +} + +fn record_roster_stamps( + state: &mut RuntimeState, + room: RoomKey, + fleet: &[Location], + read: &[Location], + at: Millis, +) { + let live: Vec = fleet + .iter() + .filter_map(|location| state.ledger.intern_location(location)) + .collect(); + let indices: Vec = read + .iter() + .filter_map(|location| state.ledger.location_index(location)) + .collect(); + let Some(entry) = state.ledger.room_mut(&room) else { + return; + }; + entry.retain_roster_stamps(&live); + for index in indices { + entry.note_roster_read(index, at); + } +} + +async fn sweep_fleet_media( + ports: &Ports<'_, G, L>, + room: RoomKey, +) -> (Vec, MediaCoverage) +where + G: GatewayApi, + L: LiveKitApi, +{ + let now = ports.clock.now(); + let (fleet, required, absent_ok, unprobeable) = ports.shared.with_state(|state| { + let fleet = fleet_locations(state, now); + let required = believed_footprint(state, room); + let absent_ok = absent_ok_locations(state, room, &fleet); + let unprobeable = unprobeable_locations(state, &fleet, now); + (fleet, required, absent_ok, unprobeable) + }); + let (sightings, mut coverage) = sweep_rosters( + ports.livekit, + room, + fleet, + required, + &absent_ok, + |location| { + if unprobeable.contains(location) { + return false; + } + ports.shared.with_state_mut(|state| { + let at = ports.clock.now(); + if !state.governor.try_acquire_server_read(location, 1, at) { + return false; + } + if state.governor.try_acquire(BudgetClass::LiveKitRead, at) { + return true; + } + state.governor.refund_server_read(location, 1); + false + }) + }, + ) + .await; + for location in &unprobeable { + coverage.note_unreachable(location); + } + let settled = ports.clock.now(); + let swept_fleet = coverage.fleet().to_vec(); + let read = coverage.read().to_vec(); + ports.shared.with_state_mut(|state| { + record_roster_stamps(state, room, &swept_fleet, &read, settled); + }); + (sightings, coverage) +} + +async fn take_preflight( + ports: &Ports<'_, G, L>, + room: RoomKey, + media_present: Vec, +) -> Result<(TurnRecord, GatewayPreflight), AbortReason> +where + G: GatewayApi, + L: LiveKitApi, +{ + let opened_at = ports.clock.now(); + let mut token = ports + .shared + .with_state_mut(|state| open_preflight(state, room, opened_at))?; + + if !hold_gateway_slot(ports.shared) { + ports + .shared + .with_state_mut(|state| state.sequencer.abandon(token)); + return Err(AbortReason::Budget); + } + let reads = read_preflight(ports.gateway, room, ports.clock, &mut token).await; + release_gateway_slot(ports.shared); + let sealed_at = ports.clock.now(); + + ports.shared.with_state_mut(|state| match reads { + Err(error) => { + state.sequencer.abandon(token); + Err(error.abort_reason()) + } + Ok(reads) => seal_preflight(&mut state.sequencer, token, sealed_at, reads, media_present) + .map_err(|_| AbortReason::PreflightDisagreed), + }) +} + +fn render_locations(locations: &[Location]) -> String { + locations + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") +} + +struct CustodyView { + target_home: Option, + user_homes: Vec, + stamped: Vec, +} + +fn custody_view( + state: &RuntimeState, + key: &ConnectionKey, + user_id: UserId, + fleet: &[Location], +) -> CustodyView { + let entry = state.ledger.connection(key); + let target_home = entry + .and_then(|entry| entry.last_location) + .and_then(|index| state.ledger.location(index).cloned()); + let divergence_since = entry + .and_then(|entry| entry.state.divergence_since()) + .unwrap_or(Millis::new(0)); + let room_entry = state.ledger.room(&key.room); + let user_homes = room_entry + .map(|room_entry| { + room_entry + .connections() + .iter() + .filter(|connection| connection.user_id == user_id) + .filter_map(ConnectionLedger::believed_home) + .filter_map(|index| state.ledger.location(index).cloned()) + .collect::>() + }) + .unwrap_or_default(); + let stamped = fleet + .iter() + .filter(|location| { + state + .ledger + .location_index(location) + .and_then(|index| room_entry.and_then(|entry| entry.roster_read_at(index))) + .is_some_and(|at| at >= divergence_since) + }) + .cloned() + .collect(); + CustodyView { + target_home, + user_homes, + stamped, + } +} + +async fn remove_gateway_state( + ports: &Ports<'_, G, L>, + metrics: &ReconMetrics, + facts: &TurnFacts, + decision: &Decision, +) -> Actuation +where + G: GatewayApi, + L: LiveKitApi, +{ + let key = decision.connection.clone(); + let room = key.room; + let target = key.connection.clone(); + let user_id = decision.user_id; + + let (swept, coverage) = sweep_fleet_media(ports, room).await; + if let Some(reason) = media_coverage_refusal(&coverage, ports.policy.connection_id_guard, room) + { + if let Some(cause) = media_coverage_cause(&coverage, ports.policy.connection_id_guard, room) + { + metrics.record_unknown_media_cause(cause); + } + return refuse(metrics, Lane::Destructive, reason); + } + + let fleet = coverage.fleet().to_vec(); + let answered = coverage.answered(); + let read = coverage.read().to_vec(); + let silent = coverage.silent(); + + let (record, preflight) = match take_preflight(ports, room, swept).await { + Err(reason) => return refuse(metrics, Lane::Destructive, reason), + Ok(taken) => taken, + }; + + let now = ports.clock.now(); + let window = PreflightWindow { + now, + wall_now: ports.clock.wall_now(), + preflight_max_age_ms: ports.policy.preflight_max_age_ms, + pending_join_skew_ms: ports.policy.pending_join_skew_ms, + }; + + let gate = ports.shared.with_state_mut(|state| { + if let Err(reason) = preflight_is_current( + &state.sequencer, + room, + &record, + now, + ports.policy.preflight_max_age_ms, + ) { + return RemovalGate::Denied { + counterfactual: false, + reason, + cause: None, + }; + } + if let Err(reason) = gateway_removal_preflight(&GatewayRemovalGuard { + room, + target: &target, + user_id, + posture: ports.policy.connection_id_guard, + dm_posture: ports.policy.dm_posture, + preflight: &preflight, + window, + }) { + return RemovalGate::Denied { + counterfactual: false, + reason, + cause: None, + }; + } + if let Some(reason) = + media_sibling_refusal(facts, &target, user_id, ports.policy.connection_id_guard) + { + return RemovalGate::Denied { + counterfactual: false, + reason, + cause: None, + }; + } + let ledger_view = custody_view(state, &key, user_id, &fleet); + if let Some(cause) = media_custody_cause(&MediaCustody { + room, + posture: ports.policy.connection_id_guard, + fleet: &fleet, + answered: &answered, + read: &read, + target_home: ledger_view.target_home.as_ref(), + user_homes: &ledger_view.user_homes, + stamped: &ledger_view.stamped, + }) { + return RemovalGate::Denied { + counterfactual: false, + reason: AbortReason::UnknownMedia, + cause: Some(cause), + }; + } + let written_off = written_off_locations(state, &fleet); + if let Some(cause) = fleet_silence_cause(&FleetSilence { + room, + posture: ports.policy.connection_id_guard, + fleet: &fleet, + answered: &answered, + written_off: &written_off, + }) { + return RemovalGate::Denied { + counterfactual: false, + reason: AbortReason::UnknownMedia, + cause: Some(cause), + }; + } + authorize( + state, + &AuthorizationRequest { + connection: key.clone(), + user_id, + kind: ActionKind::RemoveGatewayState, + location: None, + authorizing_turn: facts.authorizing_turn, + preflight_turn: record.turn(), + preflight_at: record.opened_at(), + now, + }, + ) + }); + + let authorization = match gate { + RemovalGate::Denied { + counterfactual: is_counterfactual, + reason, + cause, + } => { + if let Some(cause) = cause { + metrics.record_unknown_media_cause(cause); + tracing::info!( + channel_id = room.channel_id().get(), + guild_id = room.guild_id().map(|guild| guild.get()), + connection = %target, + user_id = user_id.get(), + cause = cause.label(), + silent = %render_locations(&silent), + "a user scoped disconnect was held because the fleet did not answer in full" + ); + } + return if is_counterfactual { + counterfactual(metrics, Lane::Destructive, reason) + } else { + refuse(metrics, Lane::Destructive, reason) + }; + } + RemovalGate::Authorized(authorized) => authorized.authorization, + }; + + if !hold_gateway_slot(ports.shared) { + ports.shared.with_state_mut(|state| { + state.governor.refund_identity(&key); + }); + return refuse(metrics, Lane::Destructive, AbortReason::Budget); + } + metrics.record_mutation( + ActionKind::RemoveGatewayState, + room.scope(), + MutationOutcome::Issued, + ); + let started = ports.clock.now(); + let result = execute_gateway_removal(ports.gateway, authorization).await; + release_gateway_slot(ports.shared); + metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); + + let settled = ports.clock.now(); + ports.shared.with_state_mut(|state| { + state.ledger.record_action_issued( + &key, + ActionKind::RemoveGatewayState, + settled, + facts.authorizing_turn, + ); + match &result { + Err(fault) => { + note_mutation_fault(state, fault, settled); + if mutations_are_paused(fault) { + state.governor.refund_identity(&key); + state.ledger.refund_action_attempt(&key); + } + } + Ok(_) => note_mutation_success(state), + } + }); + + match result { + Err(fault) => { + metrics.record_mutation( + ActionKind::RemoveGatewayState, + room.scope(), + MutationOutcome::Failed, + ); + metrics.record_gateway_rpc("disconnect_user_if_in_channel", fault.label()); + Actuation::Failed { + lane: Lane::Destructive, + call: "disconnect_user_if_in_channel", + fault: fault.label(), + } + } + Ok(outcome) => { + metrics.record_mutation( + ActionKind::RemoveGatewayState, + room.scope(), + disconnect_outcome(&outcome), + ); + metrics.record_gateway_rpc("disconnect_user_if_in_channel", outcome.label()); + Actuation::Issued { + lane: Lane::Destructive, + call: "disconnect_user_if_in_channel", + outcome: outcome.label(), + } + } + } +} + +const fn disconnect_outcome(outcome: &DisconnectOutcome) -> MutationOutcome { + match outcome { + DisconnectOutcome::Removed => MutationOutcome::Succeeded, + DisconnectOutcome::Ignored { .. } | DisconnectOutcome::CallNotFound => { + MutationOutcome::NoChange + } + DisconnectOutcome::Failed(_) => MutationOutcome::Failed, + } +} + +pub async fn execute_gateway_removal( + gateway: &G, + authorization: MutationAuthorization, +) -> Result +where + G: GatewayApi, +{ + let room = authorization.connection.room; + let connection = authorization.connection.connection.clone(); + let user_id = authorization.user_id; + drop(authorization); + + gateway + .disconnect_user_if_in_channel(room, user_id, Some(&connection)) + .await +} + +async fn remove_participant( + ports: &Ports<'_, G, L>, + metrics: &ReconMetrics, + facts: &TurnFacts, + decision: &Decision, + location: &Location, +) -> Actuation +where + G: GatewayApi, + L: LiveKitApi, +{ + let key = decision.connection.clone(); + let room = key.room; + let target = key.connection.clone(); + let user_id = decision.user_id; + + let Some((since, joined_at, verdict)) = ports + .shared + .with_state(|state| media_divergence_facts(&state.ledger, &key)) + else { + return refuse(metrics, Lane::Destructive, AbortReason::PreflightDisagreed); + }; + + let (record, preflight) = match take_preflight(ports, room, facts.media.clone()).await { + Err(reason) => return refuse(metrics, Lane::Destructive, reason), + Ok(taken) => taken, + }; + + let now = ports.clock.now(); + let window = PreflightWindow { + now, + wall_now: ports.clock.wall_now(), + preflight_max_age_ms: ports.policy.preflight_max_age_ms, + pending_join_skew_ms: ports.policy.pending_join_skew_ms, + }; + + let gate = ports.shared.with_state_mut(|state| { + if let Err(reason) = preflight_is_current( + &state.sequencer, + room, + &record, + now, + ports.policy.preflight_max_age_ms, + ) { + return RemovalGate::Denied { + counterfactual: false, + reason, + cause: None, + }; + } + if let Err(reason) = media_removal_preflight(&MediaRemovalGuard { + room, + target: &target, + user_id, + dm_posture: ports.policy.dm_posture, + divergence_since: since, + participant_joined_at: joined_at, + repair_verdict: verdict, + server_health: state.server_health.health(location), + preflight: &preflight, + window, + }) { + return RemovalGate::Denied { + counterfactual: false, + reason, + cause: None, + }; + } + authorize( + state, + &AuthorizationRequest { + connection: key.clone(), + user_id, + kind: ActionKind::RemoveParticipant, + location: Some(location.clone()), + authorizing_turn: facts.authorizing_turn, + preflight_turn: record.turn(), + preflight_at: record.opened_at(), + now, + }, + ) + }); + + let authorization = match gate { + RemovalGate::Denied { + counterfactual: is_counterfactual, + reason, + cause, + } => { + if let Some(cause) = cause { + metrics.record_unknown_media_cause(cause); + } + return if is_counterfactual { + counterfactual(metrics, Lane::Destructive, reason) + } else { + refuse(metrics, Lane::Destructive, reason) + }; + } + RemovalGate::Authorized(authorized) => authorized.authorization, + }; + + if !hold_media_slot(ports.shared, location) { + ports.shared.with_state_mut(|state| { + state.governor.refund_identity(&key); + }); + return refuse(metrics, Lane::Destructive, AbortReason::Budget); + } + metrics.record_mutation( + ActionKind::RemoveParticipant, + room.scope(), + MutationOutcome::Issued, + ); + let started = ports.clock.now(); + let outcome = execute_media_removal(ports.livekit, authorization, location.clone()).await; + release_media_slot(ports.shared, location); + metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); + + let settled = ports.clock.now(); + ports.shared.with_state_mut(|state| { + state.ledger.record_action_issued( + &key, + ActionKind::RemoveParticipant, + settled, + facts.authorizing_turn, + ); + }); + + metrics.record_mutation( + ActionKind::RemoveParticipant, + room.scope(), + remove_outcome(outcome), + ); + metrics.record_livekit_call("remove_participant", outcome.label()); + + if outcome.is_success() { + Actuation::Issued { + lane: Lane::Destructive, + call: "remove_participant", + outcome: outcome.label(), + } + } else { + Actuation::Failed { + lane: Lane::Destructive, + call: "remove_participant", + fault: outcome.label(), + } + } +} + +const fn remove_outcome(outcome: RemoveOutcome) -> MutationOutcome { + if outcome.is_success() { + MutationOutcome::Succeeded + } else { + MutationOutcome::Failed + } +} + +pub async fn execute_media_removal( + livekit: &L, + authorization: MutationAuthorization, + location: Location, +) -> RemoveOutcome +where + L: LiveKitApi, +{ + let room = authorization.connection.room; + let identity = ParticipantIdentity { + user_id: authorization.user_id, + connection_id: authorization.connection.connection.clone(), + }; + drop(authorization); + + livekit.remove_participant(&location, room, &identity).await +} + +fn authorize(state: &mut RuntimeState, request: &AuthorizationRequest) -> RemovalGate { + match state.governor.authorize(request) { + AuthorizationOutcome::Denied(reason) => RemovalGate::Denied { + counterfactual: matches!(reason, AbortReason::Mode), + reason, + cause: None, + }, + AuthorizationOutcome::Granted(authorization) => { + RemovalGate::Authorized(AuthorizedRemoval { authorization }) + } + } +} diff --git a/fluxer_recon/src/budget.rs b/fluxer_recon/src/budget.rs new file mode 100644 index 000000000..dfe6d05a3 --- /dev/null +++ b/fluxer_recon/src/budget.rs @@ -0,0 +1,1046 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; + +use crate::guards::AbortReason; +use crate::health::{ModeClamp, ModeClamps, ReconMode, WarmupGate, effective_mode}; +use crate::ids::{ConnectionKey, GuildId, Location, Millis, RoomKey, TurnId, UserId}; +use crate::ledger::ActionKind; + +pub const SLIDING_WINDOW_MS: u64 = 60_000; +pub const BREAKER_HYSTERESIS_MS: u64 = 60_000; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TokenBucket { + capacity: f64, + tokens: f64, + refill_per_ms: f64, + last_refill: Millis, +} + +impl TokenBucket { + pub fn new(capacity: f64, refill_per_ms: f64, now: Millis) -> Self { + Self { + capacity, + tokens: capacity, + refill_per_ms, + last_refill: now, + } + } + + pub fn per_second(rate: u32, now: Millis) -> Self { + Self::new(f64::from(rate), f64::from(rate) / 1_000.0, now) + } + + fn refill(&mut self, now: Millis) { + let elapsed = now.saturating_since(self.last_refill); + if elapsed == 0 { + return; + } + self.last_refill = now; + self.tokens = (self.tokens + (elapsed as f64) * self.refill_per_ms).min(self.capacity); + } + + pub fn tokens(&mut self, now: Millis) -> f64 { + self.refill(now); + self.tokens + } + + pub fn has_capacity(&mut self, now: Millis) -> bool { + self.tokens(now) >= 1.0 + } + + pub fn try_acquire(&mut self, now: Millis) -> bool { + self.try_acquire_many(1, now) + } + + pub fn try_acquire_many(&mut self, count: u32, now: Millis) -> bool { + self.refill(now); + let wanted = f64::from(count); + if self.tokens < wanted { + return false; + } + self.tokens -= wanted; + true + } + + pub fn refund(&mut self, count: u32) { + self.tokens = (self.tokens + f64::from(count)).min(self.capacity); + } + + pub const fn capacity(&self) -> f64 { + self.capacity + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SlidingLimiter { + limit: u32, + window: SlidingMinute, +} + +impl SlidingLimiter { + pub fn new(limit: u32) -> Self { + Self { + limit, + window: SlidingMinute::default(), + } + } + + pub fn count(&self, now: Millis) -> u32 { + self.window.count(now) + } + + pub const fn limit(&self) -> u32 { + self.limit + } + + pub fn remaining(&self, now: Millis) -> u32 { + self.limit.saturating_sub(self.count(now)) + } + + pub fn has_capacity(&self, now: Millis) -> bool { + self.count(now) < self.limit + } + + pub fn try_acquire(&mut self, now: Millis) -> bool { + if !self.has_capacity(now) { + return false; + } + self.window.record(now); + true + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BudgetClass { + GatewayRead, + GatewayMutate, + LiveKitRead, + LiveKitMutate, +} + +impl BudgetClass { + pub const ALL: [Self; 4] = [ + Self::GatewayRead, + Self::GatewayMutate, + Self::LiveKitRead, + Self::LiveKitMutate, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::GatewayRead => "gateway_read", + Self::GatewayMutate => "gateway_mutate", + Self::LiveKitRead => "livekit_read", + Self::LiveKitMutate => "livekit_mutate", + } + } + + pub const fn for_action(action: ActionKind) -> Self { + match action { + ActionKind::RemoveGatewayState => Self::GatewayMutate, + ActionKind::RemoveParticipant => Self::LiveKitMutate, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InflightSemaphore { + limit: usize, + held: usize, +} + +impl InflightSemaphore { + pub const fn new(limit: usize) -> Self { + Self { limit, held: 0 } + } + + pub const fn try_acquire(&mut self) -> bool { + if self.held >= self.limit { + return false; + } + self.held += 1; + true + } + + pub const fn release(&mut self) { + self.held = self.held.saturating_sub(1); + } + + pub const fn held(&self) -> usize { + self.held + } + + pub const fn limit(&self) -> usize { + self.limit + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct SlidingMinute { + events: Vec, +} + +impl SlidingMinute { + fn count(&self, now: Millis) -> u32 { + let live = self + .events + .iter() + .filter(|at| now.saturating_since(**at) < SLIDING_WINDOW_MS) + .count(); + u32::try_from(live).unwrap_or(u32::MAX) + } + + fn record(&mut self, now: Millis) { + self.events + .retain(|at| now.saturating_since(*at) < SLIDING_WINDOW_MS); + self.events.push(now); + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MutationLane { + Destructive, + Constructive, +} + +impl MutationLane { + pub const fn label(self) -> &'static str { + match self { + Self::Destructive => "destructive", + Self::Constructive => "constructive", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct IdentityTally { + destructive: u32, + constructive: u32, + last_at: Millis, +} + +impl Default for IdentityTally { + fn default() -> Self { + Self { + destructive: 0, + constructive: 0, + last_at: Millis::ZERO, + } + } +} + +pub const IDENTITY_TALLY_RETENTION_MS: u64 = 86_400_000; +pub const MAX_TRACKED_IDENTITIES: usize = 65_536; +const IDENTITY_TALLY_BYTES: u64 = 96; +const SLIDING_WINDOW_ENTRY_BYTES: u64 = 64; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct LaneWindows { + destructive: SlidingMinute, + constructive: SlidingMinute, +} + +impl LaneWindows { + const fn window(&self, lane: MutationLane) -> &SlidingMinute { + match lane { + MutationLane::Destructive => &self.destructive, + MutationLane::Constructive => &self.constructive, + } + } + + const fn window_mut(&mut self, lane: MutationLane) -> &mut SlidingMinute { + match lane { + MutationLane::Destructive => &mut self.destructive, + MutationLane::Constructive => &mut self.constructive, + } + } + + fn count(&self, lane: MutationLane, now: Millis) -> u32 { + self.window(lane).count(now) + } + + fn record(&mut self, lane: MutationLane, now: Millis) { + self.window_mut(lane).record(now); + } + + fn is_idle(&self, now: Millis) -> bool { + self.destructive.count(now) == 0 && self.constructive.count(now) == 0 + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MutationLedger { + per_room: BTreeMap, + per_guild: BTreeMap, + global: LaneWindows, + per_identity: BTreeMap, + issued_total: u64, + pruned_identities_total: u64, +} + +impl MutationLedger { + pub fn new() -> Self { + Self::default() + } + + pub fn room_count(&self, room: RoomKey, now: Millis) -> u32 { + self.room_count_in(room, MutationLane::Destructive, now) + } + + pub fn room_count_in(&self, room: RoomKey, lane: MutationLane, now: Millis) -> u32 { + self.per_room + .get(&room) + .map_or(0, |windows| windows.count(lane, now)) + } + + pub fn guild_count(&self, guild: GuildId, now: Millis) -> u32 { + self.guild_count_in(guild, MutationLane::Destructive, now) + } + + pub fn guild_count_in(&self, guild: GuildId, lane: MutationLane, now: Millis) -> u32 { + self.per_guild + .get(&guild) + .map_or(0, |windows| windows.count(lane, now)) + } + + pub fn global_count(&self, now: Millis) -> u32 { + self.global_count_in(MutationLane::Destructive, now) + } + + pub fn global_count_in(&self, lane: MutationLane, now: Millis) -> u32 { + self.global.count(lane, now) + } + + pub fn identity_count(&self, key: &ConnectionKey) -> u32 { + self.per_identity + .get(key) + .map_or(0, |tally| tally.destructive) + } + + pub fn constructive_identity_count(&self, key: &ConnectionKey) -> u32 { + self.per_identity + .get(key) + .map_or(0, |tally| tally.constructive) + } + + pub fn tracked_identities(&self) -> usize { + self.per_identity.len() + } + + pub const fn pruned_identities_total(&self) -> u64 { + self.pruned_identities_total + } + + pub fn tracked_bytes(&self) -> u64 { + let identities = (self.per_identity.len() as u64).saturating_mul(IDENTITY_TALLY_BYTES); + let windows = (self.per_room.len() as u64) + .saturating_add(self.per_guild.len() as u64) + .saturating_add(1) + .saturating_mul(SLIDING_WINDOW_ENTRY_BYTES); + identities.saturating_add(windows) + } + + pub fn record(&mut self, key: &ConnectionKey, lane: MutationLane, now: Millis) { + self.per_room.entry(key.room).or_default().record(lane, now); + if let Some(guild) = key.room.guild_id() { + self.per_guild.entry(guild).or_default().record(lane, now); + } + self.global.record(lane, now); + let tally = self.per_identity.entry(key.clone()).or_default(); + match lane { + MutationLane::Destructive => { + tally.destructive = tally.destructive.saturating_add(1); + } + MutationLane::Constructive => { + tally.constructive = tally.constructive.saturating_add(1); + } + } + tally.last_at = now; + self.issued_total = self.issued_total.saturating_add(1); + } + + pub const fn issued_total(&self) -> u64 { + self.issued_total + } + + pub fn refund_identity(&mut self, key: &ConnectionKey) -> bool { + match self.per_identity.get_mut(key) { + None => false, + Some(tally) => { + tally.destructive = tally.destructive.saturating_sub(1); + if tally.destructive == 0 && tally.constructive == 0 { + self.per_identity.remove(key); + } + true + } + } + } + + pub fn forget_room(&mut self, room: &RoomKey) { + self.per_room.remove(room); + } + + pub fn prune(&mut self, now: Millis) -> usize { + self.per_room.retain(|_, windows| !windows.is_idle(now)); + self.per_guild.retain(|_, windows| !windows.is_idle(now)); + + let before = self.per_identity.len(); + self.per_identity + .retain(|_, tally| now.saturating_since(tally.last_at) < IDENTITY_TALLY_RETENTION_MS); + + while self.per_identity.len() > MAX_TRACKED_IDENTITIES { + let Some(oldest) = self + .per_identity + .iter() + .min_by_key(|(key, tally)| (tally.last_at, (*key).clone())) + .map(|(key, _)| key.clone()) + else { + break; + }; + self.per_identity.remove(&oldest); + } + + let pruned = before.saturating_sub(self.per_identity.len()); + self.pruned_identities_total = self.pruned_identities_total.saturating_add(pruned as u64); + pruned + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BreakerLimits { + pub max_divergent_fraction: f64, + pub auto_reset_ms: u64, + pub max_auto_resets: u32, +} + +impl Default for BreakerLimits { + fn default() -> Self { + Self { + max_divergent_fraction: 0.10, + auto_reset_ms: 900_000, + max_auto_resets: 2, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BreakerEvent { + Steady, + Tripped, + AutoReset, + HeldForManualReset, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CircuitBreaker { + tripped_since: Option, + below_half_since: Option, + auto_resets_used: u32, + trips_total: u64, +} + +impl CircuitBreaker { + pub const fn new() -> Self { + Self { + tripped_since: None, + below_half_since: None, + auto_resets_used: 0, + trips_total: 0, + } + } + + pub const fn is_tripped(&self) -> bool { + self.tripped_since.is_some() + } + + pub const fn trips_total(&self) -> u64 { + self.trips_total + } + + pub const fn auto_resets_used(&self) -> u32 { + self.auto_resets_used + } + + pub const fn manual_reset(&mut self) { + self.tripped_since = None; + self.below_half_since = None; + self.auto_resets_used = 0; + } + + pub const fn held_for_manual_reset(&self, limits: BreakerLimits) -> bool { + self.tripped_since.is_some() && self.auto_resets_used >= limits.max_auto_resets + } + + pub fn observe( + &mut self, + divergent_fraction: f64, + now: Millis, + limits: BreakerLimits, + ) -> BreakerEvent { + let over = divergent_fraction > limits.max_divergent_fraction; + let calm = divergent_fraction < limits.max_divergent_fraction / 2.0; + + if calm { + if self.below_half_since.is_none() { + self.below_half_since = Some(now); + } + } else { + self.below_half_since = None; + } + + let Some(tripped_since) = self.tripped_since else { + if over { + self.tripped_since = Some(now); + self.below_half_since = None; + self.trips_total = self.trips_total.saturating_add(1); + return BreakerEvent::Tripped; + } + return BreakerEvent::Steady; + }; + + if now.saturating_since(tripped_since) < limits.auto_reset_ms { + return BreakerEvent::Steady; + } + + let hysteresis_met = self + .below_half_since + .is_some_and(|since| now.saturating_since(since) >= BREAKER_HYSTERESIS_MS); + + if !hysteresis_met { + return BreakerEvent::Steady; + } + + if self.auto_resets_used >= limits.max_auto_resets { + return BreakerEvent::HeldForManualReset; + } + + self.auto_resets_used = self.auto_resets_used.saturating_add(1); + self.tripped_since = None; + self.below_half_since = None; + BreakerEvent::AutoReset + } +} + +pub struct EnforceToken(()); + +impl std::fmt::Debug for EnforceToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("EnforceToken") + } +} + +impl PartialEq for EnforceToken { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +impl Eq for EnforceToken {} + +#[derive(Debug, PartialEq, Eq)] +pub struct MutationAuthorization { + pub connection: ConnectionKey, + pub user_id: UserId, + pub kind: ActionKind, + pub location: Option, + pub authorizing_turn: TurnId, + pub preflight_turn: TurnId, + pub preflight_at: Millis, + token: EnforceToken, +} + +impl MutationAuthorization { + pub const fn token(&self) -> &EnforceToken { + &self.token + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthorizationRequest { + pub connection: ConnectionKey, + pub user_id: UserId, + pub kind: ActionKind, + pub location: Option, + pub authorizing_turn: TurnId, + pub preflight_turn: TurnId, + pub preflight_at: Millis, + pub now: Millis, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum AuthorizationOutcome { + Granted(MutationAuthorization), + Denied(AbortReason), +} + +impl AuthorizationOutcome { + pub fn granted(self) -> Option { + match self { + Self::Granted(authorization) => Some(authorization), + Self::Denied(_) => None, + } + } + + pub const fn denial(&self) -> Option { + match self { + Self::Granted(_) => None, + Self::Denied(reason) => Some(*reason), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct GovernorLimits { + pub gateway_read_rps: u32, + pub gateway_mutate_rpm: u32, + pub livekit_read_rps: u32, + pub livekit_read_rps_per_server: u32, + pub livekit_mutate_rpm: u32, + pub max_inflight_room_turns: usize, + pub max_inflight_gateway: usize, + pub max_inflight_livekit: usize, + pub max_inflight_per_server: usize, + pub max_mutations_per_room_per_min: u32, + pub max_mutations_per_guild_per_min: u32, + pub max_mutations_per_min: u32, + pub max_action_attempts: u32, + pub preflight_max_age_ms: u64, + pub breaker: BreakerLimits, +} + +impl Default for GovernorLimits { + fn default() -> Self { + Self { + gateway_read_rps: 40, + gateway_mutate_rpm: 30, + livekit_read_rps: 40, + livekit_read_rps_per_server: 40, + livekit_mutate_rpm: 30, + max_inflight_room_turns: 4, + max_inflight_gateway: 8, + max_inflight_livekit: 8, + max_inflight_per_server: 2, + max_mutations_per_room_per_min: 4, + max_mutations_per_guild_per_min: 8, + max_mutations_per_min: 30, + max_action_attempts: 3, + preflight_max_age_ms: 2_000, + breaker: BreakerLimits::default(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BudgetView { + pub warmup_complete: bool, + pub coverage_pass_complete: bool, + pub breaker_tripped: bool, + pub destructive_capacity: bool, + pub constructive_capacity: bool, +} + +impl BudgetView { + pub const fn unconstrained() -> Self { + Self { + warmup_complete: true, + coverage_pass_complete: true, + breaker_tripped: false, + destructive_capacity: true, + constructive_capacity: true, + } + } + + pub const fn destructive_block(&self) -> Option { + if !self.warmup_complete || !self.coverage_pass_complete { + return Some(AbortReason::Warmup); + } + if self.breaker_tripped { + return Some(AbortReason::Breaker); + } + if !self.destructive_capacity { + return Some(AbortReason::Budget); + } + None + } + + pub const fn constructive_block(&self) -> Option { + if self.constructive_capacity { + None + } else { + Some(AbortReason::Budget) + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Governor { + configured_mode: ReconMode, + runtime_override: Option, + clamps: ModeClamps, + warmup: WarmupGate, + limits: GovernorLimits, + buckets: BTreeMap, + mutate_windows: BTreeMap, + room_turns: InflightSemaphore, + gateway_inflight: InflightSemaphore, + livekit_inflight: InflightSemaphore, + per_server_inflight: BTreeMap, + per_server_read: BTreeMap, + mutations: MutationLedger, + breaker: CircuitBreaker, + denials: BTreeMap, +} + +impl Governor { + pub fn new( + configured_mode: ReconMode, + limits: GovernorLimits, + warmup: WarmupGate, + now: Millis, + ) -> Self { + let mut buckets = BTreeMap::new(); + buckets.insert( + BudgetClass::GatewayRead, + TokenBucket::per_second(limits.gateway_read_rps, now), + ); + buckets.insert( + BudgetClass::LiveKitRead, + TokenBucket::per_second(limits.livekit_read_rps, now), + ); + + let mut mutate_windows = BTreeMap::new(); + mutate_windows.insert( + BudgetClass::GatewayMutate, + SlidingLimiter::new(limits.gateway_mutate_rpm), + ); + mutate_windows.insert( + BudgetClass::LiveKitMutate, + SlidingLimiter::new(limits.livekit_mutate_rpm), + ); + + Self { + configured_mode, + runtime_override: None, + clamps: ModeClamps::new(), + warmup, + limits, + buckets, + mutate_windows, + room_turns: InflightSemaphore::new(limits.max_inflight_room_turns), + gateway_inflight: InflightSemaphore::new(limits.max_inflight_gateway), + livekit_inflight: InflightSemaphore::new(limits.max_inflight_livekit), + per_server_inflight: BTreeMap::new(), + per_server_read: BTreeMap::new(), + mutations: MutationLedger::new(), + breaker: CircuitBreaker::new(), + denials: BTreeMap::new(), + } + } + + pub const fn configured_mode(&self) -> ReconMode { + self.configured_mode + } + + pub const fn runtime_override(&self) -> Option { + self.runtime_override + } + + pub const fn set_runtime_override(&mut self, mode: Option) { + self.runtime_override = mode; + } + + pub const fn clamps(&self) -> &ModeClamps { + &self.clamps + } + + pub const fn clamps_mut(&mut self) -> &mut ModeClamps { + &mut self.clamps + } + + pub fn effective_mode(&self) -> ReconMode { + effective_mode(self.configured_mode, self.runtime_override, &self.clamps) + } + + pub const fn warmup(&self) -> &WarmupGate { + &self.warmup + } + + pub const fn warmup_mut(&mut self) -> &mut WarmupGate { + &mut self.warmup + } + + pub const fn breaker(&self) -> &CircuitBreaker { + &self.breaker + } + + pub fn reset_breaker(&mut self) -> bool { + let was_tripped = self.breaker.is_tripped(); + self.breaker.manual_reset(); + self.clamps + .set(ModeClamp::BreakerTripped, self.breaker.is_tripped()); + was_tripped + } + + pub const fn mutations(&self) -> &MutationLedger { + &self.mutations + } + + pub const fn mutations_mut(&mut self) -> &mut MutationLedger { + &mut self.mutations + } + + pub fn refund_identity(&mut self, key: &ConnectionKey) -> bool { + self.mutations.refund_identity(key) + } + + pub const fn limits(&self) -> GovernorLimits { + self.limits + } + + pub fn denials(&self) -> impl Iterator { + self.denials.iter() + } + + pub fn denial_count(&self, reason: AbortReason) -> u64 { + self.denials.get(&reason).copied().unwrap_or(0) + } + + pub const fn pending_actions(&self) -> usize { + 0 + } + + pub fn observe_divergence(&mut self, divergent_fraction: f64, now: Millis) -> BreakerEvent { + let event = self + .breaker + .observe(divergent_fraction, now, self.limits.breaker); + self.clamps + .set(ModeClamp::BreakerTripped, self.breaker.is_tripped()); + event + } + + pub fn try_acquire(&mut self, class: BudgetClass, now: Millis) -> bool { + if let Some(window) = self.mutate_windows.get_mut(&class) { + return window.try_acquire(now); + } + self.buckets + .get_mut(&class) + .is_some_and(|bucket| bucket.try_acquire(now)) + } + + pub fn try_acquire_many(&mut self, class: BudgetClass, count: u32, now: Millis) -> bool { + if self.mutate_windows.contains_key(&class) { + return (0..count).all(|_| self.try_acquire(class, now)); + } + self.buckets + .get_mut(&class) + .is_some_and(|bucket| bucket.try_acquire_many(count, now)) + } + + pub fn refund(&mut self, class: BudgetClass, count: u32) { + if let Some(bucket) = self.buckets.get_mut(&class) { + bucket.refund(count); + } + } + + pub fn has_capacity(&mut self, class: BudgetClass, now: Millis) -> bool { + if let Some(window) = self.mutate_windows.get(&class) { + return window.has_capacity(now); + } + self.buckets + .get_mut(&class) + .is_some_and(|bucket| bucket.has_capacity(now)) + } + + pub fn tokens(&mut self, class: BudgetClass, now: Millis) -> f64 { + if let Some(window) = self.mutate_windows.get(&class) { + return f64::from(window.remaining(now)); + } + self.buckets + .get_mut(&class) + .map_or(0.0, |bucket| bucket.tokens(now)) + } + + pub fn try_acquire_server_read( + &mut self, + location: &Location, + count: u32, + now: Millis, + ) -> bool { + let rate = self.limits.livekit_read_rps_per_server; + self.per_server_read + .entry(location.clone()) + .or_insert_with(|| TokenBucket::per_second(rate, now)) + .try_acquire_many(count, now) + } + + pub fn server_read_tokens(&mut self, location: &Location, now: Millis) -> f64 { + let rate = self.limits.livekit_read_rps_per_server; + self.per_server_read + .entry(location.clone()) + .or_insert_with(|| TokenBucket::per_second(rate, now)) + .tokens(now) + } + + pub fn refund_server_read(&mut self, location: &Location, count: u32) { + if let Some(bucket) = self.per_server_read.get_mut(location) { + bucket.refund(count); + } + } + + pub fn retain_servers(&mut self, keep: &[Location]) { + self.per_server_read + .retain(|location, bucket| keep.contains(location) || bucket.tokens < bucket.capacity); + self.per_server_inflight + .retain(|location, held| keep.contains(location) || held.held() > 0); + } + + pub fn prune_mutations(&mut self, now: Millis) -> usize { + self.mutations.prune(now) + } + + pub fn forget_room(&mut self, room: &RoomKey) { + self.mutations.forget_room(room); + } + + pub const fn room_turns(&mut self) -> &mut InflightSemaphore { + &mut self.room_turns + } + + pub const fn gateway_inflight(&mut self) -> &mut InflightSemaphore { + &mut self.gateway_inflight + } + + pub const fn livekit_inflight(&mut self) -> &mut InflightSemaphore { + &mut self.livekit_inflight + } + + pub fn server_inflight(&mut self, location: &Location) -> &mut InflightSemaphore { + self.per_server_inflight + .entry(location.clone()) + .or_insert_with(|| InflightSemaphore::new(self.limits.max_inflight_per_server)) + } + + pub fn view(&mut self, now: Millis) -> BudgetView { + let destructive_capacity = self.has_capacity(BudgetClass::GatewayMutate, now) + || self.has_capacity(BudgetClass::LiveKitMutate, now); + let constructive_capacity = self.has_capacity(BudgetClass::GatewayMutate, now); + + BudgetView { + warmup_complete: self.warmup.elapsed_complete(now), + coverage_pass_complete: self.warmup.coverage_pass_complete(), + breaker_tripped: self.breaker.is_tripped(), + destructive_capacity, + constructive_capacity, + } + } + + fn deny(&mut self, reason: AbortReason) -> AuthorizationOutcome { + *self.denials.entry(reason).or_insert(0) += 1; + AuthorizationOutcome::Denied(reason) + } + + pub fn authorize(&mut self, request: &AuthorizationRequest) -> AuthorizationOutcome { + let now = request.now; + + if !self.warmup.is_complete(now) { + return self.deny(AbortReason::Warmup); + } + + if self.breaker.is_tripped() { + return self.deny(AbortReason::Breaker); + } + + if !self.effective_mode().allows_destructive() { + return self.deny(AbortReason::Mode); + } + + if request.preflight_turn < request.authorizing_turn { + return self.deny(AbortReason::PreflightDisagreed); + } + + if now.saturating_since(request.preflight_at) > self.limits.preflight_max_age_ms { + return self.deny(AbortReason::PreflightStale); + } + + if self.mutations.identity_count(&request.connection) >= self.limits.max_action_attempts { + return self.deny(AbortReason::Budget); + } + + if self.mutations.room_count(request.connection.room, now) + >= self.limits.max_mutations_per_room_per_min + { + return self.deny(AbortReason::Budget); + } + + if let Some(guild) = request.connection.room.guild_id() + && self.mutations.guild_count(guild, now) >= self.limits.max_mutations_per_guild_per_min + { + return self.deny(AbortReason::Budget); + } + + if self.mutations.global_count(now) >= self.limits.max_mutations_per_min { + return self.deny(AbortReason::Budget); + } + + if !self.try_acquire(BudgetClass::for_action(request.kind), now) { + return self.deny(AbortReason::Budget); + } + + self.mutations + .record(&request.connection, MutationLane::Destructive, now); + + AuthorizationOutcome::Granted(MutationAuthorization { + connection: request.connection.clone(), + user_id: request.user_id, + kind: request.kind, + location: request.location.clone(), + authorizing_turn: request.authorizing_turn, + preflight_turn: request.preflight_turn, + preflight_at: request.preflight_at, + token: EnforceToken(()), + }) + } + + pub fn authorize_constructive( + &mut self, + connection: &ConnectionKey, + now: Millis, + ) -> Result<(), AbortReason> { + if !self.effective_mode().allows_constructive() { + self.deny(AbortReason::Mode); + return Err(AbortReason::Mode); + } + + let lane = MutationLane::Constructive; + + if self.mutations.room_count_in(connection.room, lane, now) + >= self.limits.max_mutations_per_room_per_min + { + self.deny(AbortReason::Budget); + return Err(AbortReason::Budget); + } + + if let Some(guild) = connection.room.guild_id() + && self.mutations.guild_count_in(guild, lane, now) + >= self.limits.max_mutations_per_guild_per_min + { + self.deny(AbortReason::Budget); + return Err(AbortReason::Budget); + } + + if self.mutations.global_count_in(lane, now) >= self.limits.max_mutations_per_min { + self.deny(AbortReason::Budget); + return Err(AbortReason::Budget); + } + + if !self.try_acquire(BudgetClass::GatewayMutate, now) { + self.deny(AbortReason::Budget); + return Err(AbortReason::Budget); + } + + self.mutations.record(connection, lane, now); + Ok(()) + } +} diff --git a/fluxer_recon/src/census.rs b/fluxer_recon/src/census.rs new file mode 100644 index 000000000..453630851 --- /dev/null +++ b/fluxer_recon/src/census.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; + +use crate::config::ReconConfig; +use crate::evidence::CensusCrossCheck; +use crate::gateway::{ActiveVoiceRooms, GatewayApi, GatewayFault, GatewayMethod}; +use crate::ids::{Epoch, Millis, RoomKey}; +use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; + +pub const CENSUS_METHOD: GatewayMethod = GatewayMethod::ActiveVoiceRooms; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CensusRead { + Counted(ActiveVoiceRooms), + Unreadable(GatewayFault), +} + +impl CensusRead { + pub const fn is_readable(&self) -> bool { + matches!(self, Self::Counted(_)) + } + + pub const fn counted(&self) -> Option<&ActiveVoiceRooms> { + match self { + Self::Counted(rooms) => Some(rooms), + Self::Unreadable(_) => None, + } + } + + pub const fn fault(&self) -> Option<&GatewayFault> { + match self { + Self::Counted(_) => None, + Self::Unreadable(fault) => Some(fault), + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Counted(_) => "counted", + Self::Unreadable(_) => "unreadable", + } + } +} + +pub async fn read_census(gateway: &G) -> CensusRead +where + G: GatewayApi, +{ + match gateway.active_voice_rooms().await { + Ok(rooms) => CensusRead::Counted(rooms), + Err(fault) => CensusRead::Unreadable(fault), + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CensusSnapshot { + counts: BTreeMap, + node_count: u32, + unparseable_rooms: u32, + taken_at: Millis, + epoch: Epoch, +} + +impl CensusSnapshot { + pub fn rooms(&self) -> impl Iterator + '_ { + self.counts.iter().map(|(room, count)| (*room, *count)) + } + + pub fn room_keys(&self) -> Vec { + self.counts.keys().copied().collect() + } + + pub fn voice_state_count(&self, room: &RoomKey) -> Option { + self.counts.get(room).copied() + } + + pub const fn node_count(&self) -> u32 { + self.node_count + } + + pub const fn unparseable_rooms(&self) -> u32 { + self.unparseable_rooms + } + + pub const fn taken_at(&self) -> Millis { + self.taken_at + } + + pub const fn epoch(&self) -> Epoch { + self.epoch + } + + pub fn len(&self) -> usize { + self.counts.len() + } + + pub fn is_empty(&self) -> bool { + self.counts.is_empty() + } + + pub const fn age_ms(&self, now: Millis) -> u64 { + now.saturating_since(self.taken_at) + } + + pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { + self.age_ms(now) <= max_age_ms + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CensusLimits { + pub max_age_ms: u64, + pub interval_ms: u64, +} + +impl CensusLimits { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + max_age_ms: config.census_max_age_ms, + interval_ms: config.census_interval_ms, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CensusOutcome { + Applied { epoch: Epoch, rooms: usize }, + Unchanged { epoch: Epoch }, + Failed, +} + +impl CensusOutcome { + pub const fn label(self) -> &'static str { + match self { + Self::Applied { .. } => "applied", + Self::Unchanged { .. } => "unchanged", + Self::Failed => "failed", + } + } + + pub const fn kept_previous_snapshot(self) -> bool { + match self { + Self::Applied { .. } | Self::Unchanged { .. } => false, + Self::Failed => true, + } + } + + pub const fn epoch(self) -> Option { + match self { + Self::Applied { epoch, .. } | Self::Unchanged { epoch } => Some(epoch), + Self::Failed => None, + } + } +} + +#[derive(Clone, Debug)] +pub struct CensusCache { + limits: CensusLimits, + snapshot: Option, + epoch: Epoch, + last_failure_at: Option, + consecutive_failures: u32, + applied_total: u64, + unchanged_total: u64, + failed_total: u64, +} + +pub const CENSUS_ENTRY_BYTES: u64 = + (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; + +impl CensusCache { + pub fn tracked_bytes(&self) -> u64 { + self.snapshot.as_ref().map_or(0, |snapshot| { + (snapshot.len() as u64).saturating_mul(CENSUS_ENTRY_BYTES) + }) + } + + pub const fn new(limits: CensusLimits) -> Self { + Self { + limits, + snapshot: None, + epoch: Epoch::FIRST, + last_failure_at: None, + consecutive_failures: 0, + applied_total: 0, + unchanged_total: 0, + failed_total: 0, + } + } + + pub const fn limits(&self) -> CensusLimits { + self.limits + } + + pub const fn snapshot(&self) -> Option<&CensusSnapshot> { + self.snapshot.as_ref() + } + + pub const fn epoch(&self) -> Epoch { + self.epoch + } + + pub const fn last_failure_at(&self) -> Option { + self.last_failure_at + } + + pub const fn consecutive_failures(&self) -> u32 { + self.consecutive_failures + } + + pub const fn applied_total(&self) -> u64 { + self.applied_total + } + + pub const fn unchanged_total(&self) -> u64 { + self.unchanged_total + } + + pub const fn failed_total(&self) -> u64 { + self.failed_total + } + + pub fn age_ms(&self, now: Millis) -> Option { + self.snapshot.as_ref().map(|snapshot| snapshot.age_ms(now)) + } + + pub fn is_fresh(&self, now: Millis) -> bool { + self.snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.is_fresh(now, self.limits.max_age_ms)) + } + + pub fn rooms(&self) -> Vec { + self.snapshot + .as_ref() + .map(CensusSnapshot::room_keys) + .unwrap_or_default() + } + + pub fn node_count(&self) -> Option { + self.snapshot.as_ref().map(CensusSnapshot::node_count) + } + + pub fn accept(&mut self, read: &CensusRead, now: Millis) -> CensusOutcome { + let counted = match read { + CensusRead::Unreadable(_) => { + self.failed_total = self.failed_total.saturating_add(1); + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.last_failure_at = Some(now); + return CensusOutcome::Failed; + } + CensusRead::Counted(counted) => counted, + }; + + self.consecutive_failures = 0; + let counts: BTreeMap = counted + .rooms + .iter() + .map(|entry| (entry.room, entry.voice_state_count)) + .collect(); + + let unchanged = self + .snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.counts == counts); + if unchanged { + self.unchanged_total = self.unchanged_total.saturating_add(1); + if let Some(snapshot) = self.snapshot.as_mut() { + snapshot.taken_at = now; + snapshot.node_count = counted.node_count; + snapshot.unparseable_rooms = counted.unparseable_rooms; + } + return CensusOutcome::Unchanged { epoch: self.epoch }; + } + + self.epoch = self.epoch.next(); + self.applied_total = self.applied_total.saturating_add(1); + let rooms = counts.len(); + self.snapshot = Some(CensusSnapshot { + counts, + node_count: counted.node_count, + unparseable_rooms: counted.unparseable_rooms, + taken_at: now, + epoch: self.epoch, + }); + CensusOutcome::Applied { + epoch: self.epoch, + rooms, + } + } + + pub fn cross_check(&self, room: &RoomKey, now: Millis) -> CensusCrossCheck { + let Some(snapshot) = self.snapshot.as_ref() else { + return CensusCrossCheck::Missing; + }; + if !snapshot.is_fresh(now, self.limits.max_age_ms) { + return CensusCrossCheck::Stale; + } + CensusCrossCheck::Fresh { + voice_state_count: snapshot.voice_state_count(room).unwrap_or(0), + } + } + + pub fn forget(&mut self) { + self.snapshot = None; + } +} diff --git a/fluxer_recon/src/clock.rs b/fluxer_recon/src/clock.rs new file mode 100644 index 000000000..16fc9e1e4 --- /dev/null +++ b/fluxer_recon/src/clock.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::ids::{Millis, WallMillis}; + +pub trait Clock: Send + Sync { + fn now(&self) -> Millis; + + fn wall_now(&self) -> WallMillis; + + fn elapsed_since(&self, earlier: Millis) -> u64 { + self.now().saturating_since(earlier) + } +} + +pub type SharedClock = Arc; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WallAnchor { + monotonic: Millis, + wall: WallMillis, +} + +impl WallAnchor { + pub const fn new(monotonic: Millis, wall: WallMillis) -> Self { + Self { monotonic, wall } + } + + pub const fn projected(&self, monotonic: Millis) -> WallMillis { + self.wall + .saturating_add_millis(monotonic.saturating_since(self.monotonic)) + } + + pub const fn guard(&self, monotonic: Millis, observed: WallMillis) -> WallMillis { + let projected = self.projected(monotonic); + if observed.get() < projected.get() { + observed + } else { + projected + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SystemClock; + +impl SystemClock { + pub const fn new() -> Self { + Self + } + + pub fn shared() -> SharedClock { + Arc::new(Self) + } +} + +impl Clock for SystemClock { + fn now(&self) -> Millis { + Millis::new(u64::try_from(fluxer_svc::metrics::now_ms()).unwrap_or(0)) + } + + fn wall_now(&self) -> WallMillis { + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + WallMillis::new(u64::try_from(since_epoch.as_millis()).unwrap_or(u64::MAX)) + } +} + +#[derive(Debug, Default)] +struct TestClockState { + monotonic: AtomicU64, + wall: AtomicU64, +} + +#[derive(Clone, Debug, Default)] +pub struct TestClock { + state: Arc, +} + +impl TestClock { + pub fn new() -> Self { + Self::default() + } + + pub fn starting_at(monotonic: Millis, wall: WallMillis) -> Self { + Self { + state: Arc::new(TestClockState { + monotonic: AtomicU64::new(monotonic.get()), + wall: AtomicU64::new(wall.get()), + }), + } + } + + pub fn shared(&self) -> SharedClock { + Arc::new(self.clone()) + } + + pub fn advance(&self, delta_ms: u64) -> Millis { + let previous = self.state.monotonic.fetch_add(delta_ms, Ordering::SeqCst); + Millis::new(previous.saturating_add(delta_ms)) + } + + pub fn advance_wall(&self, delta_ms: u64) -> WallMillis { + let previous = self.state.wall.fetch_add(delta_ms, Ordering::SeqCst); + WallMillis::new(previous.saturating_add(delta_ms)) + } + + pub fn set_wall(&self, wall: WallMillis) { + self.state.wall.store(wall.get(), Ordering::SeqCst); + } +} + +impl Clock for TestClock { + fn now(&self) -> Millis { + Millis::new(self.state.monotonic.load(Ordering::SeqCst)) + } + + fn wall_now(&self) -> WallMillis { + WallMillis::new(self.state.wall.load(Ordering::SeqCst)) + } +} diff --git a/fluxer_recon/src/config.rs b/fluxer_recon/src/config.rs new file mode 100644 index 000000000..1e56ce378 --- /dev/null +++ b/fluxer_recon/src/config.rs @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::env; +use std::fmt::Display; +use std::str::FromStr; + +use crate::health::ReconMode; +use crate::topology::default_region_id; + +const MIN_COVERAGE_TARGET_MS: u64 = 1_000; +const MIN_REQUIRED_CORROBORATIONS: u32 = 2; +const READ_RPS_RANGE: (u32, u32) = (1, 200); +const MUTATE_RPM_RANGE: (u32, u32) = (0, 600); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConnectionIdGuard { + AssumeAbsent, + Honoured, +} + +impl ConnectionIdGuard { + pub fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "assume_absent" => Some(Self::AssumeAbsent), + "honored" | "honoured" => Some(Self::Honoured), + _ => None, + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::AssumeAbsent => "assume_absent", + Self::Honoured => "honored", + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ReconConfig { + pub mode: ReconMode, + pub worker_threads: usize, + pub tick_ms: u64, + pub turns_per_tick: usize, + pub coverage_target_ms: u64, + pub coverage_period_hard_cap_ms: u64, + pub expected_rooms: usize, + pub hot_period_ms: u64, + pub max_hot_rooms: usize, + pub suspicion_hold_ms: u64, + pub required_corroborations: u32, + pub min_corroboration_gap_ms: u64, + pub max_corroboration_window_ms: u64, + pub min_divergence_ms: u64, + pub preflight_max_age_ms: u64, + pub warmup_seconds: u64, + pub pending_join_skew_ms: u64, + pub census_interval_ms: u64, + pub census_max_age_ms: u64, + pub discovery_interval_ms: u64, + pub topology_refresh_ms: u64, + pub topology_max_age_ms: u64, + pub topology_drain_grace_ms: u64, + pub room_cliff_hold_ms: u64, + pub server_cliff_hold_ms: u64, + pub server_cliff_fraction: f64, + pub gateway_read_rps: u32, + pub gateway_mutate_rpm: u32, + pub livekit_read_rps: u32, + pub livekit_read_rps_per_server: u32, + pub livekit_mutate_rpm: u32, + pub max_inflight_room_turns: usize, + pub max_inflight_gateway: usize, + pub max_inflight_livekit: usize, + pub max_inflight_per_server: usize, + pub max_mutations_per_room_per_min: u32, + pub max_mutations_per_guild_per_min: u32, + pub max_mutations_per_min: u32, + pub max_action_attempts: u32, + pub max_divergent_fraction: f64, + pub breaker_auto_reset_ms: u64, + pub max_breaker_auto_resets: u32, + pub max_tracked_rooms: usize, + pub max_tracked_connections: usize, + pub max_connections_per_room: usize, + pub memory_budget_bytes: u64, + pub dm_media_eviction: bool, + pub dm_gateway_eviction: bool, + pub gateway_connection_id_guard: ConnectionIdGuard, + pub webhook_enabled: bool, + pub webhook_port: u16, + pub livekit_internal_url: Option, + pub livekit_default_region_id: Option, +} + +impl ReconConfig { + pub fn from_env() -> anyhow::Result { + Self::from_env_reader(|name| env::var(name).ok()) + } + + pub fn from_env_reader(get: F) -> anyhow::Result + where + F: Fn(&str) -> Option, + { + let shard_count = number(&get, "FLUXER_SVC_SHARD_COUNT", 1u32)?; + let shard_id = number(&get, "FLUXER_SVC_SHARD_ID", 0u32)?; + if shard_count != 1 || shard_id != 0 { + anyhow::bail!( + "fluxer-recon runs as a single replica: FLUXER_SVC_SHARD_COUNT must be 1 and FLUXER_SVC_SHARD_ID must be 0, got {shard_count} and {shard_id}" + ); + } + + let config = Self { + mode: mode(&get, "FLUXER_RECON_MODE", ReconMode::Observing)?, + worker_threads: number(&get, "FLUXER_RECON_WORKER_THREADS", 2)?, + tick_ms: number(&get, "FLUXER_RECON_TICK_MS", 50)?, + turns_per_tick: number(&get, "FLUXER_RECON_TURNS_PER_TICK", 4)?, + coverage_target_ms: number(&get, "FLUXER_RECON_COVERAGE_TARGET_MS", 30_000)?, + coverage_period_hard_cap_ms: number( + &get, + "FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS", + 120_000, + )?, + expected_rooms: number(&get, "FLUXER_RECON_EXPECTED_ROOMS", 800)?, + hot_period_ms: number(&get, "FLUXER_RECON_HOT_PERIOD_MS", 1_000)?, + max_hot_rooms: number(&get, "FLUXER_RECON_MAX_HOT_ROOMS", 64)?, + suspicion_hold_ms: number(&get, "FLUXER_RECON_SUSPICION_HOLD_MS", 30_000)?, + required_corroborations: number(&get, "FLUXER_RECON_REQUIRED_CORROBORATIONS", 3)?, + min_corroboration_gap_ms: number(&get, "FLUXER_RECON_MIN_CORROBORATION_GAP_MS", 1_000)?, + max_corroboration_window_ms: number( + &get, + "FLUXER_RECON_MAX_CORROBORATION_WINDOW_MS", + 60_000, + )?, + min_divergence_ms: number(&get, "FLUXER_RECON_MIN_DIVERGENCE_MS", 15_000)?, + preflight_max_age_ms: number(&get, "FLUXER_RECON_PREFLIGHT_MAX_AGE_MS", 1_000)?, + warmup_seconds: number(&get, "FLUXER_RECON_WARMUP_SECONDS", 120)?, + pending_join_skew_ms: number(&get, "FLUXER_RECON_PENDING_JOIN_SKEW_MS", 5_000)?, + census_interval_ms: number(&get, "FLUXER_RECON_CENSUS_INTERVAL_MS", 60_000)?, + census_max_age_ms: number(&get, "FLUXER_RECON_CENSUS_MAX_AGE_MS", 300_000)?, + discovery_interval_ms: number(&get, "FLUXER_RECON_DISCOVERY_INTERVAL_MS", 15_000)?, + topology_refresh_ms: number(&get, "FLUXER_RECON_TOPOLOGY_REFRESH_MS", 60_000)?, + topology_max_age_ms: number(&get, "FLUXER_RECON_TOPOLOGY_MAX_AGE_MS", 600_000)?, + topology_drain_grace_ms: number(&get, "FLUXER_RECON_TOPOLOGY_DRAIN_GRACE_MS", 600_000)?, + room_cliff_hold_ms: number(&get, "FLUXER_RECON_ROOM_CLIFF_HOLD_MS", 60_000)?, + server_cliff_hold_ms: number(&get, "FLUXER_RECON_SERVER_CLIFF_HOLD_MS", 300_000)?, + server_cliff_fraction: number(&get, "FLUXER_RECON_SERVER_CLIFF_FRACTION", 0.5)?, + gateway_read_rps: clamped( + number(&get, "FLUXER_RECON_GATEWAY_READ_RPS", 40)?, + READ_RPS_RANGE, + ), + gateway_mutate_rpm: clamped( + number(&get, "FLUXER_RECON_GATEWAY_MUTATE_RPM", 30)?, + MUTATE_RPM_RANGE, + ), + livekit_read_rps: clamped( + number(&get, "FLUXER_RECON_LIVEKIT_READ_RPS", 40)?, + READ_RPS_RANGE, + ), + livekit_read_rps_per_server: clamped( + number(&get, "FLUXER_RECON_LIVEKIT_READ_RPS_PER_SERVER", 40)?, + READ_RPS_RANGE, + ), + livekit_mutate_rpm: clamped( + number(&get, "FLUXER_RECON_LIVEKIT_MUTATE_RPM", 30)?, + MUTATE_RPM_RANGE, + ), + max_inflight_room_turns: number(&get, "FLUXER_RECON_MAX_INFLIGHT_ROOM_TURNS", 4)?, + max_inflight_gateway: number(&get, "FLUXER_RECON_MAX_INFLIGHT_GATEWAY", 8)?, + max_inflight_livekit: number(&get, "FLUXER_RECON_MAX_INFLIGHT_LIVEKIT", 8)?, + max_inflight_per_server: number(&get, "FLUXER_RECON_MAX_INFLIGHT_PER_SERVER", 2)?, + max_mutations_per_room_per_min: number( + &get, + "FLUXER_RECON_MAX_MUTATIONS_PER_ROOM_PER_MIN", + 4, + )?, + max_mutations_per_guild_per_min: number( + &get, + "FLUXER_RECON_MAX_MUTATIONS_PER_GUILD_PER_MIN", + 8, + )?, + max_mutations_per_min: number(&get, "FLUXER_RECON_MAX_MUTATIONS_PER_MIN", 30)?, + max_action_attempts: number(&get, "FLUXER_RECON_MAX_ACTION_ATTEMPTS", 3)?, + max_divergent_fraction: number(&get, "FLUXER_RECON_MAX_DIVERGENT_FRACTION", 0.10)?, + breaker_auto_reset_ms: number(&get, "FLUXER_RECON_BREAKER_AUTO_RESET_MS", 900_000)?, + max_breaker_auto_resets: number(&get, "FLUXER_RECON_MAX_BREAKER_AUTO_RESETS", 2)?, + max_tracked_rooms: number(&get, "FLUXER_RECON_MAX_TRACKED_ROOMS", 8_192)?, + max_tracked_connections: number(&get, "FLUXER_RECON_MAX_TRACKED_CONNECTIONS", 65_536)?, + max_connections_per_room: number(&get, "FLUXER_RECON_MAX_CONNECTIONS_PER_ROOM", 512)?, + memory_budget_bytes: number(&get, "FLUXER_RECON_MEMORY_BUDGET_BYTES", 16_777_216)?, + dm_media_eviction: boolean(&get, "FLUXER_RECON_DM_MEDIA_EVICTION", false)?, + dm_gateway_eviction: boolean(&get, "FLUXER_RECON_DM_GATEWAY_EVICTION", true)?, + gateway_connection_id_guard: connection_id_guard( + &get, + "FLUXER_RECON_GATEWAY_CONNECTION_ID_GUARD", + ConnectionIdGuard::AssumeAbsent, + )?, + webhook_enabled: boolean(&get, "FLUXER_RECON_WEBHOOK_ENABLED", false)?, + webhook_port: number(&get, "FLUXER_RECON_WEBHOOK_PORT", 8_092)?, + livekit_internal_url: text(&get, "FLUXER_LIVEKIT_INTERNAL_URL"), + livekit_default_region_id: text(&get, "FLUXER_LIVEKIT_DEFAULT_REGION") + .as_deref() + .and_then(default_region_id), + }; + + validate(&config)?; + Ok(config) + } +} + +fn validate(config: &ReconConfig) -> anyhow::Result<()> { + positive_usize(config.worker_threads, "FLUXER_RECON_WORKER_THREADS")?; + positive_usize(config.turns_per_tick, "FLUXER_RECON_TURNS_PER_TICK")?; + positive_usize( + config.max_inflight_room_turns, + "FLUXER_RECON_MAX_INFLIGHT_ROOM_TURNS", + )?; + positive_usize( + config.max_inflight_gateway, + "FLUXER_RECON_MAX_INFLIGHT_GATEWAY", + )?; + positive_usize( + config.max_inflight_livekit, + "FLUXER_RECON_MAX_INFLIGHT_LIVEKIT", + )?; + positive_usize( + config.max_inflight_per_server, + "FLUXER_RECON_MAX_INFLIGHT_PER_SERVER", + )?; + positive_usize(config.expected_rooms, "FLUXER_RECON_EXPECTED_ROOMS")?; + positive_usize(config.max_tracked_rooms, "FLUXER_RECON_MAX_TRACKED_ROOMS")?; + positive_usize( + config.max_tracked_connections, + "FLUXER_RECON_MAX_TRACKED_CONNECTIONS", + )?; + positive_usize( + config.max_connections_per_room, + "FLUXER_RECON_MAX_CONNECTIONS_PER_ROOM", + )?; + positive_millis(config.tick_ms, "FLUXER_RECON_TICK_MS")?; + positive_millis(config.hot_period_ms, "FLUXER_RECON_HOT_PERIOD_MS")?; + positive_millis(config.census_interval_ms, "FLUXER_RECON_CENSUS_INTERVAL_MS")?; + positive_millis( + config.discovery_interval_ms, + "FLUXER_RECON_DISCOVERY_INTERVAL_MS", + )?; + positive_millis( + config.topology_refresh_ms, + "FLUXER_RECON_TOPOLOGY_REFRESH_MS", + )?; + positive_millis( + config.preflight_max_age_ms, + "FLUXER_RECON_PREFLIGHT_MAX_AGE_MS", + )?; + positive_millis( + config.min_corroboration_gap_ms, + "FLUXER_RECON_MIN_CORROBORATION_GAP_MS", + )?; + fraction( + config.server_cliff_fraction, + "FLUXER_RECON_SERVER_CLIFF_FRACTION", + )?; + fraction( + config.max_divergent_fraction, + "FLUXER_RECON_MAX_DIVERGENT_FRACTION", + )?; + + if config.required_corroborations < MIN_REQUIRED_CORROBORATIONS { + anyhow::bail!( + "FLUXER_RECON_REQUIRED_CORROBORATIONS must be at least {MIN_REQUIRED_CORROBORATIONS}, got {}", + config.required_corroborations + ); + } + + let spread = config + .min_corroboration_gap_ms + .saturating_mul(u64::from(config.required_corroborations.saturating_sub(1))); + if config.max_corroboration_window_ms <= spread { + anyhow::bail!( + "FLUXER_RECON_MAX_CORROBORATION_WINDOW_MS ({}) must exceed the gap times one fewer than the required corroborations ({spread})", + config.max_corroboration_window_ms + ); + } + + if config.coverage_target_ms < MIN_COVERAGE_TARGET_MS { + anyhow::bail!( + "FLUXER_RECON_COVERAGE_TARGET_MS must be at least {MIN_COVERAGE_TARGET_MS}, got {}", + config.coverage_target_ms + ); + } + + if config.coverage_target_ms > config.coverage_period_hard_cap_ms { + anyhow::bail!( + "FLUXER_RECON_COVERAGE_TARGET_MS ({}) must not exceed FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({})", + config.coverage_target_ms, + config.coverage_period_hard_cap_ms + ); + } + + Ok(()) +} + +fn text(get: &F, name: &str) -> Option +where + F: Fn(&str) -> Option, +{ + get(name) + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} + +fn positive_usize(value: usize, name: &str) -> anyhow::Result<()> { + if value == 0 { + anyhow::bail!("{name} must be greater than zero"); + } + Ok(()) +} + +fn positive_millis(value: u64, name: &str) -> anyhow::Result<()> { + if value == 0 { + anyhow::bail!("{name} must be greater than zero"); + } + Ok(()) +} + +fn fraction(value: f64, name: &str) -> anyhow::Result<()> { + if !(value > 0.0 && value <= 1.0) { + anyhow::bail!("{name} must lie in (0, 1], got {value}"); + } + Ok(()) +} + +fn clamped(value: u32, range: (u32, u32)) -> u32 { + let (low, high) = range; + value.clamp(low, high) +} + +fn read(get: &F, name: &str) -> Option +where + F: Fn(&str) -> Option, +{ + get(name).filter(|value| !value.is_empty()) +} + +fn number(get: &F, name: &str, default: T) -> anyhow::Result +where + F: Fn(&str) -> Option, + T: FromStr, + T::Err: Display, +{ + match read(get, name) { + None => Ok(default), + Some(raw) => raw + .trim() + .parse::() + .map_err(|error| anyhow::anyhow!("invalid {name}: {raw} ({error})")), + } +} + +fn boolean(get: &F, name: &str, default: bool) -> anyhow::Result +where + F: Fn(&str) -> Option, +{ + match read(get, name) { + None => Ok(default), + Some(raw) => parse_bool(&raw) + .ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw} is not a boolean")), + } +} + +fn mode(get: &F, name: &str, default: ReconMode) -> anyhow::Result +where + F: Fn(&str) -> Option, +{ + match read(get, name) { + None => Ok(default), + Some(raw) => ReconMode::parse(&raw).ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw}")), + } +} + +fn connection_id_guard( + get: &F, + name: &str, + default: ConnectionIdGuard, +) -> anyhow::Result +where + F: Fn(&str) -> Option, +{ + match read(get, name) { + None => Ok(default), + Some(raw) => { + ConnectionIdGuard::parse(&raw).ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw}")) + } + } +} + +fn parse_bool(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "y" | "on" => Some(true), + "0" | "false" | "no" | "n" | "off" => Some(false), + _ => None, + } +} diff --git a/fluxer_recon/src/control.rs b/fluxer_recon/src/control.rs new file mode 100644 index 000000000..b35aa182c --- /dev/null +++ b/fluxer_recon/src/control.rs @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use serde::{Deserialize, Serialize}; + +use fluxer_svc::transport::{Transport, TransportMessage, TransportSubscriber, reply_message}; + +use crate::budget::BudgetClass; +use crate::health::{ModeClamp, ReconMode}; +use crate::ids::{ChannelId, GuildId, Millis, RoomKey}; +use crate::runtime::{Shared, monotonic_now}; +use crate::singleton::{RESUBSCRIBE_BACKOFF_MS, next_backoff_ms}; +use crate::suspicion::SuspicionSource; + +pub const CONTROL_SUBJECT: &str = "svc.recon"; +pub const INSTANCE_SUBJECT: &str = "svc.recon.instance"; +pub const EXPLAIN_DECISION_LIMIT: usize = 16; + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum RawId { + Number(u64), + Text(String), +} + +impl RawId { + pub fn value(&self) -> Option { + match self { + Self::Number(value) => Some(*value), + Self::Text(text) => parse_u64_strict(text), + } + } +} + +fn parse_u64_strict(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse::().ok() +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum ControlRequest { + Status, + Explain { + #[serde(default)] + guild_id: Option, + channel_id: RawId, + }, + Mode { + #[serde(default)] + mode: Option, + }, + Budget, + Topology { + #[serde(default)] + refresh: Option, + }, + Clamps { + #[serde(default)] + release: Option, + }, + Suspect { + #[serde(default)] + guild_id: Option, + channel_id: RawId, + }, + Cliffs { + #[serde(default)] + release: Option, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ControlError { + DecodeFailed, + UnknownMode, + UnknownClamp, + InvalidChannelId, + InvalidGuildId, +} + +impl ControlError { + pub const fn label(self) -> &'static str { + match self { + Self::DecodeFailed => "decode_failed", + Self::UnknownMode => "unknown_mode", + Self::UnknownClamp => "unknown_clamp", + Self::InvalidChannelId => "invalid_channel_id", + Self::InvalidGuildId => "invalid_guild_id", + } + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct ErrorResponse { + pub ok: bool, + pub error: &'static str, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ModeResponse { + pub ok: bool, + pub configured: &'static str, + pub runtime_override: Option<&'static str>, + pub effective: &'static str, + pub clamps: Vec<&'static str>, + pub changed: bool, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ServerHealthEntry { + pub region: String, + pub server: String, + pub health: &'static str, + pub consecutive_failures: u32, +} + +#[derive(Clone, Debug, Serialize)] +pub struct PeerEntry { + pub instance_id: String, + pub started_at: u64, + pub first_seen_ms_ago: u64, + pub last_seen_ms_ago: u64, + pub detections: u64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct StatusResponse { + pub ok: bool, + pub instance_id: String, + pub started_at: u64, + pub uptime_ms: u64, + pub ready: bool, + pub draining: bool, + pub configured_mode: &'static str, + pub runtime_override: Option<&'static str>, + pub effective_mode: &'static str, + pub clamps: Vec<&'static str>, + pub warmup_elapsed: bool, + pub coverage_pass_complete: bool, + pub tracked_rooms: usize, + pub tracked_connections: usize, + pub tracked_bytes: u64, + pub divergent_connections: usize, + pub divergent_fraction: f64, + pub coverage_period_ms: u64, + pub breaker_tripped: bool, + pub breaker_trips_total: u64, + pub breaker_auto_resets_used: u32, + pub breaker_held_for_manual_reset: bool, + pub mutations_paused_ms_remaining: u64, + pub topology_age_ms: Option, + pub topology_servers: usize, + pub census_age_ms: Option, + pub last_turn: u64, + pub servers: Vec, + pub peer: Option, +} + +#[derive(Clone, Debug, Serialize)] +pub struct BucketEntry { + pub class: &'static str, + pub tokens: f64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct DenialEntry { + pub reason: &'static str, + pub count: u64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct BudgetResponse { + pub ok: bool, + pub effective_mode: &'static str, + pub buckets: Vec, + pub mutations_issued_total: u64, + pub mutations_last_minute: u32, + pub max_mutations_per_min: u32, + pub max_action_attempts: u32, + pub pending_actions: usize, + pub scheduler_starved_total: u64, + pub ledger_evicted_total: u64, + pub denials: Vec, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ConnectionDump { + pub connection_id: String, + pub user_id: String, + pub state: &'static str, + pub corroborations: u32, + pub divergence_since_ms: Option, + pub fingerprint: Option, + pub last_seen_ms_ago: u64, + pub action_attempts: u8, + pub repair_attempts: u8, + pub last_repair_verdict: Option<&'static str>, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ExplainResponse { + pub ok: bool, + pub scope: &'static str, + pub guild_id: Option, + pub channel_id: String, + pub tracked: bool, + pub last_turn: Option, + pub last_seen_ms_ago: Option, + pub partially_unreadable: bool, + pub no_candidates: bool, + pub gateway_cliff_since_ms_ago: Option, + pub media_cliff_since_ms_ago: Option, + pub last_known_locations: Vec, + pub connections: Vec, + pub decisions: Vec, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ClampEntry { + pub clamp: &'static str, + pub engaged: bool, + pub self_clearing: bool, + pub expires_in_ms: Option, +} + +#[derive(Clone, Debug, Serialize)] +pub struct ClampsResponse { + pub ok: bool, + pub configured_mode: &'static str, + pub effective_mode: &'static str, + pub ceiling: &'static str, + pub clamps: Vec, + pub released: Vec<&'static str>, + pub breaker_tripped: bool, + pub breaker_auto_resets_used: u32, + pub breaker_held_for_manual_reset: bool, + pub mutations_paused_windows: u32, + pub mutations_paused_hold_ms: u64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct CliffsResponse { + pub ok: bool, + pub cliffed: Vec, + pub disputed: Vec, + pub holed: Vec, + pub released: Vec, + pub hold_ms: u64, + pub tripped_total: u64, + pub disputed_total: u64, + pub holes_total: u64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct SuspectResponse { + pub ok: bool, + pub scope: &'static str, + pub guild_id: Option, + pub channel_id: String, + pub admitted: &'static str, + pub raised: &'static str, + pub hold_ms: u64, + pub suspect_rooms: usize, +} + +#[derive(Clone, Debug, Serialize)] +pub struct TopologyResponse { + pub ok: bool, + pub servers: usize, + pub age_ms: Option, + pub max_age_ms: u64, + pub stale: bool, + pub refresh_requested: bool, +} + +fn clamp_labels(shared: &Shared) -> Vec<&'static str> { + shared.with_state(|state| { + state + .governor + .clamps() + .engaged() + .map(ModeClamp::label) + .collect() + }) +} + +fn error(code: ControlError) -> serde_json::Value { + serde_json::json!(ErrorResponse { + ok: false, + error: code.label(), + }) +} + +fn room_key(guild_id: Option<&RawId>, channel_id: &RawId) -> Result { + let channel = channel_id.value().ok_or(ControlError::InvalidChannelId)?; + match guild_id { + None => Ok(RoomKey::Dm { + channel_id: ChannelId::new(channel), + }), + Some(raw) => { + let guild = raw.value().ok_or(ControlError::InvalidGuildId)?; + Ok(RoomKey::Guild { + guild_id: GuildId::new(guild), + channel_id: ChannelId::new(channel), + }) + } + } +} + +pub fn status(shared: &Shared, now: Millis) -> StatusResponse { + shared.with_state(|state| StatusResponse { + ok: true, + instance_id: shared.instance().id.clone(), + started_at: shared.instance().started_at, + uptime_ms: now.saturating_since(shared.boot_at()), + ready: shared.readiness().is_ready(), + draining: shared.readiness().is_draining(), + configured_mode: state.governor.configured_mode().as_str(), + runtime_override: state.governor.runtime_override().map(ReconMode::as_str), + effective_mode: state.governor.effective_mode().as_str(), + clamps: state + .governor + .clamps() + .engaged() + .map(ModeClamp::label) + .collect(), + warmup_elapsed: state.governor.warmup().elapsed_complete(now), + coverage_pass_complete: state.governor.warmup().coverage_pass_complete(), + tracked_rooms: state.ledger.tracked_rooms(), + tracked_connections: state.ledger.tracked_connections(), + tracked_bytes: state.ledger.tracked_bytes(), + divergent_connections: state.ledger.divergent_connections(), + divergent_fraction: state.ledger.divergent_fraction(), + coverage_period_ms: state.coverage_period_ms, + breaker_tripped: state.governor.breaker().is_tripped(), + breaker_trips_total: state.governor.breaker().trips_total(), + breaker_auto_resets_used: state.governor.breaker().auto_resets_used(), + breaker_held_for_manual_reset: state + .governor + .breaker() + .held_for_manual_reset(state.governor.limits().breaker), + mutations_paused_ms_remaining: state.mutations_pause.remaining_ms(now), + topology_age_ms: state.topology_age_ms(now), + topology_servers: state.topology_servers, + census_age_ms: state.census_age_ms(now), + last_turn: state.turn.get(), + servers: state + .server_health + .tracked() + .map(|(location, tracker)| ServerHealthEntry { + region: location.region.to_string(), + server: location.server.to_string(), + health: tracker.state().label(), + consecutive_failures: tracker.consecutive_failures(), + }) + .collect(), + peer: state.peer.as_ref().map(|peer| PeerEntry { + instance_id: peer.instance_id.clone(), + started_at: peer.started_at, + first_seen_ms_ago: now.saturating_since(peer.first_seen_at), + last_seen_ms_ago: now.saturating_since(peer.last_seen_at), + detections: peer.detections, + }), + }) +} + +pub fn budget(shared: &Shared, now: Millis) -> BudgetResponse { + let metrics = shared.metrics().clone(); + shared.with_state_mut(|state| { + let buckets = [ + BudgetClass::GatewayRead, + BudgetClass::GatewayMutate, + BudgetClass::LiveKitRead, + BudgetClass::LiveKitMutate, + ] + .into_iter() + .map(|class| BucketEntry { + class: class.label(), + tokens: state.governor.tokens(class, now), + }) + .collect(); + + BudgetResponse { + ok: true, + effective_mode: state.governor.effective_mode().as_str(), + buckets, + mutations_issued_total: state.governor.mutations().issued_total(), + mutations_last_minute: state.governor.mutations().global_count(now), + max_mutations_per_min: state.governor.limits().max_mutations_per_min, + max_action_attempts: state.governor.limits().max_action_attempts, + pending_actions: state.governor.pending_actions(), + scheduler_starved_total: metrics.scheduler_starved_total(), + ledger_evicted_total: metrics.ledger_evicted_total(), + denials: state + .governor + .denials() + .map(|(reason, count)| DenialEntry { + reason: reason.label(), + count: *count, + }) + .collect(), + } + }) +} + +pub fn explain(shared: &Shared, room: RoomKey, now: Millis) -> ExplainResponse { + let decisions = shared.journal().for_room(room, EXPLAIN_DECISION_LIMIT); + shared.with_state(|state| { + let entry = state.ledger.room(&room); + ExplainResponse { + ok: true, + scope: crate::metrics::scope_label(room.scope()), + guild_id: room.guild_id().map(|guild| guild.to_string()), + channel_id: room.channel_id().to_string(), + tracked: entry.is_some(), + last_turn: entry.map(|room| room.last_turn().get()), + last_seen_ms_ago: entry.map(|room| now.saturating_since(room.last_seen())), + partially_unreadable: entry.is_some_and(|room| room.partially_unreadable()), + no_candidates: entry.is_some_and(|room| room.no_candidates()), + gateway_cliff_since_ms_ago: entry + .and_then(|room| room.gateway_cliff().since()) + .map(|since| now.saturating_since(since)), + media_cliff_since_ms_ago: entry + .and_then(|room| room.media_cliff().since()) + .map(|since| now.saturating_since(since)), + last_known_locations: entry + .map(|room| { + room.last_known_locations() + .iter() + .filter_map(|index| state.ledger.location(*index)) + .map(ToString::to_string) + .collect() + }) + .unwrap_or_default(), + connections: entry + .map(|room| { + room.connections() + .iter() + .map(|connection| ConnectionDump { + connection_id: connection.connection.to_string(), + user_id: connection.user_id.to_string(), + state: connection.state.label(), + corroborations: connection.state.corroborations(), + divergence_since_ms: connection + .state + .divergence_since() + .map(|since| now.saturating_since(since)), + fingerprint: connection + .state + .stored_fingerprint() + .map(|fingerprint| format!("{:016x}", fingerprint.get())), + last_seen_ms_ago: now.saturating_since(connection.last_seen), + action_attempts: connection.action_attempts, + repair_attempts: connection.repair_attempts, + last_repair_verdict: connection + .last_repair_verdict + .map(|verdict| verdict.label()), + }) + .collect() + }) + .unwrap_or_default(), + decisions, + } + }) +} + +pub fn set_mode(shared: &Shared, requested: Option<&str>) -> Result { + let parsed = match requested { + None => None, + Some(raw) => Some(ReconMode::parse(raw).ok_or(ControlError::UnknownMode)?), + }; + + let transition = shared.set_mode_override(parsed); + + tracing::warn!( + configured = transition.configured.as_str(), + previous_override = transition.previous_override.map(ReconMode::as_str), + requested = transition.requested.map(ReconMode::as_str), + effective_before = transition.effective_before.as_str(), + effective_after = transition.effective_after.as_str(), + "recon mode override applied over the control plane" + ); + + Ok(ModeResponse { + ok: true, + configured: transition.configured.as_str(), + runtime_override: transition.requested.map(ReconMode::as_str), + effective: transition.effective_after.as_str(), + clamps: clamp_labels(shared), + changed: transition.changed(), + }) +} + +pub fn read_mode(shared: &Shared) -> ModeResponse { + shared.with_state(|state| ModeResponse { + ok: true, + configured: state.governor.configured_mode().as_str(), + runtime_override: state.governor.runtime_override().map(ReconMode::as_str), + effective: state.governor.effective_mode().as_str(), + clamps: state + .governor + .clamps() + .engaged() + .map(ModeClamp::label) + .collect(), + changed: false, + }) +} + +pub fn topology(shared: &Shared, refresh: bool, now: Millis) -> TopologyResponse { + let max_age_ms = shared.config().topology_max_age_ms; + shared.with_state_mut(|state| { + if refresh { + state.topology_refresh_requested = true; + } + let age_ms = state.topology_age_ms(now); + TopologyResponse { + ok: true, + servers: state.topology_servers, + age_ms, + max_age_ms, + stale: age_ms.is_none_or(|age| age > max_age_ms), + refresh_requested: state.topology_refresh_requested, + } + }) +} + +fn release_clamp(shared: &Shared, clamp: ModeClamp) -> bool { + shared.with_state_mut(|state| { + let was_engaged = state.governor.clamps().is_engaged(clamp); + match clamp { + ModeClamp::BreakerTripped => { + state.governor.reset_breaker(); + } + ModeClamp::MutationsPaused => state.mutations_pause.clear(), + ModeClamp::Warmup => state.governor.warmup_mut().force_complete(), + ModeClamp::ColdStart => state.governor.warmup_mut().note_coverage_pass(), + ModeClamp::SingletonConflict => state.peer = None, + ModeClamp::TopologyStale | ModeClamp::NatsReconnect | ModeClamp::CoverageOverrun => {} + } + state.governor.clamps_mut().release(clamp); + was_engaged + }) +} + +pub fn clamps( + shared: &Shared, + release: Option<&str>, + now: Millis, +) -> Result { + let wanted: Vec = match release { + None => Vec::new(), + Some(name) if name.trim().eq_ignore_ascii_case("all") => ModeClamp::ALL.to_vec(), + Some(name) => vec![ModeClamp::parse(name).ok_or(ControlError::UnknownClamp)?], + }; + + let mut released: Vec<&'static str> = Vec::new(); + for clamp in wanted { + if release_clamp(shared, clamp) { + released.push(clamp.label()); + } + } + + if !released.is_empty() { + tracing::warn!( + released = ?released, + "an operator released mode clamps over the control plane, so the effective mode may \ + rise before the condition that engaged them is gone" + ); + } + + Ok(shared.with_state_mut(|state| { + let limits = state.governor.limits().breaker; + ClampsResponse { + ok: true, + configured_mode: state.governor.configured_mode().as_str(), + effective_mode: state.governor.effective_mode().as_str(), + ceiling: state.governor.clamps().ceiling().as_str(), + clamps: ModeClamp::ALL + .into_iter() + .map(|clamp| ClampEntry { + clamp: clamp.label(), + engaged: state.governor.clamps().is_engaged(clamp), + self_clearing: clamp.is_self_clearing(), + expires_in_ms: state + .governor + .clamps() + .deadline(clamp) + .map(|until| until.saturating_since(now)), + }) + .collect(), + released, + breaker_tripped: state.governor.breaker().is_tripped(), + breaker_auto_resets_used: state.governor.breaker().auto_resets_used(), + breaker_held_for_manual_reset: state.governor.breaker().held_for_manual_reset(limits), + mutations_paused_windows: state.mutations_pause.windows(), + mutations_paused_hold_ms: state.mutations_pause.hold_ms(), + } + })) +} + +pub fn cliffs(shared: &Shared, release: bool, now: Millis) -> CliffsResponse { + let hold_ms = shared.config().server_cliff_hold_ms; + let response = shared.with_state_mut(|state| { + let cliffed = state.cliffs.cliffed(now); + let disputed = state.cliffs.disputed(); + let holed = state.suspicion.holed_servers(); + let mut released: Vec = Vec::new(); + + if release { + for location in cliffed.iter().chain(holed.iter()) { + let forgotten = state.cliffs.forget(location); + let filled = state.suspicion.clear_hole(location); + if forgotten || filled { + released.push(location.to_string()); + } + } + released.sort(); + released.dedup(); + } + + CliffsResponse { + ok: true, + cliffed: cliffed.iter().map(ToString::to_string).collect(), + disputed: disputed.iter().map(ToString::to_string).collect(), + holed: holed.iter().map(ToString::to_string).collect(), + released, + hold_ms, + tripped_total: state.cliffs.tripped_total(), + disputed_total: state.cliffs.disputed_total(), + holes_total: state.suspicion.holes_total(), + } + }); + + if !response.released.is_empty() { + tracing::warn!( + released = ?response.released, + "an operator dropped server holds over the control plane, so those servers read as \ + authoritative again before their hold window ended" + ); + } + + response +} + +pub fn suspect(shared: &Shared, room: RoomKey, now: Millis) -> SuspectResponse { + let (admitted, raised, suspect_rooms, hold_ms) = shared.with_state_mut(|state| { + let admitted = state.directory.note_suspicion(room, now); + let raised = state + .suspicion + .note_room(room, SuspicionSource::Control, now); + ( + admitted.label(), + raised.label(), + state.suspicion.suspect_rooms(now).len(), + state.suspicion.limits().hold_ms, + ) + }); + + tracing::info!( + channel_id = room.channel_id().get(), + guild_id = room.guild_id().map(|guild| guild.get()), + admitted, + raised, + "an operator raised suspicion over the control plane, which only reorders reads" + ); + + SuspectResponse { + ok: true, + scope: crate::metrics::scope_label(room.scope()), + guild_id: room.guild_id().map(|guild| guild.to_string()), + channel_id: room.channel_id().to_string(), + admitted, + raised, + hold_ms, + suspect_rooms, + } +} + +pub fn dispatch(shared: &Shared, request: &ControlRequest, now: Millis) -> serde_json::Value { + match request { + ControlRequest::Status => serde_json::json!(status(shared, now)), + ControlRequest::Budget => serde_json::json!(budget(shared, now)), + ControlRequest::Mode { mode } => match mode { + None => serde_json::json!(read_mode(shared)), + Some(raw) => match set_mode(shared, Some(raw)) { + Ok(response) => serde_json::json!(response), + Err(code) => error(code), + }, + }, + ControlRequest::Explain { + guild_id, + channel_id, + } => match room_key(guild_id.as_ref(), channel_id) { + Ok(room) => serde_json::json!(explain(shared, room, now)), + Err(code) => error(code), + }, + ControlRequest::Topology { refresh } => { + serde_json::json!(topology(shared, refresh.unwrap_or(false), now)) + } + ControlRequest::Clamps { release } => match clamps(shared, release.as_deref(), now) { + Ok(response) => serde_json::json!(response), + Err(code) => error(code), + }, + ControlRequest::Suspect { + guild_id, + channel_id, + } => match room_key(guild_id.as_ref(), channel_id) { + Ok(room) => serde_json::json!(suspect(shared, room, now)), + Err(code) => error(code), + }, + ControlRequest::Cliffs { release } => { + serde_json::json!(cliffs(shared, release.unwrap_or(false), now)) + } + } +} + +pub fn handle(shared: &Shared, payload: &[u8], now: Millis) -> Vec { + let response = match serde_json::from_slice::(payload) { + Ok(request) => dispatch(shared, &request, now), + Err(parse_error) => { + tracing::debug!(error = %parse_error, "rejecting an undecodable control request"); + error(ControlError::DecodeFailed) + } + }; + serde_json::to_vec(&response) + .unwrap_or_else(|_| b"{\"ok\":false,\"error\":\"encode_failed\"}".to_vec()) +} + +pub async fn run_control(shared: Shared, transport: T) -> anyhow::Result<()> { + let mut backoff_ms = RESUBSCRIBE_BACKOFF_MS; + + loop { + let mut subscription = match transport.subscribe(CONTROL_SUBJECT).await { + Ok(subscription) => { + backoff_ms = RESUBSCRIBE_BACKOFF_MS; + subscription + } + Err(error) => { + tracing::error!( + error = %error, + subject = CONTROL_SUBJECT, + retry_in_ms = backoff_ms, + "the control plane could not subscribe, retrying rather than ending the task, \ + because the escape hatch for a clamped service must outlive a broken \ + subscription" + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + backoff_ms = next_backoff_ms(backoff_ms); + continue; + } + }; + + tracing::info!( + subject = CONTROL_SUBJECT, + instance_id = shared.instance().id, + "recon control plane listening" + ); + + loop { + tokio::select! { + message = subscription.next() => { + let Some(message) = message else { + tracing::warn!("control subscription ended, will re-subscribe"); + break; + }; + if !message.has_reply() { + continue; + } + let response = handle(&shared, message.payload(), monotonic_now()); + if let Err(error) = reply_message(&message, &transport, &response).await { + tracing::debug!(error = %error, "failed to answer a control request"); + } + } + _ = transport.wait_for_reconnect() => { + tracing::info!("NATS reconnected, re-subscribing the control plane"); + break; + } + } + } + } +} diff --git a/fluxer_recon/src/decide.rs b/fluxer_recon/src/decide.rs new file mode 100644 index 000000000..f1a241845 --- /dev/null +++ b/fluxer_recon/src/decide.rs @@ -0,0 +1,873 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::budget::BudgetView; +use crate::config::ReconConfig; +use crate::evidence::{PendingJoin, Presence, RoomObservation, Side, UnknownReason}; +use crate::gateway::Nonce; +use crate::guards::{AbortReason, DmPosture}; +use crate::ids::{ConnectionId, ConnectionKey, Location, Millis, RoomKey, Scope, TurnId, UserId}; +use crate::ledger::{ + ActionKind, ConnectionState, Fingerprint, FingerprintInput, Ledger, LocationIndex, + RepairVerdict, WedgedReason, fingerprint, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecideParams { + pub required_corroborations: u32, + pub min_corroboration_gap_ms: u64, + pub max_corroboration_window_ms: u64, + pub min_divergence_ms: u64, + pub max_action_attempts: u32, + pub pending_join_skew_ms: u64, + pub dm_posture: DmPosture, +} + +impl Default for DecideParams { + fn default() -> Self { + Self { + required_corroborations: 3, + min_corroboration_gap_ms: 1_000, + max_corroboration_window_ms: 60_000, + min_divergence_ms: 15_000, + max_action_attempts: 3, + pending_join_skew_ms: 5_000, + dm_posture: DmPosture::default(), + } + } +} + +impl DecideParams { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + required_corroborations: config.required_corroborations, + min_corroboration_gap_ms: config.min_corroboration_gap_ms, + max_corroboration_window_ms: config.max_corroboration_window_ms, + min_divergence_ms: config.min_divergence_ms, + max_action_attempts: config.max_action_attempts, + pending_join_skew_ms: config.pending_join_skew_ms, + dm_posture: DmPosture { + dm_gateway_eviction: config.dm_gateway_eviction, + dm_media_eviction: config.dm_media_eviction, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ActionClass { + None, + Constructive, + Destructive, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DecisionAction { + Hold, + ConfirmConnection { nonce: Nonce }, + RepairState, + RemoveGatewayState, + RemoveParticipant { location: Location }, +} + +impl DecisionAction { + pub const fn class(&self) -> ActionClass { + match self { + Self::Hold => ActionClass::None, + Self::ConfirmConnection { .. } | Self::RepairState => ActionClass::Constructive, + Self::RemoveGatewayState | Self::RemoveParticipant { .. } => ActionClass::Destructive, + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Hold => "hold", + Self::ConfirmConnection { .. } => "confirm_connection", + Self::RepairState => "repair_state", + Self::RemoveGatewayState => "remove_gateway_state", + Self::RemoveParticipant { .. } => "remove_participant", + } + } + + pub const fn kind(&self) -> Option { + match self { + Self::Hold | Self::ConfirmConnection { .. } | Self::RepairState => None, + Self::RemoveGatewayState => Some(ActionKind::RemoveGatewayState), + Self::RemoveParticipant { .. } => Some(ActionKind::RemoveParticipant), + } + } + + pub const fn is_destructive(&self) -> bool { + matches!(self.class(), ActionClass::Destructive) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DecisionReason { + SideUnknown(UnknownReason), + FingerprintChanged, + BothSidesPresent, + BothSidesAbsent, + DivergenceOpened, + CorroborationRecorded, + CorroborationSpacedTooTightly, + CorroborationWindowExpired, + DivergenceTooYoung, + EvidenceComplete, + PendingJoinOutstanding, + RepairAttempted, + RepairSucceeded, + RepairMadeNoChange, + RepairNotPossible, + RepairNotDefinitive, + RepairExhausted, + ActionUnresolved, + ActionsExhausted, + ParticipantJoinedAfterDivergence, + LocationUnavailable, + WedgedHolding, +} + +impl DecisionReason { + pub const fn label(&self) -> &'static str { + match self { + Self::SideUnknown(_) => "side_unknown", + Self::FingerprintChanged => "fingerprint_changed", + Self::BothSidesPresent => "both_sides_present", + Self::BothSidesAbsent => "both_sides_absent", + Self::DivergenceOpened => "divergence_opened", + Self::CorroborationRecorded => "corroboration_recorded", + Self::CorroborationSpacedTooTightly => "corroboration_spaced_too_tightly", + Self::CorroborationWindowExpired => "corroboration_window_expired", + Self::DivergenceTooYoung => "divergence_too_young", + Self::EvidenceComplete => "evidence_complete", + Self::PendingJoinOutstanding => "pending_join_outstanding", + Self::RepairAttempted => "repair_attempted", + Self::RepairSucceeded => "repair_succeeded", + Self::RepairMadeNoChange => "repair_made_no_change", + Self::RepairNotPossible => "repair_not_possible", + Self::RepairNotDefinitive => "repair_not_definitive", + Self::RepairExhausted => "repair_exhausted", + Self::ActionUnresolved => "action_unresolved", + Self::ActionsExhausted => "actions_exhausted", + Self::ParticipantJoinedAfterDivergence => "participant_joined_after_divergence", + Self::LocationUnavailable => "location_unavailable", + Self::WedgedHolding => "wedged_holding", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Decision { + pub connection: ConnectionKey, + pub user_id: UserId, + pub action: DecisionAction, + pub reason: DecisionReason, + pub from: &'static str, + pub to: &'static str, + pub blocked_by: Option, +} + +impl Decision { + pub const fn is_actionable(&self) -> bool { + self.blocked_by.is_none() && !matches!(self.action, DecisionAction::Hold) + } + + pub const fn is_destructive(&self) -> bool { + self.action.is_destructive() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DecisionSet { + pub room: RoomKey, + pub turn: TurnId, + pub at: Millis, + pub decisions: Vec, +} + +impl DecisionSet { + pub fn empty(room: RoomKey, turn: TurnId, at: Millis) -> Self { + Self { + room, + turn, + at, + decisions: Vec::new(), + } + } + + pub fn actionable(&self) -> impl Iterator { + self.decisions.iter().filter(|entry| entry.is_actionable()) + } + + pub fn destructive(&self) -> impl Iterator { + self.decisions.iter().filter(|entry| entry.is_destructive()) + } + + pub fn actionable_destructive(&self) -> impl Iterator { + self.decisions + .iter() + .filter(|entry| entry.is_actionable() && entry.is_destructive()) + } + + pub fn blocked(&self) -> impl Iterator { + self.decisions + .iter() + .filter_map(|entry| entry.blocked_by.map(|reason| (entry, reason))) + } + + pub fn for_connection(&self, connection: &ConnectionId) -> Option<&Decision> { + self.decisions + .iter() + .find(|entry| &entry.connection.connection == connection) + } +} + +struct Context<'a> { + params: &'a DecideParams, + room: RoomKey, + now: Millis, + fingerprint: Fingerprint, + location: Option, + location_value: Option<&'a Location>, + joined_at: Option, + pending_join: Option<&'a PendingJoin>, + budget: &'a BudgetView, + action_attempts: u32, + repair_attempts: u32, + repair_verdict: Option, +} + +struct Outcome { + state: ConnectionState, + action: DecisionAction, + reason: DecisionReason, + blocked_by: Option, +} + +impl Outcome { + const fn hold(state: ConnectionState, reason: DecisionReason) -> Self { + Self { + state, + action: DecisionAction::Hold, + reason, + blocked_by: None, + } + } +} + +pub fn decide( + ledger: &mut Ledger, + observation: &RoomObservation, + now: Millis, + budget: &BudgetView, +) -> DecisionSet { + let params = *ledger.params(); + let room = observation.room; + + if ledger.ensure_room(room, now).is_err() { + return DecisionSet::empty(room, observation.turn, now); + } + + let no_candidates = observation.candidates.is_empty(); + let partially_unreadable = matches!( + observation.authority.media.unknown_reason(), + Some(UnknownReason::RoomPartiallyUnreadable) + ); + if let Some(entry) = ledger.room_mut(&room) { + entry.note_turn(observation.turn, now); + entry.set_no_candidates(no_candidates); + entry.set_partially_unreadable(partially_unreadable); + } + + let mut work: Vec<(ConnectionId, UserId)> = observation + .connections + .iter() + .map(|entry| (entry.connection.clone(), entry.user_id)) + .collect(); + if let Some(entry) = ledger.room(&room) { + for tracked in entry.connections() { + if !work.iter().any(|(id, _)| id == &tracked.connection) { + work.push((tracked.connection.clone(), tracked.user_id)); + } + } + } + work.sort(); + work.dedup(); + + let unsighted = observation.unsighted_presence(); + let mut decisions = Vec::with_capacity(work.len()); + + for (connection, user_id) in work { + let observed = observation.observation_for(&connection); + let presence: Side = + observed.map_or_else(|| unsighted.clone(), |entry| entry.presence.clone()); + let siblings = observed + .map(|entry| entry.gateway_siblings.clone()) + .unwrap_or_default(); + let locations = observed + .map(|entry| entry.media_locations.clone()) + .unwrap_or_default(); + let joined_at = observed.and_then(|entry| entry.participant_joined_at); + + let print = fingerprint(&FingerprintInput { + room, + connection: &connection, + user_id, + gateway_siblings: &siblings, + media_locations: &locations, + participant_joined_at: joined_at, + census_epoch: observation.census_epoch, + topology_epoch: observation.topology_epoch, + }); + + let location_index = locations + .first() + .and_then(|location| ledger.intern_location(location)); + if let (Some(index), Some(entry)) = (location_index, ledger.room_mut(&room)) { + entry.note_location(index); + } + + let pending_join = observation.pending_join_for( + &connection, + observation.wall_at, + params.pending_join_skew_ms, + ); + + let Ok(entry) = ledger.upsert_connection(room, &connection, user_id, now) else { + continue; + }; + if let Some(index) = location_index { + entry.last_location = Some(index); + } + + let context = Context { + params: ¶ms, + room, + now, + fingerprint: print, + location: location_index, + location_value: locations.first(), + joined_at, + pending_join, + budget, + action_attempts: u32::from(entry.action_attempts), + repair_attempts: u32::from(entry.repair_attempts), + repair_verdict: entry.last_repair_verdict, + }; + + let from = entry.state.label(); + let outcome = transition(&entry.state, &presence, &context); + + if matches!( + outcome.state, + ConnectionState::Nascent | ConnectionState::Consistent + ) { + entry.last_repair_verdict = None; + } + entry.state = outcome.state; + + decisions.push(Decision { + connection: ConnectionKey::new(room, connection), + user_id, + action: outcome.action, + reason: outcome.reason, + from, + to: entry.state.label(), + blocked_by: outcome.blocked_by, + }); + } + + DecisionSet { + room, + turn: observation.turn, + at: now, + decisions, + } +} + +fn transition(prior: &ConnectionState, presence: &Side, ctx: &Context<'_>) -> Outcome { + match (&presence.gateway, &presence.media) { + (Presence::Unknown(reason), Presence::Unknown(_)) + | (Presence::Unknown(reason), Presence::Present) + | (Presence::Unknown(reason), Presence::Absent) + | (Presence::Present, Presence::Unknown(reason)) + | (Presence::Absent, Presence::Unknown(reason)) => Outcome::hold( + ConnectionState::Nascent, + DecisionReason::SideUnknown(reason.clone()), + ), + (Presence::Present, Presence::Present) => Outcome::hold( + ConnectionState::Consistent, + DecisionReason::BothSidesPresent, + ), + (Presence::Absent, Presence::Absent) => Outcome::hold( + ConnectionState::Retired { + at: retired_at(prior, ctx.now), + }, + DecisionReason::BothSidesAbsent, + ), + (Presence::Present, Presence::Absent) => gateway_only_lane(prior, ctx), + (Presence::Absent, Presence::Present) => media_only_lane(prior, ctx), + } +} + +const fn retired_at(prior: &ConnectionState, now: Millis) -> Millis { + match prior { + ConnectionState::Retired { at } => *at, + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::GatewayOnly { .. } + | ConnectionState::MediaOnly { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::ActionTaken { .. } + | ConnectionState::Wedged { .. } => now, + } +} + +fn fingerprint_broke(prior: &ConnectionState, ctx: &Context<'_>) -> bool { + prior + .stored_fingerprint() + .is_some_and(|stored| stored != ctx.fingerprint) +} + +fn nascent_on_fingerprint_change() -> Outcome { + Outcome::hold(ConnectionState::Nascent, DecisionReason::FingerprintChanged) +} + +fn gateway_only_lane(prior: &ConnectionState, ctx: &Context<'_>) -> Outcome { + if fingerprint_broke(prior, ctx) { + return nascent_on_fingerprint_change(); + } + + match prior { + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::MediaOnly { .. } + | ConnectionState::Retired { .. } => open_gateway_divergence(ctx), + ConnectionState::GatewayOnly { + since, + corroborations, + last_corroboration, + fingerprint: _, + } => corroborate_gateway(*since, *corroborations, *last_corroboration, ctx), + ConnectionState::ActionTaken { kind, since, .. } => match kind { + ActionKind::RemoveGatewayState => { + if ctx.action_attempts >= ctx.params.max_action_attempts { + return wedged( + WedgedReason::ActionIneffective, + ctx, + DecisionReason::ActionsExhausted, + ); + } + evaluate_gateway( + *since, + ctx.params.required_corroborations, + ctx.now, + ctx, + DecisionReason::ActionUnresolved, + ) + } + ActionKind::RemoveParticipant => open_gateway_divergence(ctx), + }, + ConnectionState::Wedged { + reason, + at, + fingerprint: print, + } => Outcome::hold( + ConnectionState::Wedged { + reason: *reason, + at: *at, + fingerprint: *print, + }, + DecisionReason::WedgedHolding, + ), + } +} + +fn open_gateway_divergence(ctx: &Context<'_>) -> Outcome { + Outcome::hold( + ConnectionState::GatewayOnly { + since: ctx.now, + corroborations: 1, + fingerprint: ctx.fingerprint, + last_corroboration: ctx.now, + }, + DecisionReason::DivergenceOpened, + ) +} + +fn corroborate_gateway( + since: Millis, + corroborations: u32, + last_corroboration: Millis, + ctx: &Context<'_>, +) -> Outcome { + if ctx.now.saturating_since(since) > ctx.params.max_corroboration_window_ms { + return Outcome::hold( + ConnectionState::GatewayOnly { + since: ctx.now, + corroborations: 1, + fingerprint: ctx.fingerprint, + last_corroboration: ctx.now, + }, + DecisionReason::CorroborationWindowExpired, + ); + } + + if ctx.now.saturating_since(last_corroboration) < ctx.params.min_corroboration_gap_ms { + return evaluate_gateway( + since, + corroborations, + last_corroboration, + ctx, + DecisionReason::CorroborationSpacedTooTightly, + ); + } + + evaluate_gateway( + since, + corroborations.saturating_add(1), + ctx.now, + ctx, + DecisionReason::CorroborationRecorded, + ) +} + +fn evaluate_gateway( + since: Millis, + corroborations: u32, + last_corroboration: Millis, + ctx: &Context<'_>, + progress: DecisionReason, +) -> Outcome { + let state = ConnectionState::GatewayOnly { + since, + corroborations, + fingerprint: ctx.fingerprint, + last_corroboration, + }; + + if corroborations < ctx.params.required_corroborations { + return Outcome::hold(state, progress); + } + + if ctx.now.saturating_since(since) < ctx.params.min_divergence_ms { + return Outcome::hold(state, DecisionReason::DivergenceTooYoung); + } + + if ctx.action_attempts >= ctx.params.max_action_attempts { + return wedged( + WedgedReason::ActionIneffective, + ctx, + DecisionReason::ActionsExhausted, + ); + } + + let blocked_by = if ctx.params.dm_posture.allows_gateway_removal(ctx.room) { + ctx.budget.destructive_block() + } else { + Some(AbortReason::DmPosture) + }; + + Outcome { + state, + action: DecisionAction::RemoveGatewayState, + reason: DecisionReason::EvidenceComplete, + blocked_by, + } +} + +fn media_only_lane(prior: &ConnectionState, ctx: &Context<'_>) -> Outcome { + if fingerprint_broke(prior, ctx) { + return nascent_on_fingerprint_change(); + } + + match prior { + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::GatewayOnly { .. } + | ConnectionState::Retired { .. } => open_media_divergence(ctx), + ConnectionState::PendingJoin { last_confirm, .. } => match ctx.pending_join { + None => open_media_divergence(ctx), + Some(join) => { + let state = ConnectionState::PendingJoin { + expires_at: join.expires_at, + nonce: join.nonce.clone(), + last_confirm: *last_confirm, + }; + if ctx.now.saturating_since(*last_confirm) < ctx.params.min_corroboration_gap_ms { + return Outcome::hold(state, DecisionReason::PendingJoinOutstanding); + } + Outcome { + state, + action: DecisionAction::ConfirmConnection { + nonce: join.nonce.clone(), + }, + reason: DecisionReason::PendingJoinOutstanding, + blocked_by: ctx.budget.constructive_block(), + } + } + }, + ConnectionState::Repairing { + since, + last_attempt, + } => repairing_lane(*since, *last_attempt, ctx), + ConnectionState::MediaOnly { + since, + corroborations, + last_corroboration, + location, + participant_joined_at, + repair_verdict, + fingerprint: _, + } => corroborate_media( + *since, + *corroborations, + *last_corroboration, + *location, + *participant_joined_at, + *repair_verdict, + ctx, + ), + ConnectionState::ActionTaken { kind, since, .. } => match kind { + ActionKind::RemoveParticipant => { + if ctx.action_attempts >= ctx.params.max_action_attempts { + return wedged( + WedgedReason::ActionIneffective, + ctx, + DecisionReason::ActionsExhausted, + ); + } + reopen_media_divergence(*since, ctx, DecisionReason::ActionUnresolved) + } + ActionKind::RemoveGatewayState => open_media_divergence(ctx), + }, + ConnectionState::Wedged { + reason, + at, + fingerprint: print, + } => Outcome::hold( + ConnectionState::Wedged { + reason: *reason, + at: *at, + fingerprint: *print, + }, + DecisionReason::WedgedHolding, + ), + } +} + +fn open_media_divergence(ctx: &Context<'_>) -> Outcome { + if let Some(join) = ctx.pending_join { + return Outcome { + state: ConnectionState::PendingJoin { + expires_at: join.expires_at, + nonce: join.nonce.clone(), + last_confirm: Millis::ZERO, + }, + action: DecisionAction::ConfirmConnection { + nonce: join.nonce.clone(), + }, + reason: DecisionReason::PendingJoinOutstanding, + blocked_by: ctx.budget.constructive_block(), + }; + } + + match ctx.room.scope() { + Scope::Guild => { + if ctx.repair_attempts > ctx.params.max_action_attempts { + return wedged( + WedgedReason::RepairExhausted, + ctx, + DecisionReason::RepairExhausted, + ); + } + Outcome { + state: ConnectionState::Repairing { + since: ctx.now, + last_attempt: Millis::ZERO, + }, + action: DecisionAction::RepairState, + reason: DecisionReason::RepairAttempted, + blocked_by: ctx.budget.constructive_block(), + } + } + Scope::Dm => reopen_media_divergence(ctx.now, ctx, DecisionReason::DivergenceOpened), + } +} + +fn repairing_lane(since: Millis, last_attempt: Millis, ctx: &Context<'_>) -> Outcome { + match ctx.repair_verdict { + Some(RepairVerdict::Repaired) => { + Outcome::hold(ConnectionState::Consistent, DecisionReason::RepairSucceeded) + } + Some(RepairVerdict::NotRepairable) => { + reopen_media_divergence(since, ctx, DecisionReason::RepairNotPossible) + } + Some(RepairVerdict::NoChange) | None => { + if ctx.repair_attempts > ctx.params.max_action_attempts { + return wedged( + WedgedReason::RepairExhausted, + ctx, + DecisionReason::RepairExhausted, + ); + } + let state = ConnectionState::Repairing { + since, + last_attempt, + }; + if ctx.now.saturating_since(last_attempt) < ctx.params.min_corroboration_gap_ms { + return Outcome::hold(state, DecisionReason::RepairMadeNoChange); + } + Outcome { + state, + action: DecisionAction::RepairState, + reason: DecisionReason::RepairAttempted, + blocked_by: ctx.budget.constructive_block(), + } + } + } +} + +fn reopen_media_divergence(since: Millis, ctx: &Context<'_>, reason: DecisionReason) -> Outcome { + let Some(location) = ctx.location else { + return Outcome::hold( + ConnectionState::Nascent, + DecisionReason::LocationUnavailable, + ); + }; + + Outcome::hold( + ConnectionState::MediaOnly { + since, + corroborations: 1, + fingerprint: ctx.fingerprint, + last_corroboration: ctx.now, + location, + participant_joined_at: ctx.joined_at.unwrap_or(ctx.now), + repair_verdict: ctx.repair_verdict, + }, + reason, + ) +} + +#[allow(clippy::too_many_arguments)] +fn corroborate_media( + since: Millis, + corroborations: u32, + last_corroboration: Millis, + location: LocationIndex, + participant_joined_at: Millis, + repair_verdict: Option, + ctx: &Context<'_>, +) -> Outcome { + let joined_at = ctx.joined_at.unwrap_or(participant_joined_at); + + if joined_at > since { + return Outcome::hold( + ConnectionState::MediaOnly { + since: ctx.now, + corroborations: 1, + fingerprint: ctx.fingerprint, + last_corroboration: ctx.now, + location, + participant_joined_at: joined_at, + repair_verdict, + }, + DecisionReason::ParticipantJoinedAfterDivergence, + ); + } + + if ctx.now.saturating_since(since) > ctx.params.max_corroboration_window_ms { + return Outcome::hold( + ConnectionState::MediaOnly { + since: ctx.now, + corroborations: 1, + fingerprint: ctx.fingerprint, + last_corroboration: ctx.now, + location, + participant_joined_at: joined_at, + repair_verdict, + }, + DecisionReason::CorroborationWindowExpired, + ); + } + + let spaced = + ctx.now.saturating_since(last_corroboration) >= ctx.params.min_corroboration_gap_ms; + let corroborations = if spaced { + corroborations.saturating_add(1) + } else { + corroborations + }; + let last_corroboration = if spaced { ctx.now } else { last_corroboration }; + let progress = if spaced { + DecisionReason::CorroborationRecorded + } else { + DecisionReason::CorroborationSpacedTooTightly + }; + + let state = ConnectionState::MediaOnly { + since, + corroborations, + fingerprint: ctx.fingerprint, + last_corroboration, + location, + participant_joined_at: joined_at, + repair_verdict, + }; + + if corroborations < ctx.params.required_corroborations { + return Outcome::hold(state, progress); + } + + if ctx.now.saturating_since(since) < ctx.params.min_divergence_ms { + return Outcome::hold(state, DecisionReason::DivergenceTooYoung); + } + + if matches!(ctx.room, RoomKey::Guild { .. }) + && repair_verdict != Some(RepairVerdict::NotRepairable) + { + return Outcome::hold(state, DecisionReason::RepairNotDefinitive); + } + + if ctx.action_attempts >= ctx.params.max_action_attempts { + return wedged( + WedgedReason::ActionIneffective, + ctx, + DecisionReason::ActionsExhausted, + ); + } + + let Some(target) = ctx.location_value else { + return Outcome::hold(state, DecisionReason::LocationUnavailable); + }; + + let blocked_by = if ctx.params.dm_posture.allows_media_removal(ctx.room) { + ctx.budget.destructive_block() + } else { + Some(AbortReason::DmPosture) + }; + + Outcome { + state, + action: DecisionAction::RemoveParticipant { + location: target.clone(), + }, + reason: DecisionReason::EvidenceComplete, + blocked_by, + } +} + +fn wedged(reason: WedgedReason, ctx: &Context<'_>, decision: DecisionReason) -> Outcome { + Outcome::hold( + ConnectionState::Wedged { + reason, + at: ctx.now, + fingerprint: ctx.fingerprint, + }, + decision, + ) +} diff --git a/fluxer_recon/src/discovery.rs b/fluxer_recon/src/discovery.rs new file mode 100644 index 000000000..2b3484373 --- /dev/null +++ b/fluxer_recon/src/discovery.rs @@ -0,0 +1,1656 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::census::CensusSnapshot; +use crate::clock::Clock; +use crate::config::ReconConfig; +use crate::evidence::{ + CandidateSet, CandidateSources, CensusCrossCheck, GatewayAuthorityInput, GatewayRead, + GatewayVoiceState, LocationProbe, MediaAuthorityInput, MediaParticipant, PendingJoin, + ProbeResult, RoomObservation, RoomReads, ServerCliffConfig, ServerCliffDetector, Side, + SideAuthority, TopologyFreshness, UnknownReason, gateway_authority, media_authority, + unread_believed_home, +}; +use crate::gateway::GatewayApi; +use crate::health::{ProbeOutcome, ServerHealth, ServerHealthMap}; +use crate::ids::{ConnectionId, Epoch, Location, Millis, RoomKey, TurnId, WallMillis}; +use crate::ledger::{LOCATION_ENTRY_BYTES, LOCATION_MAP_BYTES, MAP_ENTRY_OVERHEAD_BYTES}; +use crate::livekit::{LiveKitApi, LiveKitFault, ParticipantRecord, ReadResult}; +use crate::names::{parse_participant_identity, parse_room_name}; +use crate::turn::{Digest, Fresh, ReadSource, TurnError, TurnToken}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum RoomSource { + GatewayCensus, + MediaRoomList, + PinnedServer, + Ledger, + Suspicion, +} + +impl RoomSource { + pub const ALL: [Self; 5] = [ + Self::GatewayCensus, + Self::MediaRoomList, + Self::PinnedServer, + Self::Ledger, + Self::Suspicion, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::GatewayCensus => "gateway_census", + Self::MediaRoomList => "media_room_list", + Self::PinnedServer => "pinned_server", + Self::Ledger => "ledger", + Self::Suspicion => "suspicion", + } + } + + pub const fn index(self) -> usize { + match self { + Self::GatewayCensus => 0, + Self::MediaRoomList => 1, + Self::PinnedServer => 2, + Self::Ledger => 3, + Self::Suspicion => 4, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RoomProvenance { + seen: [Option; RoomSource::ALL.len()], +} + +impl RoomProvenance { + pub const fn new() -> Self { + Self { + seen: [None; RoomSource::ALL.len()], + } + } + + const fn note(&mut self, source: RoomSource, now: Millis) { + self.seen[source.index()] = Some(now); + } + + pub const fn has(&self, source: RoomSource) -> bool { + self.seen[source.index()].is_some() + } + + pub const fn last_seen_by(&self, source: RoomSource) -> Option { + self.seen[source.index()] + } + + pub fn count(&self) -> usize { + self.seen.iter().filter(|entry| entry.is_some()).count() + } + + pub fn sources(&self) -> Vec { + RoomSource::ALL + .into_iter() + .filter(|source| self.has(*source)) + .collect() + } + + pub fn only(&self, source: RoomSource) -> bool { + self.has(source) && self.count() == 1 + } + + pub const fn is_gateway_only(&self) -> bool { + self.has(RoomSource::GatewayCensus) && !self.has(RoomSource::MediaRoomList) + } + + pub const fn is_media_only(&self) -> bool { + self.has(RoomSource::MediaRoomList) && !self.has(RoomSource::GatewayCensus) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Admission { + Admitted, + Refreshed, + Refused, +} + +impl Admission { + pub const fn label(self) -> &'static str { + match self { + Self::Admitted => "admitted", + Self::Refreshed => "refreshed", + Self::Refused => "refused", + } + } + + pub const fn is_tracked(self) -> bool { + matches!(self, Self::Admitted | Self::Refreshed) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DirectoryLimits { + pub max_rooms: usize, +} + +impl DirectoryLimits { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + max_rooms: config.max_tracked_rooms, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RoomEntry { + provenance: RoomProvenance, + locations: BTreeMap, + pinned: Option, + first_at: Millis, + last_at: Millis, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ServerRoomList { + Listed { + rooms: Vec, + unparseable: usize, + }, + Unreadable(LiveKitFault), +} + +impl ServerRoomList { + pub const fn is_readable(&self) -> bool { + matches!(self, Self::Listed { .. }) + } + + pub const fn is_complete(&self) -> bool { + matches!(self, Self::Listed { unparseable: 0, .. }) + } + + pub fn rooms(&self) -> &[RoomKey] { + match self { + Self::Listed { rooms, .. } => rooms, + Self::Unreadable(_) => &[], + } + } + + pub const fn unparseable(&self) -> usize { + match self { + Self::Listed { unparseable, .. } => *unparseable, + Self::Unreadable(_) => 0, + } + } + + pub const fn fault(&self) -> Option { + match self { + Self::Listed { .. } => None, + Self::Unreadable(fault) => Some(*fault), + } + } + + pub const fn outcome(&self) -> ProbeOutcome { + match self { + Self::Listed { .. } => ProbeOutcome::Success, + Self::Unreadable(fault) => { + if fault.is_auth_failure() { + ProbeOutcome::AuthFailure + } else { + ProbeOutcome::Failure + } + } + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Listed { .. } => "listed", + Self::Unreadable(_) => "unreadable", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RoomListEffect { + Applied { + listed: usize, + added: usize, + withdrawn: usize, + }, + AdditiveOnly { + listed: usize, + added: usize, + unparseable: usize, + }, + Unreadable(LiveKitFault), +} + +impl RoomListEffect { + pub const fn label(self) -> &'static str { + match self { + Self::Applied { .. } => "applied", + Self::AdditiveOnly { .. } => "additive_only", + Self::Unreadable(_) => "unreadable", + } + } + + pub const fn withdrew_locations(self) -> bool { + matches!(self, Self::Applied { .. }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiscoveryPass { + pub at: Millis, + pub servers: usize, + pub readable: usize, + pub complete: usize, + pub rooms_added: usize, + pub locations_withdrawn: usize, + pub unreadable: Vec<(Location, LiveKitFault)>, +} + +impl DiscoveryPass { + pub const fn empty(at: Millis) -> Self { + Self { + at, + servers: 0, + readable: 0, + complete: 0, + rooms_added: 0, + locations_withdrawn: 0, + unreadable: Vec::new(), + } + } + + pub fn is_complete(&self) -> bool { + self.servers > 0 && self.complete == self.servers + } + + pub fn unreadable_servers(&self) -> usize { + self.unreadable.len() + } +} + +#[derive(Clone, Debug)] +pub struct RoomDirectory { + limits: DirectoryLimits, + rooms: BTreeMap, + listings: BTreeMap, + relocated: Vec, + admitted_total: u64, + refused_total: u64, + forgotten_total: u64, + unreadable_lists_total: u64, + incomplete_lists_total: u64, + withdrawn_total: u64, +} + +pub const DIRECTORY_ENTRY_BYTES: u64 = + (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; + +impl RoomDirectory { + pub fn tracked_bytes(&self) -> u64 { + self.rooms + .values() + .map(|entry| { + let map = if entry.locations.is_empty() { + 0 + } else { + LOCATION_MAP_BYTES + }; + DIRECTORY_ENTRY_BYTES.saturating_add(map).saturating_add( + (entry.locations.len() as u64).saturating_mul(LOCATION_ENTRY_BYTES), + ) + }) + .fold(0u64, u64::saturating_add) + .saturating_add((self.listings.len() as u64).saturating_mul(LOCATION_ENTRY_BYTES)) + } + + pub const fn new(limits: DirectoryLimits) -> Self { + Self { + limits, + rooms: BTreeMap::new(), + listings: BTreeMap::new(), + relocated: Vec::new(), + admitted_total: 0, + refused_total: 0, + forgotten_total: 0, + unreadable_lists_total: 0, + incomplete_lists_total: 0, + withdrawn_total: 0, + } + } + + pub const fn limits(&self) -> DirectoryLimits { + self.limits + } + + pub fn len(&self) -> usize { + self.rooms.len() + } + + pub fn is_empty(&self) -> bool { + self.rooms.is_empty() + } + + pub const fn admitted_total(&self) -> u64 { + self.admitted_total + } + + pub const fn refused_total(&self) -> u64 { + self.refused_total + } + + pub const fn forgotten_total(&self) -> u64 { + self.forgotten_total + } + + pub const fn unreadable_lists_total(&self) -> u64 { + self.unreadable_lists_total + } + + pub const fn incomplete_lists_total(&self) -> u64 { + self.incomplete_lists_total + } + + pub const fn withdrawn_total(&self) -> u64 { + self.withdrawn_total + } + + pub fn rooms(&self) -> impl Iterator { + self.rooms + .iter() + .map(|(room, entry)| (*room, &entry.provenance)) + } + + pub fn room_keys(&self) -> Vec { + self.rooms.keys().copied().collect() + } + + pub fn contains(&self, room: &RoomKey) -> bool { + self.rooms.contains_key(room) + } + + pub fn provenance(&self, room: &RoomKey) -> Option<&RoomProvenance> { + self.rooms.get(room).map(|entry| &entry.provenance) + } + + pub fn known_only_to(&self, source: RoomSource) -> Vec { + self.rooms + .iter() + .filter(|(_, entry)| entry.provenance.only(source)) + .map(|(room, _)| *room) + .collect() + } + + pub fn gateway_only_rooms(&self) -> Vec { + self.rooms + .iter() + .filter(|(_, entry)| entry.provenance.is_gateway_only()) + .map(|(room, _)| *room) + .collect() + } + + pub fn media_only_rooms(&self) -> Vec { + self.rooms + .iter() + .filter(|(_, entry)| entry.provenance.is_media_only()) + .map(|(room, _)| *room) + .collect() + } + + pub fn listed_completely_at(&self, location: &Location) -> Option { + self.listings.get(location).copied() + } + + pub fn room_is_absent_from(&self, location: &Location, room: &RoomKey) -> bool { + self.listings.contains_key(location) + && !self + .rooms + .get(room) + .is_some_and(|entry| entry.locations.contains_key(location)) + } + + pub fn forget_location(&mut self, location: &Location) { + self.listings.remove(location); + } + + pub fn locations_for(&self, room: &RoomKey) -> Vec { + self.rooms + .get(room) + .map(|entry| entry.locations.keys().cloned().collect()) + .unwrap_or_default() + } + + pub fn pinned_for(&self, room: &RoomKey) -> Option { + self.rooms.get(room).and_then(|entry| entry.pinned.clone()) + } + + pub fn first_seen_at(&self, room: &RoomKey) -> Option { + self.rooms.get(room).map(|entry| entry.first_at) + } + + pub fn last_seen_at(&self, room: &RoomKey) -> Option { + self.rooms.get(room).map(|entry| entry.last_at) + } + + pub fn note(&mut self, room: RoomKey, source: RoomSource, now: Millis) -> Admission { + if let Some(entry) = self.rooms.get_mut(&room) { + entry.provenance.note(source, now); + entry.last_at = now; + return Admission::Refreshed; + } + if self.rooms.len() >= self.limits.max_rooms { + self.refused_total = self.refused_total.saturating_add(1); + return Admission::Refused; + } + let mut provenance = RoomProvenance::new(); + provenance.note(source, now); + self.rooms.insert( + room, + RoomEntry { + provenance, + locations: BTreeMap::new(), + pinned: None, + first_at: now, + last_at: now, + }, + ); + self.admitted_total = self.admitted_total.saturating_add(1); + Admission::Admitted + } + + pub fn note_census(&mut self, snapshot: &CensusSnapshot, now: Millis) -> Vec { + snapshot + .room_keys() + .into_iter() + .filter(|room| self.note(*room, RoomSource::GatewayCensus, now) == Admission::Admitted) + .collect() + } + + pub fn drain_relocated(&mut self) -> Vec { + let mut moved = std::mem::take(&mut self.relocated); + moved.sort_unstable(); + moved.dedup(); + moved + } + + pub fn note_pinned(&mut self, room: RoomKey, location: Location, now: Millis) -> Admission { + let admission = self.note(room, RoomSource::PinnedServer, now); + if admission.is_tracked() + && let Some(entry) = self.rooms.get_mut(&room) + { + entry.locations.insert(location.clone(), now); + entry.pinned = Some(location); + } + admission + } + + pub fn note_ledger(&mut self, room: RoomKey, now: Millis) -> Admission { + self.note(room, RoomSource::Ledger, now) + } + + pub fn note_suspicion(&mut self, room: RoomKey, now: Millis) -> Admission { + self.note(room, RoomSource::Suspicion, now) + } + + pub fn apply_room_list( + &mut self, + location: &Location, + list: &ServerRoomList, + now: Millis, + ) -> RoomListEffect { + let (rooms, unparseable) = match list { + ServerRoomList::Unreadable(fault) => { + self.unreadable_lists_total = self.unreadable_lists_total.saturating_add(1); + self.listings.remove(location); + return RoomListEffect::Unreadable(*fault); + } + ServerRoomList::Listed { rooms, unparseable } => (rooms, *unparseable), + }; + + let listed: BTreeSet = rooms.iter().copied().collect(); + let mut added = 0usize; + for room in &listed { + if self.note(*room, RoomSource::MediaRoomList, now) == Admission::Admitted { + added = added.saturating_add(1); + } + if let Some(entry) = self.rooms.get_mut(room) { + let elsewhere = + !entry.locations.is_empty() && !entry.locations.contains_key(location); + entry.locations.insert(location.clone(), now); + if elsewhere { + self.relocated.push(*room); + } + } + } + + if unparseable > 0 { + self.incomplete_lists_total = self.incomplete_lists_total.saturating_add(1); + self.listings.remove(location); + return RoomListEffect::AdditiveOnly { + listed: listed.len(), + added, + unparseable, + }; + } + + let mut withdrawn = 0usize; + for (room, entry) in &mut self.rooms { + if listed.contains(room) { + continue; + } + if entry.pinned.as_ref() == Some(location) { + continue; + } + if entry.locations.remove(location).is_some() { + withdrawn = withdrawn.saturating_add(1); + } + } + self.withdrawn_total = self.withdrawn_total.saturating_add(withdrawn as u64); + self.listings.insert(location.clone(), now); + + RoomListEffect::Applied { + listed: listed.len(), + added, + withdrawn, + } + } + + pub fn apply_pass( + &mut self, + results: &[(Location, ServerRoomList)], + now: Millis, + ) -> DiscoveryPass { + let mut pass = DiscoveryPass::empty(now); + pass.servers = results.len(); + + for (location, list) in results { + match self.apply_room_list(location, list, now) { + RoomListEffect::Unreadable(fault) => { + pass.unreadable.push((location.clone(), fault)); + } + RoomListEffect::AdditiveOnly { added, .. } => { + pass.readable = pass.readable.saturating_add(1); + pass.rooms_added = pass.rooms_added.saturating_add(added); + } + RoomListEffect::Applied { + added, withdrawn, .. + } => { + pass.readable = pass.readable.saturating_add(1); + pass.complete = pass.complete.saturating_add(1); + pass.rooms_added = pass.rooms_added.saturating_add(added); + pass.locations_withdrawn = pass.locations_withdrawn.saturating_add(withdrawn); + } + } + } + + pass + } + + pub fn sources_for( + &self, + room: &RoomKey, + gateway_hints: Vec, + ledger_last_known: Vec, + ) -> CandidateSources { + CandidateSources { + discovered: self.locations_for(room), + gateway_hints, + pinned: self.pinned_for(room), + ledger_last_known, + } + } + + pub fn forget(&mut self, room: &RoomKey) -> bool { + let removed = self.rooms.remove(room).is_some(); + if removed { + self.forgotten_total = self.forgotten_total.saturating_add(1); + } + removed + } + + pub fn retain(&mut self, mut keep: F) + where + F: FnMut(&RoomKey, &RoomProvenance) -> bool, + { + let dropped: Vec = self + .rooms + .iter() + .filter(|(room, entry)| !keep(room, &entry.provenance)) + .map(|(room, _)| *room) + .collect(); + for room in dropped { + self.forget(&room); + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReadClock { + pub now: Millis, + pub wall_now: WallMillis, +} + +impl ReadClock { + pub fn from_clock(clock: &dyn Clock) -> Self { + Self { + now: clock.now(), + wall_now: clock.wall_now(), + } + } + + pub const fn join_instant(&self, joined_at_unix_seconds: u64) -> Millis { + if joined_at_unix_seconds == 0 { + return self.now; + } + let joined_wall = WallMillis::new(joined_at_unix_seconds.saturating_mul(1_000)); + let age_ms = self.wall_now.saturating_since(joined_wall); + Millis::new(self.now.get().saturating_sub(age_ms)) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RoomRoster { + participants: Vec, + unparseable_identities: usize, +} + +impl RoomRoster { + pub fn participants(&self) -> &[MediaParticipant] { + &self.participants + } + + pub const fn unparseable_identities(&self) -> usize { + self.unparseable_identities + } + + pub fn len(&self) -> usize { + self.participants.len() + } + + pub fn is_empty(&self) -> bool { + self.participants.is_empty() + } + + pub fn connections(&self) -> Vec { + let mut connections: Vec = self + .participants + .iter() + .map(|participant| participant.connection.clone()) + .collect(); + connections.sort(); + connections.dedup(); + connections + } + + fn digest(&self) -> u64 { + let mut digest = Digest::new().number(self.unparseable_identities as u64); + for participant in &self.participants { + digest = digest + .text(participant.connection.as_str()) + .number(participant.user_id.get()) + .number(participant.joined_at.get()) + .flag(participant.is_publisher); + } + digest.finish() + } +} + +pub fn roster_from_records( + records: &[ParticipantRecord], + location: &Location, + clock: ReadClock, +) -> RoomRoster { + let mut participants: Vec = Vec::new(); + let mut unparseable_identities = 0usize; + + for record in records { + match parse_participant_identity(&record.identity) { + Err(_) => unparseable_identities = unparseable_identities.saturating_add(1), + Ok(identity) => participants.push(MediaParticipant { + connection: identity.connection_id, + user_id: identity.user_id, + location: location.clone(), + joined_at: clock.join_instant(record.joined_at_unix_seconds), + is_publisher: record.is_publisher, + }), + } + } + + RoomRoster { + participants, + unparseable_identities, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocationReadout { + location: Location, + roster: ReadResult, + at: Millis, +} + +impl LocationReadout { + pub const fn read(location: Location, roster: RoomRoster, at: Millis) -> Self { + Self { + location, + roster: ReadResult::Read(roster), + at, + } + } + + pub const fn unreadable(location: Location, fault: LiveKitFault, at: Millis) -> Self { + Self { + location, + roster: ReadResult::Unreadable(fault), + at, + } + } + + pub const fn location(&self) -> &Location { + &self.location + } + + pub const fn at(&self) -> Millis { + self.at + } + + pub fn is_readable(&self) -> bool { + self.roster.is_readable() + } + + pub fn roster(&self) -> Option<&RoomRoster> { + self.roster.as_read() + } + + pub fn fault(&self) -> Option { + self.roster.fault() + } + + pub fn participants_seen(&self) -> u32 { + self.roster + .as_read() + .map_or(0, |roster| u32::try_from(roster.len()).unwrap_or(u32::MAX)) + } + + pub fn outcome(&self) -> ProbeOutcome { + match &self.roster { + ReadResult::Read(_) => ProbeOutcome::Success, + ReadResult::Unreadable(fault) if fault.is_auth_failure() => ProbeOutcome::AuthFailure, + ReadResult::Unreadable(_) => ProbeOutcome::Failure, + } + } + + pub fn probe(&self, health: ServerHealth, cliffed: bool) -> LocationProbe { + let result = match &self.roster { + ReadResult::Unreadable(fault) => ProbeResult::Failed(*fault), + ReadResult::Read(roster) => ProbeResult::Ok { + participants: roster.participants.clone(), + unparseable_identities: roster.unparseable_identities, + }, + }; + LocationProbe { + location: self.location.clone(), + result, + health, + cliffed, + } + } + + fn digest(&self) -> u64 { + let digest = Digest::new() + .text(self.location.region.as_str()) + .text(self.location.server.as_str()); + match &self.roster { + ReadResult::Unreadable(fault) => digest.text("unreadable").text(fault.label()).finish(), + ReadResult::Read(roster) => digest.text("read").number(roster.digest()).finish(), + } + } +} + +pub fn record_health(readouts: &[LocationReadout], health: &mut ServerHealthMap, now: Millis) { + for readout in readouts { + health.record(&readout.location, readout.outcome(), now); + } +} + +fn digest_gateway_read(read: &GatewayRead) -> u64 { + match read { + GatewayRead::Failed(fault) => Digest::new().text("failed").text(fault.label()).finish(), + GatewayRead::Ok { states } => { + let mut digest = Digest::new().text("ok"); + for state in states { + digest = digest + .text(state.connection.as_ref().map_or("", ConnectionId::as_str)) + .flag(state.connection.is_some()) + .number(state.user_id.get()) + .number(state.channel_id.get()); + } + digest.finish() + } + } +} + +fn digest_pending_joins(joins: &[PendingJoin]) -> u64 { + let mut digest = Digest::new().text("pending_joins"); + for join in joins { + digest = digest + .text(join.connection.as_str()) + .number(join.user_id.get()) + .number(join.expires_at.get()); + } + digest.finish() +} + +pub async fn list_server_rooms(livekit: &L, location: &Location) -> ServerRoomList +where + L: LiveKitApi, +{ + match livekit.list_rooms(location).await { + ReadResult::Unreadable(fault) => ServerRoomList::Unreadable(fault), + ReadResult::Read(names) => { + let mut rooms: Vec = Vec::new(); + let mut unparseable = 0usize; + for name in &names { + match parse_room_name(name) { + Err(_) => unparseable = unparseable.saturating_add(1), + Ok(room) => rooms.push(room), + } + } + rooms.sort(); + rooms.dedup(); + ServerRoomList::Listed { rooms, unparseable } + } + } +} + +pub async fn list_fleet_rooms( + livekit: &L, + locations: &[Location], + mut admit: F, +) -> Vec<(Location, ServerRoomList)> +where + L: LiveKitApi, + F: FnMut(&Location) -> bool, +{ + let mut results: Vec<(Location, ServerRoomList)> = Vec::new(); + for location in locations { + if !admit(location) { + continue; + } + let list = list_server_rooms(livekit, location).await; + results.push((location.clone(), list)); + } + results +} + +pub async fn read_location_roster( + livekit: &L, + location: &Location, + room: RoomKey, + clock: ReadClock, +) -> LocationReadout +where + L: LiveKitApi, +{ + match livekit.list_participants(location, room).await { + ReadResult::Unreadable(fault) => { + LocationReadout::unreadable(location.clone(), fault, clock.now) + } + ReadResult::Read(records) => LocationReadout::read( + location.clone(), + roster_from_records(&records, location, clock), + clock.now, + ), + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CliffVerdict { + pub fraction: bool, + pub disputed: bool, +} + +impl CliffVerdict { + pub const fn is_cliffed(self) -> bool { + self.fraction || self.disputed + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServerPass { + pub location: Location, + pub participants_seen: u32, + pub expected_participants: u32, + pub expectations_accounted: bool, + pub coverage_complete: bool, +} + +impl ServerPass { + pub const fn new( + location: Location, + participants_seen: u32, + expected_participants: u32, + ) -> Self { + Self { + location, + participants_seen, + expected_participants, + expectations_accounted: false, + coverage_complete: true, + } + } + + #[must_use] + pub const fn accounted(mut self) -> Self { + self.expectations_accounted = true; + self + } + + pub const fn says_nobody_is_expected(&self) -> bool { + self.expectations_accounted && self.expected_participants == 0 + } + + #[must_use] + pub const fn partially_covered(mut self) -> Self { + self.coverage_complete = false; + self + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ExpectationSource { + View, + Ledger, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PassCounters { + seen: BTreeMap, + expected: BTreeMap, + seen_by_room: BTreeMap<(Location, RoomKey), u32>, + expected_by_room: BTreeMap<(Location, RoomKey, ExpectationSource), u32>, + expectations_accounted: bool, + skipped: BTreeSet, +} + +impl PassCounters { + pub const fn new() -> Self { + Self { + seen: BTreeMap::new(), + expected: BTreeMap::new(), + seen_by_room: BTreeMap::new(), + expected_by_room: BTreeMap::new(), + expectations_accounted: false, + skipped: BTreeSet::new(), + } + } + + pub const fn note_expectations_accounted(&mut self) { + self.expectations_accounted = true; + } + + pub const fn expectations_accounted(&self) -> bool { + self.expectations_accounted + } + + pub fn note_skipped(&mut self, location: &Location) { + self.skipped.insert(location.clone()); + } + + pub fn is_fully_covered(&self, location: &Location) -> bool { + !self.skipped.contains(location) + } + + pub fn note_seen(&mut self, location: &Location, count: u32) { + let entry = self.seen.entry(location.clone()).or_insert(0); + *entry = entry.saturating_add(count); + } + + pub fn note_expected(&mut self, location: &Location, count: u32) { + let entry = self.expected.entry(location.clone()).or_insert(0); + *entry = entry.saturating_add(count); + } + + pub fn note_room_seen(&mut self, location: &Location, room: RoomKey, count: u32) { + let previous = self + .seen_by_room + .insert((location.clone(), room), count) + .unwrap_or(0); + let entry = self.seen.entry(location.clone()).or_insert(0); + *entry = entry.saturating_sub(previous).saturating_add(count); + } + + pub fn note_room_expected( + &mut self, + location: &Location, + room: RoomKey, + source: ExpectationSource, + count: u32, + ) { + let previous = self + .expected_by_room + .insert((location.clone(), room, source), count) + .unwrap_or(0); + let entry = self.expected.entry(location.clone()).or_insert(0); + *entry = entry.saturating_sub(previous).saturating_add(count); + } + + pub fn note_view(&mut self, view: &RoomView) { + let room = view.observation().room; + let readable = view.readable_locations(); + for readout in view.readouts() { + if !readout.is_readable() { + continue; + } + self.note_room_seen(readout.location(), room, readout.participants_seen()); + } + for location in view.observation().candidates.locations() { + if !readable.contains(location) { + self.note_skipped(location); + } + } + for (location, expected) in view.expectations() { + self.note_room_expected(location, room, ExpectationSource::View, *expected); + } + } + + pub fn seen(&self, location: &Location) -> u32 { + self.seen.get(location).copied().unwrap_or(0) + } + + pub fn expected(&self, location: &Location) -> u32 { + self.expected.get(location).copied().unwrap_or(0) + } + + pub fn locations(&self) -> Vec { + let mut locations: Vec = self + .seen + .keys() + .chain(self.expected.keys()) + .cloned() + .collect(); + locations.sort(); + locations.dedup(); + locations + } + + pub fn passes(&self) -> Vec { + self.seen + .iter() + .map(|(location, participants_seen)| ServerPass { + location: location.clone(), + participants_seen: *participants_seen, + expected_participants: self.expected(location), + expectations_accounted: self.expectations_accounted, + coverage_complete: self.is_fully_covered(location), + }) + .collect() + } + + pub fn is_empty(&self) -> bool { + self.seen.is_empty() && self.expected.is_empty() + } + + pub fn clear(&mut self) { + self.seen.clear(); + self.expected.clear(); + self.seen_by_room.clear(); + self.expected_by_room.clear(); + self.expectations_accounted = false; + self.skipped.clear(); + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Watch { + detector: ServerCliffDetector, + baseline: u32, + last_nonempty_at: Option, + dispute_since: Option, + expected_at_dispute: u32, +} + +#[derive(Clone, Debug)] +pub struct ServerCliffWatch { + config: ServerCliffConfig, + watches: BTreeMap, + tripped_total: u64, + disputed_total: u64, + released_total: u64, +} + +impl ServerCliffWatch { + pub const fn new(config: ServerCliffConfig) -> Self { + Self { + config, + watches: BTreeMap::new(), + tripped_total: 0, + disputed_total: 0, + released_total: 0, + } + } + + pub fn from_config(config: &ReconConfig) -> Self { + Self::new(ServerCliffConfig { + fraction: config.server_cliff_fraction, + hold_ms: config.server_cliff_hold_ms, + }) + } + + pub const fn config(&self) -> ServerCliffConfig { + self.config + } + + pub const fn tripped_total(&self) -> u64 { + self.tripped_total + } + + pub const fn disputed_total(&self) -> u64 { + self.disputed_total + } + + pub const fn released_total(&self) -> u64 { + self.released_total + } + + pub fn tracked(&self) -> usize { + self.watches.len() + } + + pub fn observe(&mut self, pass: &ServerPass, now: Millis) -> CliffVerdict { + let config = self.config; + let watch = self.watches.entry(pass.location.clone()).or_default(); + + let fraction = if pass.coverage_complete { + watch.detector.observe(pass.participants_seen, now, config) + } else { + watch.detector.is_cliffed(now, config) + }; + let mut armed = false; + let mut released = false; + + if pass.participants_seen > 0 { + watch.baseline = pass.participants_seen; + watch.last_nonempty_at = Some(now); + watch.expected_at_dispute = 0; + released = watch.dispute_since.take().is_some(); + } else if watch.baseline > 0 || pass.expected_participants > 0 { + if watch.dispute_since.is_none() { + watch.dispute_since = Some(now); + armed = true; + } + watch.expected_at_dispute = pass.expected_participants; + if let Some(since) = watch.dispute_since + && pass.says_nobody_is_expected() + && now.saturating_since(since) >= config.hold_ms + { + watch.dispute_since = None; + watch.expected_at_dispute = 0; + released = true; + } + } + + if pass.coverage_complete + && pass.says_nobody_is_expected() + && let Some(since) = watch.detector.cliffed_since() + && now.saturating_since(since) >= config.hold_ms + { + watch.detector.release(); + } + + let disputed = watch.dispute_since.is_some(); + if armed { + self.disputed_total = self.disputed_total.saturating_add(1); + } + if released { + self.released_total = self.released_total.saturating_add(1); + } + if fraction { + self.tripped_total = self.tripped_total.saturating_add(1); + } + + CliffVerdict { fraction, disputed } + } + + pub fn observe_pass(&mut self, counters: &PassCounters, now: Millis) -> Vec { + counters + .passes() + .into_iter() + .filter_map(|pass| { + let verdict = self.observe(&pass, now); + verdict.is_cliffed().then_some(pass.location) + }) + .collect() + } + + pub fn is_cliffed(&self, location: &Location, now: Millis) -> bool { + self.watches.get(location).is_some_and(|watch| { + watch.dispute_since.is_some() || watch.detector.is_cliffed(now, self.config) + }) + } + + pub fn cliffed_since(&self, location: &Location) -> Option { + let watch = self.watches.get(location)?; + match (watch.dispute_since, watch.detector.cliffed_since()) { + (Some(dispute), Some(fraction)) => Some(dispute.min(fraction)), + (Some(dispute), None) => Some(dispute), + (None, fraction) => fraction, + } + } + + pub fn is_disputed(&self, location: &Location) -> bool { + self.watches + .get(location) + .is_some_and(|watch| watch.dispute_since.is_some()) + } + + pub fn disputed(&self) -> Vec { + self.watches + .iter() + .filter(|(_, watch)| watch.dispute_since.is_some()) + .map(|(location, _)| location.clone()) + .collect() + } + + pub fn cliffed(&self, now: Millis) -> Vec { + self.watches + .keys() + .filter(|location| self.is_cliffed(location, now)) + .cloned() + .collect() + } + + pub fn forget(&mut self, location: &Location) -> bool { + self.watches.remove(location).is_some() + } + + pub fn retain(&mut self, mut keep: F) + where + F: FnMut(&Location) -> bool, + { + self.watches.retain(|location, _| keep(location)); + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomTurnReads { + pub gateway: GatewayRead, + pub pending_joins: Vec, + pub pending_joins_read: bool, + pub gateway_calls: u32, + pub readouts: Vec, +} + +impl RoomTurnReads { + pub fn gateway_connections(&self) -> Vec { + let mut connections: Vec = match &self.gateway { + GatewayRead::Failed(_) => Vec::new(), + GatewayRead::Ok { states } => states + .iter() + .filter_map(|state| state.connection.clone()) + .collect(), + }; + connections.sort(); + connections.dedup(); + connections + } + + pub fn media_connections(&self) -> Vec { + let mut connections: Vec = self + .readouts + .iter() + .filter_map(LocationReadout::roster) + .flat_map(RoomRoster::connections) + .collect(); + connections.sort(); + connections.dedup(); + connections + } + + pub fn gateway_states(&self) -> &[GatewayVoiceState] { + match &self.gateway { + GatewayRead::Ok { states } => states, + GatewayRead::Failed(_) => &[], + } + } + + pub fn media_only_connections(&self) -> Vec { + let gateway = self.gateway_connections(); + self.media_connections() + .into_iter() + .filter(|connection| !gateway.contains(connection)) + .collect() + } +} + +#[derive(Clone, Copy, Debug)] +pub struct RoomContext<'a> { + pub room: RoomKey, + pub turn: TurnId, + pub at: Millis, + pub wall_at: WallMillis, + pub candidates: &'a CandidateSet, + pub health: &'a ServerHealthMap, + pub cliffs: &'a ServerCliffWatch, + pub holed: &'a [Location], + pub believed_homes: &'a [Location], + pub room_cliffed: bool, + pub census: CensusCrossCheck, + pub topology: TopologyFreshness, + pub census_epoch: Epoch, + pub topology_epoch: Epoch, + pub max_connections_per_room: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomView { + observation: RoomObservation, + gateway: GatewayRead, + readouts: Vec, + probes: Vec, + expectations: BTreeMap, + pending_joins_read: bool, +} + +impl RoomView { + pub const fn observation(&self) -> &RoomObservation { + &self.observation + } + + pub const fn room(&self) -> RoomKey { + self.observation.room + } + + pub const fn turn(&self) -> TurnId { + self.observation.turn + } + + pub const fn at(&self) -> Millis { + self.observation.at + } + + pub const fn authority(&self) -> &Side { + &self.observation.authority + } + + pub const fn gateway_read(&self) -> &GatewayRead { + &self.gateway + } + + pub const fn gateway_readable(&self) -> bool { + matches!(self.gateway, GatewayRead::Ok { .. }) + } + + pub const fn pending_joins_read(&self) -> bool { + self.pending_joins_read + } + + pub fn readouts(&self) -> &[LocationReadout] { + &self.readouts + } + + pub fn probes(&self) -> &[LocationProbe] { + &self.probes + } + + pub const fn expectations(&self) -> &BTreeMap { + &self.expectations + } + + pub fn readable_locations(&self) -> Vec { + self.readouts + .iter() + .filter(|readout| readout.is_readable()) + .map(|readout| readout.location().clone()) + .collect() + } + + pub fn unreadable_locations(&self) -> Vec<(Location, LiveKitFault)> { + self.readouts + .iter() + .filter_map(|readout| { + readout + .fault() + .map(|fault| (readout.location().clone(), fault)) + }) + .collect() + } + + pub fn unprobed_candidates(&self) -> Vec { + self.observation + .candidates + .locations() + .iter() + .filter(|location| { + !self + .readouts + .iter() + .any(|readout| readout.location() == *location) + }) + .cloned() + .collect() + } + + pub fn participants_seen(&self) -> u32 { + self.readouts + .iter() + .map(LocationReadout::participants_seen) + .fold(0u32, u32::saturating_add) + } + + pub fn locations_seen(&self) -> Vec { + let mut locations: Vec = self + .readouts + .iter() + .filter(|readout| readout.participants_seen() > 0) + .map(|readout| readout.location().clone()) + .collect(); + locations.sort(); + locations.dedup(); + locations + } +} + +fn expectations_from(reads: &RoomTurnReads) -> BTreeMap { + let media = reads.media_connections(); + let mut expectations: BTreeMap = BTreeMap::new(); + + for state in reads.gateway_states() { + let Some(connection) = state.connection.as_ref() else { + continue; + }; + if media.contains(connection) { + continue; + } + let Some(hint) = state.hint.as_ref() else { + continue; + }; + let entry = expectations.entry(hint.clone()).or_insert(0); + *entry = entry.saturating_add(1); + } + + expectations +} + +pub fn observe_room(reads: RoomTurnReads, context: &RoomContext<'_>) -> RoomView { + let probes: Vec = reads + .readouts + .iter() + .map(|readout| { + let location = readout.location(); + let cliffed = + context.cliffs.is_cliffed(location, context.at) || context.holed.contains(location); + readout.probe(context.health.health(location), cliffed) + }) + .collect(); + + let mut connections = reads.gateway_connections(); + connections.extend(reads.media_connections()); + connections.sort(); + connections.dedup(); + + let gateway_side = gateway_authority(&GatewayAuthorityInput { + read: &reads.gateway, + room_cliffed: context.room_cliffed, + census: context.census, + topology: context.topology, + }); + let media_side = match unread_believed_home(context.believed_homes, context.candidates, &probes) + { + Some(_) => SideAuthority::Unknown(UnknownReason::MediaLocationErrored( + LiveKitFault::ServerMissing, + )), + None => media_authority(&MediaAuthorityInput { + candidates: context.candidates, + probes: &probes, + connections_in_room: connections.len(), + max_connections_per_room: context.max_connections_per_room, + }), + }; + + let observation = RoomObservation::assemble( + &RoomReads { + room: context.room, + turn: context.turn, + at: context.at, + wall_at: context.wall_at, + gateway: &reads.gateway, + probes: &probes, + pending_joins: reads.pending_joins.clone(), + candidates: context.candidates.clone(), + census_epoch: context.census_epoch, + topology_epoch: context.topology_epoch, + }, + Side::new(gateway_side, media_side), + ); + + RoomView { + expectations: expectations_from(&reads), + observation, + gateway: reads.gateway, + readouts: reads.readouts, + probes, + pending_joins_read: reads.pending_joins_read, + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoomReadPlan<'a> { + pub room: RoomKey, + pub locations: &'a [Location], +} + +pub async fn read_room( + gateway: &G, + livekit: &L, + plan: &RoomReadPlan<'_>, + clock: &dyn Clock, + mut admit: F, + token: &mut TurnToken, +) -> Result, TurnError> +where + G: GatewayApi, + L: LiveKitApi, + F: FnMut(&Location) -> bool, +{ + let states = gateway.voice_states_for_channel(plan.room).await; + let gateway_read = match states { + Ok(states) => GatewayRead::Ok { states }, + Err(fault) => GatewayRead::Failed(fault), + }; + let fresh_gateway = token.read( + ReadSource::GatewayVoiceStates, + clock.now(), + digest_gateway_read(&gateway_read), + gateway_read, + ); + + let mut readouts: Vec = Vec::new(); + for location in plan.locations { + if !admit(location) { + continue; + } + let readout = + read_location_roster(livekit, location, plan.room, ReadClock::from_clock(clock)).await; + readouts.push(readout); + } + + let mut fresh_readouts = match readouts.first() { + None => token.read( + ReadSource::MediaParticipants, + clock.now(), + Digest::new().text("no_locations_read").finish(), + Vec::new(), + ), + Some(first) => token.read( + ReadSource::MediaParticipants, + first.at(), + first.digest(), + vec![first.clone()], + ), + }; + for readout in readouts.iter().skip(1) { + let next = token.read( + ReadSource::MediaParticipants, + readout.at(), + readout.digest(), + readout.clone(), + ); + fresh_readouts = fresh_readouts.zip(next)?.map(|(mut all, one)| { + all.push(one); + all + }); + } + + let reads = fresh_gateway + .zip(fresh_readouts)? + .map(|(gateway, readouts)| RoomTurnReads { + gateway, + pending_joins: Vec::new(), + pending_joins_read: false, + gateway_calls: 1, + readouts, + }); + + let gateway_readable = matches!(reads.peek().gateway, GatewayRead::Ok { .. }); + if !gateway_readable || reads.peek().media_only_connections().is_empty() { + return Ok(reads); + } + + let joins = gateway.pending_joins_for_channel(plan.room).await; + let at = clock.now(); + match joins { + Ok(joins) => { + let fresh_joins = token.read( + ReadSource::GatewayPendingJoins, + at, + digest_pending_joins(&joins), + joins, + ); + Ok(reads.zip(fresh_joins)?.map(|(mut reads, joins)| { + reads.pending_joins = joins; + reads.pending_joins_read = true; + reads.gateway_calls = 2; + reads + })) + } + Err(fault) => { + let fresh_fault = token.read( + ReadSource::GatewayPendingJoins, + at, + Digest::new().text("failed").text(fault.label()).finish(), + fault, + ); + Ok(reads.zip(fresh_fault)?.map(|(mut reads, fault)| { + reads.gateway = GatewayRead::Failed(fault); + reads.pending_joins = Vec::new(); + reads.pending_joins_read = false; + reads.gateway_calls = 2; + reads + })) + } + } +} diff --git a/fluxer_recon/src/evidence.rs b/fluxer_recon/src/evidence.rs new file mode 100644 index 000000000..d1c38b345 --- /dev/null +++ b/fluxer_recon/src/evidence.rs @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::gateway::{GatewayFault, Nonce}; +use crate::health::ServerHealth; +use crate::ids::{ + ChannelId, ConnectionId, Epoch, GuildId, Location, Millis, RoomKey, TurnId, UserId, WallMillis, +}; +use crate::livekit::LiveKitFault; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SideKind { + Gateway, + Media, +} + +impl SideKind { + pub const ALL: [Self; 2] = [Self::Gateway, Self::Media]; + + pub const fn label(self) -> &'static str { + match self { + Self::Gateway => "gateway", + Self::Media => "media", + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Side { + pub gateway: T, + pub media: T, +} + +impl Side { + pub const fn new(gateway: T, media: T) -> Self { + Self { gateway, media } + } + + pub const fn get(&self, kind: SideKind) -> &T { + match kind { + SideKind::Gateway => &self.gateway, + SideKind::Media => &self.media, + } + } + + pub fn map U>(&self, f: F) -> Side { + Side { + gateway: f(SideKind::Gateway, &self.gateway), + media: f(SideKind::Media, &self.media), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UnknownReason { + GatewayReadFailed(GatewayFault), + GatewayCliff, + CensusDisagrees, + MediaLocationErrored(LiveKitFault), + MediaServerUnhealthy, + MediaServerCliff, + NoCandidateLocations, + TopologyStale, + UnparseableIdentityInRoom, + RoomPartiallyUnreadable, +} + +impl UnknownReason { + pub const fn label(&self) -> &'static str { + match self { + Self::GatewayReadFailed(_) => "gateway_read_failed", + Self::GatewayCliff => "gateway_cliff", + Self::CensusDisagrees => "census_disagrees", + Self::MediaLocationErrored(_) => "media_location_errored", + Self::MediaServerUnhealthy => "media_server_unhealthy", + Self::MediaServerCliff => "media_server_cliff", + Self::NoCandidateLocations => "no_candidate_locations", + Self::TopologyStale => "topology_stale", + Self::UnparseableIdentityInRoom => "unparseable_identity_in_room", + Self::RoomPartiallyUnreadable => "room_partially_unreadable", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Presence { + Present, + Absent, + Unknown(UnknownReason), +} + +impl Presence { + pub const fn is_present(&self) -> bool { + matches!(self, Self::Present) + } + + pub const fn is_absent(&self) -> bool { + matches!(self, Self::Absent) + } + + pub const fn is_unknown(&self) -> bool { + matches!(self, Self::Unknown(_)) + } + + pub const fn unknown_reason(&self) -> Option<&UnknownReason> { + match self { + Self::Unknown(reason) => Some(reason), + Self::Present | Self::Absent => None, + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Present => "present", + Self::Absent => "absent", + Self::Unknown(_) => "unknown", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SideAuthority { + Authoritative, + Unknown(UnknownReason), +} + +impl SideAuthority { + pub const fn is_authoritative(&self) -> bool { + matches!(self, Self::Authoritative) + } + + pub const fn unknown_reason(&self) -> Option<&UnknownReason> { + match self { + Self::Authoritative => None, + Self::Unknown(reason) => Some(reason), + } + } + + pub fn project(&self, sighted: bool) -> Presence { + match self { + Self::Unknown(reason) => Presence::Unknown(reason.clone()), + Self::Authoritative => { + if sighted { + Presence::Present + } else { + Presence::Absent + } + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoomCliffInput { + pub dropped: usize, + pub remaining: usize, +} + +pub const fn room_cliff_trips(input: RoomCliffInput) -> bool { + input.dropped >= 2 || (input.dropped >= 1 && input.remaining == 0) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PriorConnection { + pub connection: ConnectionId, + pub was_consistent: bool, +} + +pub fn room_cliff_input( + prior: &[PriorConnection], + gateway_now: &[ConnectionId], + media_now: &[ConnectionId], +) -> RoomCliffInput { + let dropped = prior + .iter() + .filter(|entry| entry.was_consistent) + .filter(|entry| !gateway_now.contains(&entry.connection)) + .filter(|entry| media_now.contains(&entry.connection)) + .count(); + + RoomCliffInput { + dropped, + remaining: gateway_now.len(), + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CliffWindow { + since: Option, + last: Option, +} + +impl CliffWindow { + pub const fn clear() -> Self { + Self { + since: None, + last: None, + } + } + + pub const fn since(&self) -> Option { + self.since + } + + pub const fn trip(&mut self, now: Millis) { + if self.since.is_none() { + self.since = Some(now); + } + self.last = Some(now); + } + + pub const fn release(&mut self) { + self.since = None; + self.last = None; + } + + pub const fn is_held(&self, now: Millis, hold_ms: u64) -> bool { + match self.last { + None => false, + Some(last) => now.saturating_since(last) < hold_ms, + } + } + + pub const fn expire(&mut self, now: Millis, hold_ms: u64) { + if !self.is_held(now, hold_ms) { + self.release(); + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ServerCliffConfig { + pub fraction: f64, + pub hold_ms: u64, +} + +pub const SERVER_CLIFF_TRAILING_PASSES: usize = 5; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ServerCliffDetector { + trailing: [u32; SERVER_CLIFF_TRAILING_PASSES], + filled: usize, + cursor: usize, + window: CliffWindow, +} + +impl ServerCliffDetector { + pub const fn new() -> Self { + Self { + trailing: [0; SERVER_CLIFF_TRAILING_PASSES], + filled: 0, + cursor: 0, + window: CliffWindow::clear(), + } + } + + pub fn observe( + &mut self, + participants_seen: u32, + now: Millis, + config: ServerCliffConfig, + ) -> bool { + let trailing_max = self.trailing_max(); + let tripped = if trailing_max == 0 { + false + } else { + let drop = + f64::from(trailing_max.saturating_sub(participants_seen)) / f64::from(trailing_max); + drop > config.fraction + }; + + if tripped { + self.window.trip(now); + } else { + self.record(participants_seen); + self.window.expire(now, config.hold_ms); + } + + self.is_cliffed(now, config) + } + + pub const fn release(&mut self) { + self.trailing = [0; SERVER_CLIFF_TRAILING_PASSES]; + self.filled = 0; + self.cursor = 0; + self.window.release(); + } + + fn record(&mut self, participants_seen: u32) { + self.trailing[self.cursor] = participants_seen; + self.cursor = (self.cursor + 1) % SERVER_CLIFF_TRAILING_PASSES; + self.filled = self + .filled + .saturating_add(1) + .min(SERVER_CLIFF_TRAILING_PASSES); + } + + pub const fn is_cliffed(&self, now: Millis, config: ServerCliffConfig) -> bool { + self.window.is_held(now, config.hold_ms) + } + + pub const fn cliffed_since(&self) -> Option { + self.window.since() + } + + fn trailing_max(&self) -> u32 { + self.trailing + .iter() + .take(self.filled) + .copied() + .max() + .unwrap_or(0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GatewayRead { + Ok { states: Vec }, + Failed(GatewayFault), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayVoiceState { + pub connection: Option, + pub user_id: UserId, + pub channel_id: ChannelId, + pub guild_id: Option, + pub hint: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PendingJoin { + pub connection: ConnectionId, + pub user_id: UserId, + pub expires_at: WallMillis, + pub nonce: Nonce, +} + +impl PendingJoin { + pub const fn is_unexpired(&self, wall_now: WallMillis, skew_ms: u64) -> bool { + wall_now.get() <= self.expires_at.get().saturating_add(skew_ms) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CensusCrossCheck { + Fresh { voice_state_count: u32 }, + Stale, + Missing, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TopologyFreshness { + pub age_ms: u64, + pub max_age_ms: u64, +} + +impl TopologyFreshness { + pub const fn is_stale(self) -> bool { + self.age_ms > self.max_age_ms + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayAuthorityInput<'a> { + pub read: &'a GatewayRead, + pub room_cliffed: bool, + pub census: CensusCrossCheck, + pub topology: TopologyFreshness, +} + +pub fn gateway_authority(input: &GatewayAuthorityInput<'_>) -> SideAuthority { + let states = match input.read { + GatewayRead::Failed(fault) => { + return SideAuthority::Unknown(UnknownReason::GatewayReadFailed(fault.clone())); + } + GatewayRead::Ok { states } => states, + }; + + if input.room_cliffed { + return SideAuthority::Unknown(UnknownReason::GatewayCliff); + } + + if let CensusCrossCheck::Fresh { voice_state_count } = input.census + && voice_state_count > 0 + && states.is_empty() + { + return SideAuthority::Unknown(UnknownReason::CensusDisagrees); + } + + if input.topology.is_stale() { + return SideAuthority::Unknown(UnknownReason::TopologyStale); + } + + SideAuthority::Authoritative +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProbeResult { + Ok { + participants: Vec, + unparseable_identities: usize, + }, + Failed(LiveKitFault), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MediaParticipant { + pub connection: ConnectionId, + pub user_id: UserId, + pub location: Location, + pub joined_at: Millis, + pub is_publisher: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MediaSighting { + pub connection: ConnectionId, + pub user_id: UserId, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocationProbe { + pub location: Location, + pub result: ProbeResult, + pub health: ServerHealth, + pub cliffed: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CandidateSet { + locations: Vec, +} + +impl CandidateSet { + pub fn from_locations(mut locations: Vec) -> Self { + locations.sort(); + locations.dedup(); + Self { locations } + } + + pub fn locations(&self) -> &[Location] { + &self.locations + } + + pub fn is_empty(&self) -> bool { + self.locations.is_empty() + } + + pub fn len(&self) -> usize { + self.locations.len() + } + + pub fn contains(&self, location: &Location) -> bool { + self.locations.binary_search(location).is_ok() + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CandidateSources { + pub discovered: Vec, + pub gateway_hints: Vec, + pub pinned: Option, + pub ledger_last_known: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServerRecord { + pub location: Location, + pub removed_at: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TopologyLens<'a> { + pub servers: &'a [ServerRecord], + pub now: Millis, + pub drain_grace_ms: u64, +} + +impl TopologyLens<'_> { + pub fn live_locations(&self) -> Vec { + let mut locations: Vec = self + .servers + .iter() + .filter(|record| !self.is_drained(&record.location)) + .map(|record| record.location.clone()) + .collect(); + locations.sort(); + locations.dedup(); + locations + } + + fn is_drained(&self, location: &Location) -> bool { + self.servers + .iter() + .find(|record| &record.location == location) + .and_then(|record| record.removed_at) + .is_some_and(|removed_at| self.now.saturating_since(removed_at) > self.drain_grace_ms) + } +} + +pub fn candidate_set(sources: &CandidateSources, topology: &TopologyLens<'_>) -> CandidateSet { + let mut locations: Vec = sources + .discovered + .iter() + .chain(sources.gateway_hints.iter()) + .chain(sources.pinned.iter()) + .chain(sources.ledger_last_known.iter()) + .filter(|location| !topology.is_drained(location)) + .cloned() + .collect(); + + locations.sort(); + locations.dedup(); + CandidateSet { locations } +} + +pub fn unread_believed_home<'a>( + homes: &'a [Location], + candidates: &CandidateSet, + probes: &[LocationProbe], +) -> Option<&'a Location> { + homes.iter().find(|home| { + !candidates.contains(home) + || !probes.iter().any(|probe| { + &&probe.location == home && matches!(probe.result, ProbeResult::Ok { .. }) + }) + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MediaAuthorityInput<'a> { + pub candidates: &'a CandidateSet, + pub probes: &'a [LocationProbe], + pub connections_in_room: usize, + pub max_connections_per_room: usize, +} + +pub fn media_authority(input: &MediaAuthorityInput<'_>) -> SideAuthority { + if input.candidates.is_empty() { + return SideAuthority::Unknown(UnknownReason::NoCandidateLocations); + } + + for location in input.candidates.locations() { + let Some(probe) = input + .probes + .iter() + .find(|probe| &probe.location == location) + else { + return SideAuthority::Unknown(UnknownReason::MediaLocationErrored( + LiveKitFault::ServerMissing, + )); + }; + + match &probe.result { + ProbeResult::Failed(fault) => { + return SideAuthority::Unknown(UnknownReason::MediaLocationErrored(*fault)); + } + ProbeResult::Ok { + unparseable_identities, + participants: _, + } => { + if *unparseable_identities > 0 { + return SideAuthority::Unknown(UnknownReason::UnparseableIdentityInRoom); + } + } + } + + if !probe.health.is_healthy() { + return SideAuthority::Unknown(UnknownReason::MediaServerUnhealthy); + } + + if probe.cliffed { + return SideAuthority::Unknown(UnknownReason::MediaServerCliff); + } + } + + if input.connections_in_room > input.max_connections_per_room { + return SideAuthority::Unknown(UnknownReason::RoomPartiallyUnreadable); + } + + SideAuthority::Authoritative +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConnectionObservation { + pub connection: ConnectionId, + pub user_id: UserId, + pub presence: Side, + pub gateway_siblings: Vec, + pub media_locations: Vec, + pub participant_joined_at: Option, + pub gateway_state_has_connection_id: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomObservation { + pub room: RoomKey, + pub turn: TurnId, + pub at: Millis, + pub wall_at: WallMillis, + pub authority: Side, + pub connections: Vec, + pub pending_joins: Vec, + pub candidates: CandidateSet, + pub census_epoch: Epoch, + pub topology_epoch: Epoch, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomReads<'a> { + pub room: RoomKey, + pub turn: TurnId, + pub at: Millis, + pub wall_at: WallMillis, + pub gateway: &'a GatewayRead, + pub probes: &'a [LocationProbe], + pub pending_joins: Vec, + pub candidates: CandidateSet, + pub census_epoch: Epoch, + pub topology_epoch: Epoch, +} + +impl RoomObservation { + pub fn assemble(reads: &RoomReads<'_>, authority: Side) -> Self { + let gateway_states: &[GatewayVoiceState] = match reads.gateway { + GatewayRead::Ok { states } => states, + GatewayRead::Failed(_) => &[], + }; + + let participants: Vec<&MediaParticipant> = reads + .probes + .iter() + .filter_map(|probe| match &probe.result { + ProbeResult::Ok { participants, .. } => Some(participants.iter()), + ProbeResult::Failed(_) => None, + }) + .flatten() + .collect(); + + let mut keys: Vec<(ConnectionId, UserId)> = gateway_states + .iter() + .filter_map(|state| { + state + .connection + .as_ref() + .map(|connection| (connection.clone(), state.user_id)) + }) + .chain( + participants + .iter() + .map(|participant| (participant.connection.clone(), participant.user_id)), + ) + .collect(); + keys.sort(); + keys.dedup(); + + let connections = keys + .into_iter() + .map(|(connection, user_id)| { + let gateway_sighted = gateway_states + .iter() + .any(|state| state.connection.as_ref() == Some(&connection)); + + let media_sightings: Vec<&&MediaParticipant> = participants + .iter() + .filter(|participant| participant.connection == connection) + .collect(); + + let media_sighted = !media_sightings.is_empty(); + + let mut gateway_siblings: Vec = gateway_states + .iter() + .filter(|state| state.user_id == user_id) + .filter_map(|state| state.connection.clone()) + .collect(); + gateway_siblings.sort(); + gateway_siblings.dedup(); + + let mut media_locations: Vec = media_sightings + .iter() + .map(|participant| participant.location.clone()) + .collect(); + media_locations.sort(); + media_locations.dedup(); + + let participant_joined_at = media_sightings + .iter() + .map(|participant| participant.joined_at) + .min(); + + let gateway_state_has_connection_id = !gateway_states + .iter() + .any(|state| state.user_id == user_id && state.connection.is_none()); + + ConnectionObservation { + presence: Side::new( + authority.gateway.project(gateway_sighted), + authority.media.project(media_sighted), + ), + connection, + user_id, + gateway_siblings, + media_locations, + participant_joined_at, + gateway_state_has_connection_id, + } + }) + .collect(); + + Self { + room: reads.room, + turn: reads.turn, + at: reads.at, + wall_at: reads.wall_at, + authority, + connections, + pending_joins: reads.pending_joins.clone(), + candidates: reads.candidates.clone(), + census_epoch: reads.census_epoch, + topology_epoch: reads.topology_epoch, + } + } + + pub fn observation_for(&self, connection: &ConnectionId) -> Option<&ConnectionObservation> { + self.connections + .iter() + .find(|entry| &entry.connection == connection) + } + + pub fn unsighted_presence(&self) -> Side { + Side::new( + self.authority.gateway.project(false), + self.authority.media.project(false), + ) + } + + pub fn pending_join_for( + &self, + connection: &ConnectionId, + wall_now: WallMillis, + skew_ms: u64, + ) -> Option<&PendingJoin> { + self.pending_joins + .iter() + .find(|join| &join.connection == connection && join.is_unexpired(wall_now, skew_ms)) + } + + pub fn user_has_pending_join( + &self, + user_id: UserId, + wall_now: WallMillis, + skew_ms: u64, + ) -> bool { + self.pending_joins + .iter() + .any(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) + } +} diff --git a/fluxer_recon/src/gateway/codes.rs b/fluxer_recon/src/gateway/codes.rs new file mode 100644 index 000000000..631659385 --- /dev/null +++ b/fluxer_recon/src/gateway/codes.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Confidence { + Transient, + Definitive, + Refused, + Unknown, +} + +impl Confidence { + pub const ALL: [Self; 4] = [ + Self::Transient, + Self::Definitive, + Self::Refused, + Self::Unknown, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::Transient => "transient", + Self::Definitive => "definitive", + Self::Refused => "refused", + Self::Unknown => "unknown", + } + } + + pub const fn is_definitive(self) -> bool { + matches!(self, Self::Definitive) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum GatewayErrorCode { + Timeout, + NoResponders, + Overloaded, + GuildNotFound, + VoiceStatesError, + PendingJoinsError, + CallLookupError, + CallStateError, + CallPendingJoinsError, + DisconnectUserError, + ConfirmConnectionError, + ConnectionNotFound, + VoiceStateMismatch, + VoiceInvalidState, + VoiceNotSupported, + EventMutationsPaused, + InvalidParams, +} + +impl GatewayErrorCode { + pub const ALL: [Self; 17] = [ + Self::Timeout, + Self::NoResponders, + Self::Overloaded, + Self::GuildNotFound, + Self::VoiceStatesError, + Self::PendingJoinsError, + Self::CallLookupError, + Self::CallStateError, + Self::CallPendingJoinsError, + Self::DisconnectUserError, + Self::ConfirmConnectionError, + Self::ConnectionNotFound, + Self::VoiceStateMismatch, + Self::VoiceInvalidState, + Self::VoiceNotSupported, + Self::EventMutationsPaused, + Self::InvalidParams, + ]; + + pub const fn wire(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::NoResponders => "no_responders", + Self::Overloaded => "overloaded", + Self::GuildNotFound => "guild_not_found", + Self::VoiceStatesError => "voice_states_error", + Self::PendingJoinsError => "pending_joins_error", + Self::CallLookupError => "call_lookup_error", + Self::CallStateError => "call_state_error", + Self::CallPendingJoinsError => "call_pending_joins_error", + Self::DisconnectUserError => "disconnect_user_error", + Self::ConfirmConnectionError => "confirm_connection_error", + Self::ConnectionNotFound => "connection_not_found", + Self::VoiceStateMismatch => "voice_state_mismatch", + Self::VoiceInvalidState => "voice_invalid_state", + Self::VoiceNotSupported => "voice_not_supported", + Self::EventMutationsPaused => "event_mutations_paused", + Self::InvalidParams => "invalid_params", + } + } + + pub const fn confidence(self) -> Confidence { + match self { + Self::Timeout + | Self::NoResponders + | Self::Overloaded + | Self::GuildNotFound + | Self::VoiceStatesError + | Self::PendingJoinsError + | Self::CallLookupError + | Self::CallStateError + | Self::CallPendingJoinsError + | Self::DisconnectUserError + | Self::ConfirmConnectionError => Confidence::Transient, + Self::ConnectionNotFound + | Self::VoiceStateMismatch + | Self::VoiceInvalidState + | Self::VoiceNotSupported => Confidence::Definitive, + Self::EventMutationsPaused | Self::InvalidParams => Confidence::Refused, + } + } +} + +pub const MISSING_ERROR_LABEL: &str = "missing_error"; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum GatewayError { + Known(GatewayErrorCode), + Unrecognised(Box), + Missing, +} + +impl GatewayError { + pub fn from_wire(value: Option<&str>) -> Self { + match value { + None => Self::Missing, + Some(raw) => Self::from_code(raw), + } + } + + fn from_code(raw: &str) -> Self { + GatewayErrorCode::ALL + .into_iter() + .find(|code| code.wire() == raw) + .map_or_else(|| Self::Unrecognised(Box::from(raw)), Self::Known) + } + + pub const fn confidence(&self) -> Confidence { + match self { + Self::Known(code) => code.confidence(), + Self::Unrecognised(_) | Self::Missing => Confidence::Unknown, + } + } + + pub fn label(&self) -> &str { + match self { + Self::Known(code) => code.wire(), + Self::Unrecognised(raw) => raw, + Self::Missing => MISSING_ERROR_LABEL, + } + } + + pub const fn code(&self) -> Option { + match self { + Self::Known(code) => Some(*code), + Self::Unrecognised(_) | Self::Missing => None, + } + } +} diff --git a/fluxer_recon/src/gateway/mod.rs b/fluxer_recon/src/gateway/mod.rs new file mode 100644 index 000000000..323f4578f --- /dev/null +++ b/fluxer_recon/src/gateway/mod.rs @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub mod codes; +pub mod nats; + +use std::future::Future; +use std::time::Duration; + +use crate::evidence::{GatewayVoiceState, PendingJoin}; +use crate::gateway::codes::{Confidence, GatewayError}; +use crate::ids::{ConnectionId, RoomKey, UserId}; +use crate::ledger::RepairVerdict; + +pub const SUBJECT_PREFIX: &str = "rpc.gateway."; + +pub const MUTATION_DEADLINE: Duration = Duration::from_secs(4); +pub const CHANNEL_READ_DEADLINE: Duration = Duration::from_secs(5); +pub const CENSUS_DEADLINE: Duration = Duration::from_secs(12); + +pub const GATEWAY_GUILD_CALL_BUDGET: Duration = Duration::from_secs(4); +pub const GATEWAY_CENSUS_NODE_RPC_BUDGET: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum MethodClass { + Read, + Constructive, + Destructive, +} + +impl MethodClass { + pub const fn label(self) -> &'static str { + match self { + Self::Read => "read", + Self::Constructive => "constructive", + Self::Destructive => "destructive", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum GatewayMethod { + ActiveVoiceRooms, + VoiceStatesForChannel, + PendingJoinsForChannel, + ConfirmConnection, + RepairStateFromCache, + DisconnectUserIfInChannel, +} + +impl GatewayMethod { + pub const ALL: [Self; 6] = [ + Self::ActiveVoiceRooms, + Self::VoiceStatesForChannel, + Self::PendingJoinsForChannel, + Self::ConfirmConnection, + Self::RepairStateFromCache, + Self::DisconnectUserIfInChannel, + ]; + + pub const fn name(self) -> &'static str { + match self { + Self::ActiveVoiceRooms => "process.active_voice_rooms", + Self::VoiceStatesForChannel => "voice.get_voice_states_for_channel", + Self::PendingJoinsForChannel => "voice.get_pending_joins_for_channel", + Self::ConfirmConnection => "voice.confirm_connection", + Self::RepairStateFromCache => "voice.repair_state_from_cache", + Self::DisconnectUserIfInChannel => "voice.disconnect_user_if_in_channel", + } + } + + pub const fn deadline(self) -> Duration { + match self { + Self::ActiveVoiceRooms => CENSUS_DEADLINE, + Self::VoiceStatesForChannel | Self::PendingJoinsForChannel => CHANNEL_READ_DEADLINE, + Self::ConfirmConnection + | Self::RepairStateFromCache + | Self::DisconnectUserIfInChannel => MUTATION_DEADLINE, + } + } + + pub const fn class(self) -> MethodClass { + match self { + Self::ActiveVoiceRooms | Self::VoiceStatesForChannel | Self::PendingJoinsForChannel => { + MethodClass::Read + } + Self::ConfirmConnection | Self::RepairStateFromCache => MethodClass::Constructive, + Self::DisconnectUserIfInChannel => MethodClass::Destructive, + } + } + + pub fn subject(self) -> String { + let mut subject = String::with_capacity(SUBJECT_PREFIX.len() + self.name().len()); + subject.push_str(SUBJECT_PREFIX); + subject.push_str(self.name()); + subject + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum FaultOrigin { + CouldNotAsk, + Unintelligible, + GatewaySaidNo, +} + +impl FaultOrigin { + pub const fn label(self) -> &'static str { + match self { + Self::CouldNotAsk => "could_not_ask", + Self::Unintelligible => "unintelligible", + Self::GatewaySaidNo => "gateway_said_no", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum GatewayFault { + Timeout, + NoResponders, + TransportFailed, + DecodeFailed, + NotOk(GatewayError), +} + +impl GatewayFault { + pub const fn label(&self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::NoResponders => "no_responders", + Self::TransportFailed => "transport_failed", + Self::DecodeFailed => "decode_failed", + Self::NotOk(_) => "not_ok", + } + } + + pub const fn origin(&self) -> FaultOrigin { + match self { + Self::Timeout | Self::NoResponders | Self::TransportFailed => FaultOrigin::CouldNotAsk, + Self::DecodeFailed => FaultOrigin::Unintelligible, + Self::NotOk(_) => FaultOrigin::GatewaySaidNo, + } + } + + pub const fn is_evidence(&self) -> bool { + matches!(self.origin(), FaultOrigin::GatewaySaidNo) + } + + pub const fn confidence(&self) -> Confidence { + match self { + Self::Timeout | Self::NoResponders | Self::TransportFailed => Confidence::Transient, + Self::DecodeFailed => Confidence::Unknown, + Self::NotOk(error) => error.confidence(), + } + } + + pub const fn error(&self) -> Option<&GatewayError> { + match self { + Self::Timeout | Self::NoResponders | Self::TransportFailed | Self::DecodeFailed => None, + Self::NotOk(error) => Some(error), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub enum Nonce { + #[default] + Empty, + Value(Box), +} + +impl Nonce { + pub fn from_wire(value: Option<&str>) -> Self { + match value { + None => Self::Empty, + Some("") => Self::Empty, + Some(raw) => Self::Value(Box::from(raw)), + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Self::Empty => None, + Self::Value(value) => Some(value), + } + } + + pub const fn is_empty(&self) -> bool { + matches!(self, Self::Empty) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ActiveVoiceRoom { + pub room: RoomKey, + pub voice_state_count: u32, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ActiveVoiceRooms { + pub node_count: u32, + pub rooms: Vec, + pub unparseable_rooms: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConfirmOutcome { + Confirmed, + AlreadyConfirmed, + CallNotFound, + Failed(GatewayError), +} + +impl ConfirmOutcome { + pub const fn succeeded(&self) -> bool { + matches!(self, Self::Confirmed | Self::AlreadyConfirmed) + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Confirmed => "confirmed", + Self::AlreadyConfirmed => "already_confirmed", + Self::CallNotFound => "call_not_found", + Self::Failed(_) => "failed", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepairOutcome { + Repaired, + NoChange, + Failed(GatewayError), +} + +impl RepairOutcome { + pub const fn verdict(&self) -> RepairVerdict { + match self { + Self::Repaired => RepairVerdict::Repaired, + Self::NoChange => RepairVerdict::NoChange, + Self::Failed(error) => match error.confidence() { + Confidence::Definitive => RepairVerdict::NotRepairable, + Confidence::Transient | Confidence::Refused | Confidence::Unknown => { + RepairVerdict::NoChange + } + }, + } + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Repaired => "repaired", + Self::NoChange => "no_change", + Self::Failed(_) => "failed", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DisconnectOutcome { + Removed, + Ignored { reason: Option> }, + CallNotFound, + Failed(GatewayError), +} + +impl DisconnectOutcome { + pub const fn removed_state(&self) -> bool { + matches!(self, Self::Removed) + } + + pub const fn label(&self) -> &'static str { + match self { + Self::Removed => "removed", + Self::Ignored { .. } => "ignored", + Self::CallNotFound => "call_not_found", + Self::Failed(_) => "failed", + } + } +} + +pub trait GatewayApi { + fn active_voice_rooms( + &self, + ) -> impl Future> + Send; + + fn voice_states_for_channel( + &self, + room: RoomKey, + ) -> impl Future, GatewayFault>> + Send; + + fn pending_joins_for_channel( + &self, + room: RoomKey, + ) -> impl Future, GatewayFault>> + Send; + + fn confirm_connection( + &self, + room: RoomKey, + connection: &ConnectionId, + nonce: &Nonce, + ) -> impl Future> + Send; + + fn repair_state_from_cache( + &self, + room: RoomKey, + user_id: UserId, + connection: &ConnectionId, + ) -> impl Future> + Send; + + fn disconnect_user_if_in_channel( + &self, + room: RoomKey, + user_id: UserId, + connection: Option<&ConnectionId>, + ) -> impl Future> + Send; +} diff --git a/fluxer_recon/src/gateway/nats.rs b/fluxer_recon/src/gateway/nats.rs new file mode 100644 index 000000000..d225d5480 --- /dev/null +++ b/fluxer_recon/src/gateway/nats.rs @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use serde::Serialize; +use serde_json::Value; + +use fluxer_svc::transport::Transport; + +use crate::evidence::{GatewayVoiceState, PendingJoin}; +use crate::gateway::codes::GatewayError; +use crate::gateway::{ + ActiveVoiceRoom, ActiveVoiceRooms, ConfirmOutcome, DisconnectOutcome, GatewayApi, GatewayFault, + GatewayMethod, Nonce, RepairOutcome, +}; +use crate::ids::{ + ChannelId, ConnectionId, GuildId, Location, RegionId, RoomKey, ServerId, UserId, WallMillis, +}; + +#[derive(Debug, Serialize)] +struct NoParams {} + +#[derive(Debug, Serialize)] +struct ChannelReadParams { + channel_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + guild_id: Option, +} + +#[derive(Debug, Serialize)] +struct ConfirmParams { + channel_id: String, + connection_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + guild_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + token_nonce: Option, +} + +#[derive(Debug, Serialize)] +struct RepairParams { + channel_id: String, + user_id: String, + connection_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + guild_id: Option, +} + +#[derive(Debug, Serialize)] +struct DisconnectParams { + channel_id: String, + user_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + guild_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + connection_id: Option, +} + +fn guild_param(room: RoomKey) -> Option { + room.guild_id().map(|guild_id| guild_id.to_string()) +} + +fn channel_read_params(room: RoomKey) -> ChannelReadParams { + ChannelReadParams { + channel_id: room.channel_id().to_string(), + guild_id: guild_param(room), + } +} + +fn confirm_params(room: RoomKey, connection: &ConnectionId, nonce: &Nonce) -> ConfirmParams { + ConfirmParams { + channel_id: room.channel_id().to_string(), + connection_id: connection.as_str().to_owned(), + guild_id: guild_param(room), + token_nonce: nonce.as_str().map(str::to_owned), + } +} + +fn repair_params(room: RoomKey, user_id: UserId, connection: &ConnectionId) -> RepairParams { + RepairParams { + channel_id: room.channel_id().to_string(), + user_id: user_id.to_string(), + connection_id: connection.as_str().to_owned(), + guild_id: guild_param(room), + } +} + +fn disconnect_params( + room: RoomKey, + user_id: UserId, + connection: Option<&ConnectionId>, +) -> DisconnectParams { + DisconnectParams { + channel_id: room.channel_id().to_string(), + user_id: user_id.to_string(), + guild_id: guild_param(room), + connection_id: connection.map(|id| id.as_str().to_owned()), + } +} + +pub fn classify_transport_error(error: &anyhow::Error) -> GatewayFault { + if error.chain().any(|cause| { + cause + .to_string() + .to_ascii_lowercase() + .contains("no responders") + }) { + return GatewayFault::NoResponders; + } + if error + .chain() + .any(|cause| cause.is::()) + { + return GatewayFault::Timeout; + } + if error.chain().any(|cause| { + let text = cause.to_string().to_ascii_lowercase(); + text.contains("timed out") || text.contains("timeout") || text.contains("deadline") + }) { + return GatewayFault::Timeout; + } + GatewayFault::TransportFailed +} + +pub fn decode_envelope(bytes: &[u8]) -> Result { + let Ok(body) = serde_json::from_slice::(bytes) else { + return Err(GatewayFault::DecodeFailed); + }; + let Some(object) = body.as_object() else { + return Err(GatewayFault::DecodeFailed); + }; + let Some(ok) = object.get("ok").and_then(Value::as_bool) else { + return Err(GatewayFault::DecodeFailed); + }; + if ok { + return Ok(object.get("result").cloned().unwrap_or(Value::Null)); + } + let error = object.get("error").and_then(Value::as_str); + Err(GatewayFault::NotOk(GatewayError::from_wire(error))) +} + +fn required<'a>(result: &'a Value, key: &str) -> Result<&'a Value, GatewayFault> { + result.get(key).ok_or(GatewayFault::DecodeFailed) +} + +fn optional<'a>(result: &'a Value, key: &str) -> Option<&'a Value> { + match result.get(key) { + None | Some(Value::Null) => None, + Some(value) => Some(value), + } +} + +fn entries(result: &Value, key: &str) -> Result, GatewayFault> { + required(result, key)? + .as_array() + .cloned() + .ok_or(GatewayFault::DecodeFailed) +} + +fn decimal_u64(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse::().ok() +} + +fn snowflake(value: &Value) -> Result { + match value { + Value::String(raw) => decimal_u64(raw).ok_or(GatewayFault::DecodeFailed), + Value::Number(number) => number.as_u64().ok_or(GatewayFault::DecodeFailed), + Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => { + Err(GatewayFault::DecodeFailed) + } + } +} + +fn text(value: &Value) -> Result<&str, GatewayFault> { + value.as_str().ok_or(GatewayFault::DecodeFailed) +} + +fn count(value: &Value) -> Result { + let number = value.as_u64().ok_or(GatewayFault::DecodeFailed)?; + u32::try_from(number).map_err(|_| GatewayFault::DecodeFailed) +} + +fn flag(result: &Value, key: &str) -> Result { + match optional(result, key) { + None => Ok(false), + Some(value) => value.as_bool().ok_or(GatewayFault::DecodeFailed), + } +} + +fn reason(result: &Value) -> Result>, GatewayFault> { + match optional(result, "reason") { + None => Ok(None), + Some(value) => Ok(Some(Box::from(text(value)?))), + } +} + +fn error_of(result: &Value) -> GatewayError { + GatewayError::from_wire(optional(result, "error").and_then(Value::as_str)) +} + +fn connection_of(value: &Value) -> Result, GatewayFault> { + let raw = text(value)?; + if raw.is_empty() { + return Ok(None); + } + ConnectionId::new(raw) + .map(Some) + .map_err(|_| GatewayFault::DecodeFailed) +} + +fn location_hint(entry: &Value) -> Result, GatewayFault> { + let region = match optional(entry, "region_id") { + None => None, + Some(value) => Some(RegionId::new(text(value)?).map_err(|_| GatewayFault::DecodeFailed)?), + }; + let server = match optional(entry, "server_id") { + None => None, + Some(value) => Some(ServerId::new(text(value)?).map_err(|_| GatewayFault::DecodeFailed)?), + }; + match (region, server) { + (Some(region), Some(server)) => Ok(Some(Location::new(region, server))), + (Some(_), None) | (None, Some(_)) | (None, None) => Ok(None), + } +} + +fn room_of(entry: &Value) -> Result { + let channel_id = ChannelId::new(snowflake(required(entry, "channel_id")?)?); + match optional(entry, "guild_id") { + None => Ok(RoomKey::Dm { channel_id }), + Some(value) => Ok(RoomKey::Guild { + guild_id: GuildId::new(snowflake(value)?), + channel_id, + }), + } +} + +fn decode_active_voice_room(entry: &Value) -> Result { + Ok(ActiveVoiceRoom { + room: room_of(entry)?, + voice_state_count: count(required(entry, "voice_state_count")?)?, + }) +} + +pub fn decode_active_voice_rooms(result: &Value) -> Result { + let node_count = count(required(result, "node_count")?)?; + let mut rooms = Vec::new(); + let mut unparseable_rooms = 0u32; + + for entry in entries(result, "rooms")? { + if let Ok(room) = decode_active_voice_room(&entry) { + rooms.push(room); + } else { + unparseable_rooms = unparseable_rooms.saturating_add(1); + } + } + + Ok(ActiveVoiceRooms { + node_count, + rooms, + unparseable_rooms, + }) +} + +fn decode_voice_state(entry: &Value) -> Result { + let connection = match optional(entry, "connection_id") { + None => None, + Some(value) => connection_of(value)?, + }; + let guild_id = match optional(entry, "guild_id") { + None => None, + Some(value) => Some(GuildId::new(snowflake(value)?)), + }; + + Ok(GatewayVoiceState { + connection, + user_id: UserId::new(snowflake(required(entry, "user_id")?)?), + channel_id: ChannelId::new(snowflake(required(entry, "channel_id")?)?), + guild_id, + hint: location_hint(entry)?, + }) +} + +pub fn decode_voice_states(result: &Value) -> Result, GatewayFault> { + entries(result, "voice_states")? + .iter() + .map(decode_voice_state) + .collect() +} + +fn decode_pending_join(entry: &Value) -> Result { + let connection = ConnectionId::new(text(required(entry, "connection_id")?)?) + .map_err(|_| GatewayFault::DecodeFailed)?; + let expires_at = required(entry, "expires_at")? + .as_i64() + .ok_or(GatewayFault::DecodeFailed)?; + let nonce = match optional(entry, "token_nonce") { + None => Nonce::Empty, + Some(value) => Nonce::from_wire(Some(text(value)?)), + }; + + Ok(PendingJoin { + connection, + user_id: UserId::new(snowflake(required(entry, "user_id")?)?), + expires_at: WallMillis::new(u64::try_from(expires_at).unwrap_or(0)), + nonce, + }) +} + +pub fn decode_pending_joins(result: &Value) -> Result, GatewayFault> { + entries(result, "pending_joins")? + .iter() + .map(decode_pending_join) + .collect() +} + +pub fn decode_confirm(result: &Value) -> Result { + if !flag(result, "success")? { + return Ok(ConfirmOutcome::Failed(error_of(result))); + } + if flag(result, "call_not_found")? { + return Ok(ConfirmOutcome::CallNotFound); + } + if flag(result, "already_confirmed")? { + return Ok(ConfirmOutcome::AlreadyConfirmed); + } + Ok(ConfirmOutcome::Confirmed) +} + +pub fn decode_repair(result: &Value) -> Result { + if !flag(result, "success")? { + return Ok(RepairOutcome::Failed(error_of(result))); + } + match optional(result, "repaired") { + None => Ok(RepairOutcome::NoChange), + Some(value) => match value.as_bool().ok_or(GatewayFault::DecodeFailed)? { + true => Ok(RepairOutcome::Repaired), + false => Ok(RepairOutcome::NoChange), + }, + } +} + +pub fn decode_disconnect(result: &Value) -> Result { + if !flag(result, "success")? { + return Ok(DisconnectOutcome::Failed(error_of(result))); + } + if flag(result, "ignored")? { + return Ok(DisconnectOutcome::Ignored { + reason: reason(result)?, + }); + } + if flag(result, "call_not_found")? { + return Ok(DisconnectOutcome::CallNotFound); + } + Ok(DisconnectOutcome::Removed) +} + +pub struct NatsGateway { + transport: T, +} + +impl NatsGateway { + pub const fn new(transport: T) -> Self { + Self { transport } + } + + pub const fn transport(&self) -> &T { + &self.transport + } +} + +impl NatsGateway { + async fn call( + &self, + method: GatewayMethod, + params: &P, + ) -> Result { + let Ok(payload) = serde_json::to_vec(params) else { + return Err(GatewayFault::TransportFailed); + }; + let bytes = self + .transport + .request(&method.subject(), &payload, method.deadline()) + .await + .map_err(|error| classify_transport_error(&error))?; + decode_envelope(&bytes) + } +} + +impl GatewayApi for NatsGateway { + async fn active_voice_rooms(&self) -> Result { + let result = self + .call(GatewayMethod::ActiveVoiceRooms, &NoParams {}) + .await?; + decode_active_voice_rooms(&result) + } + + async fn voice_states_for_channel( + &self, + room: RoomKey, + ) -> Result, GatewayFault> { + let result = self + .call( + GatewayMethod::VoiceStatesForChannel, + &channel_read_params(room), + ) + .await?; + decode_voice_states(&result) + } + + async fn pending_joins_for_channel( + &self, + room: RoomKey, + ) -> Result, GatewayFault> { + let result = self + .call( + GatewayMethod::PendingJoinsForChannel, + &channel_read_params(room), + ) + .await?; + decode_pending_joins(&result) + } + + async fn confirm_connection( + &self, + room: RoomKey, + connection: &ConnectionId, + nonce: &Nonce, + ) -> Result { + let result = self + .call( + GatewayMethod::ConfirmConnection, + &confirm_params(room, connection, nonce), + ) + .await?; + decode_confirm(&result) + } + + async fn repair_state_from_cache( + &self, + room: RoomKey, + user_id: UserId, + connection: &ConnectionId, + ) -> Result { + let result = self + .call( + GatewayMethod::RepairStateFromCache, + &repair_params(room, user_id, connection), + ) + .await?; + decode_repair(&result) + } + + async fn disconnect_user_if_in_channel( + &self, + room: RoomKey, + user_id: UserId, + connection: Option<&ConnectionId>, + ) -> Result { + let result = self + .call( + GatewayMethod::DisconnectUserIfInChannel, + &disconnect_params(room, user_id, connection), + ) + .await?; + decode_disconnect(&result) + } +} diff --git a/fluxer_recon/src/guards.rs b/fluxer_recon/src/guards.rs new file mode 100644 index 000000000..15fb0cd47 --- /dev/null +++ b/fluxer_recon/src/guards.rs @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::config::ConnectionIdGuard; +use crate::evidence::{MediaSighting, PendingJoin}; +use crate::health::ServerHealth; +use crate::ids::{ConnectionId, Location, Millis, RoomKey, TurnId, UserId, WallMillis}; +use crate::ledger::RepairVerdict; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum AbortReason { + SiblingConnection, + NoConnectionId, + PendingJoin, + PreflightStale, + PreflightDisagreed, + JoinedAfterDivergence, + UnknownGateway, + UnknownMedia, + ServerUnhealthy, + Budget, + Mode, + Warmup, + DmPosture, + Breaker, +} + +impl AbortReason { + pub const ALL: [Self; 14] = [ + Self::SiblingConnection, + Self::NoConnectionId, + Self::PendingJoin, + Self::PreflightStale, + Self::PreflightDisagreed, + Self::JoinedAfterDivergence, + Self::UnknownGateway, + Self::UnknownMedia, + Self::ServerUnhealthy, + Self::Budget, + Self::Mode, + Self::Warmup, + Self::DmPosture, + Self::Breaker, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::SiblingConnection => "sibling_connection", + Self::NoConnectionId => "no_connection_id", + Self::PendingJoin => "pending_join", + Self::PreflightStale => "preflight_stale", + Self::PreflightDisagreed => "preflight_disagreed", + Self::JoinedAfterDivergence => "joined_after_divergence", + Self::UnknownGateway => "unknown_gateway", + Self::UnknownMedia => "unknown_media", + Self::ServerUnhealthy => "server_unhealthy", + Self::Budget => "budget", + Self::Mode => "mode", + Self::Warmup => "warmup", + Self::DmPosture => "dm_posture", + Self::Breaker => "breaker", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreflightVoiceState { + pub user_id: UserId, + pub connection: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayPreflight { + pub turn: TurnId, + pub taken_at: Millis, + pub voice_states: Vec, + pub pending_joins: Vec, + pub media_present: Vec, +} + +impl GatewayPreflight { + pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { + now.saturating_since(self.taken_at) <= max_age_ms + } + + pub fn states_for(&self, user_id: UserId) -> impl Iterator { + self.voice_states + .iter() + .filter(move |state| state.user_id == user_id) + } + + pub fn user_has_pending_join( + &self, + user_id: UserId, + wall_now: WallMillis, + skew_ms: u64, + ) -> bool { + self.pending_joins + .iter() + .any(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) + } + + pub fn media_holds(&self, connection: &ConnectionId) -> bool { + self.media_present + .iter() + .any(|sighting| &sighting.connection == connection) + } + + pub fn media_legs_of(&self, user_id: UserId) -> Vec { + self.media_present + .iter() + .filter(|sighting| sighting.user_id == user_id) + .map(|sighting| sighting.connection.clone()) + .collect() + } + + pub fn joining_connections_of( + &self, + user_id: UserId, + wall_now: WallMillis, + skew_ms: u64, + ) -> Vec { + self.pending_joins + .iter() + .filter(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) + .map(|join| join.connection.clone()) + .collect() + } + + pub fn live_connections_of( + &self, + user_id: UserId, + window: PreflightWindow, + ) -> Vec { + let joining = + self.joining_connections_of(user_id, window.wall_now, window.pending_join_skew_ms); + let mut live: Vec = self + .states_for(user_id) + .filter_map(|state| state.connection.clone()) + .collect(); + live.extend(self.media_legs_of(user_id)); + live.retain(|connection| !joining.contains(connection)); + live.sort(); + live.dedup(); + live + } + + pub fn gateway_holds(&self, connection: &ConnectionId) -> bool { + self.voice_states + .iter() + .any(|state| state.connection.as_ref() == Some(connection)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum UnknownMediaCause { + UnreadRequired, + Unasked, + NoCustody, + UserHomeSilent, + NoStamp, + LiveServerSilent, +} + +impl UnknownMediaCause { + pub const ALL: [Self; 6] = [ + Self::UnreadRequired, + Self::Unasked, + Self::NoCustody, + Self::UserHomeSilent, + Self::NoStamp, + Self::LiveServerSilent, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::UnreadRequired => "unread_required", + Self::Unasked => "unasked", + Self::NoCustody => "no_custody", + Self::UserHomeSilent => "user_home_silent", + Self::NoStamp => "no_stamp", + Self::LiveServerSilent => "live_server_silent", + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MediaCoverage { + fleet: Vec, + required: Vec, + read: Vec, + absent: Vec, + unreachable: Vec, +} + +impl MediaCoverage { + pub fn over(fleet: Vec, required: Vec) -> Self { + let mut fleet = fleet; + fleet.sort(); + fleet.dedup(); + let mut required = required; + required.sort(); + required.dedup(); + required.retain(|location| fleet.contains(location)); + Self { + fleet, + required, + read: Vec::new(), + absent: Vec::new(), + unreachable: Vec::new(), + } + } + + pub fn note_read(&mut self, location: &Location) { + if self.fleet.contains(location) && !self.read.contains(location) { + self.read.push(location.clone()); + } + } + + pub fn note_absent(&mut self, location: &Location) { + if self.fleet.contains(location) + && !self.read.contains(location) + && !self.absent.contains(location) + { + self.absent.push(location.clone()); + } + } + + pub fn note_unreachable(&mut self, location: &Location) { + if self.fleet.contains(location) && !self.unreachable.contains(location) { + self.unreachable.push(location.clone()); + } + } + + pub fn fleet(&self) -> &[Location] { + &self.fleet + } + + pub fn required(&self) -> &[Location] { + &self.required + } + + pub fn read(&self) -> &[Location] { + &self.read + } + + pub fn absent(&self) -> &[Location] { + &self.absent + } + + pub fn unreachable(&self) -> &[Location] { + &self.unreachable + } + + pub fn answered(&self) -> Vec { + let mut answered = self.read.clone(); + for location in &self.absent { + if !answered.contains(location) { + answered.push(location.clone()); + } + } + answered.sort(); + answered + } + + pub fn silent(&self) -> Vec { + self.fleet + .iter() + .filter(|location| !self.read.contains(location) && !self.absent.contains(location)) + .cloned() + .collect() + } + + pub fn unasked(&self) -> Option<&Location> { + self.fleet.iter().find(|location| { + !self.read.contains(location) + && !self.absent.contains(location) + && !self.unreachable.contains(location) + }) + } + + pub fn unread(&self) -> Option<&Location> { + self.required + .iter() + .find(|location| !self.read.contains(location) && !self.absent.contains(location)) + } + + pub fn is_complete(&self) -> bool { + !self.read.is_empty() && self.unasked().is_none() && self.unread().is_none() + } +} + +pub const fn connection_id_is_honoured(posture: ConnectionIdGuard, room: RoomKey) -> bool { + match room { + RoomKey::Guild { .. } => matches!(posture, ConnectionIdGuard::Honoured), + RoomKey::Dm { .. } => false, + } +} + +pub fn media_coverage_cause( + coverage: &MediaCoverage, + posture: ConnectionIdGuard, + room: RoomKey, +) -> Option { + if connection_id_is_honoured(posture, room) { + return None; + } + if coverage.unread().is_some() { + return Some(UnknownMediaCause::UnreadRequired); + } + if coverage.is_complete() { + None + } else { + Some(UnknownMediaCause::Unasked) + } +} + +pub fn media_coverage_refusal( + coverage: &MediaCoverage, + posture: ConnectionIdGuard, + room: RoomKey, +) -> Option { + media_coverage_cause(coverage, posture, room).map(|_| AbortReason::UnknownMedia) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MediaCustody<'a> { + pub room: RoomKey, + pub posture: ConnectionIdGuard, + pub fleet: &'a [Location], + pub answered: &'a [Location], + pub read: &'a [Location], + pub target_home: Option<&'a Location>, + pub user_homes: &'a [Location], + pub stamped: &'a [Location], +} + +impl MediaCustody<'_> { + fn silent(&self) -> Vec<&Location> { + self.fleet + .iter() + .filter(|location| !self.answered.contains(location)) + .collect() + } + + fn target_is_held(&self) -> bool { + self.target_home + .is_some_and(|home| self.fleet.contains(home) && self.read.contains(home)) + } +} + +pub fn media_custody_cause(custody: &MediaCustody<'_>) -> Option { + let silent = custody.silent(); + if silent.is_empty() { + return None; + } + if connection_id_is_honoured(custody.posture, custody.room) { + return if silent + .iter() + .all(|location| custody.stamped.contains(location)) + { + None + } else { + Some(UnknownMediaCause::NoStamp) + }; + } + if !custody.target_is_held() { + return Some(UnknownMediaCause::NoCustody); + } + if custody.user_homes.iter().any(|home| silent.contains(&home)) { + return Some(UnknownMediaCause::UserHomeSilent); + } + None +} + +pub fn media_custody_refusal(custody: &MediaCustody<'_>) -> Option { + media_custody_cause(custody).map(|_| AbortReason::UnknownMedia) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FleetSilence<'a> { + pub room: RoomKey, + pub posture: ConnectionIdGuard, + pub fleet: &'a [Location], + pub answered: &'a [Location], + pub written_off: &'a [Location], +} + +impl FleetSilence<'_> { + fn silent_and_still_expected(&self) -> Option<&Location> { + self.fleet.iter().find(|location| { + !self.answered.contains(location) && !self.written_off.contains(location) + }) + } +} + +pub fn fleet_silence_cause(silence: &FleetSilence<'_>) -> Option { + if connection_id_is_honoured(silence.posture, silence.room) { + return None; + } + silence + .silent_and_still_expected() + .map(|_| UnknownMediaCause::LiveServerSilent) +} + +pub fn fleet_silence_refusal(silence: &FleetSilence<'_>) -> Option { + fleet_silence_cause(silence).map(|_| AbortReason::UnknownMedia) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PreflightWindow { + pub now: Millis, + pub wall_now: WallMillis, + pub preflight_max_age_ms: u64, + pub pending_join_skew_ms: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DmPosture { + pub dm_gateway_eviction: bool, + pub dm_media_eviction: bool, +} + +impl Default for DmPosture { + fn default() -> Self { + Self { + dm_gateway_eviction: true, + dm_media_eviction: false, + } + } +} + +impl DmPosture { + pub const fn allows_gateway_removal(self, room: RoomKey) -> bool { + match room { + RoomKey::Guild { .. } => true, + RoomKey::Dm { .. } => self.dm_gateway_eviction, + } + } + + pub const fn allows_media_removal(self, room: RoomKey) -> bool { + match room { + RoomKey::Guild { .. } => true, + RoomKey::Dm { .. } => self.dm_media_eviction, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayRemovalGuard<'a> { + pub room: RoomKey, + pub target: &'a ConnectionId, + pub user_id: UserId, + pub posture: ConnectionIdGuard, + pub dm_posture: DmPosture, + pub preflight: &'a GatewayPreflight, + pub window: PreflightWindow, +} + +pub fn gateway_removal_preflight(guard: &GatewayRemovalGuard<'_>) -> Result<(), AbortReason> { + if !guard + .preflight + .is_fresh(guard.window.now, guard.window.preflight_max_age_ms) + { + return Err(AbortReason::PreflightStale); + } + + if !guard.dm_posture.allows_gateway_removal(guard.room) { + return Err(AbortReason::DmPosture); + } + + let states: Vec<&PreflightVoiceState> = guard.preflight.states_for(guard.user_id).collect(); + + if states.iter().any(|state| state.connection.is_none()) { + return Err(AbortReason::NoConnectionId); + } + + if connection_id_is_honoured(guard.posture, guard.room) { + if !states + .iter() + .any(|state| state.connection.as_ref() == Some(guard.target)) + { + return Err(AbortReason::PreflightDisagreed); + } + } else { + let live = guard + .preflight + .live_connections_of(guard.user_id, guard.window); + if live.len() != 1 { + return Err(AbortReason::SiblingConnection); + } + if &live[0] != guard.target { + return Err(AbortReason::SiblingConnection); + } + } + + if guard.preflight.user_has_pending_join( + guard.user_id, + guard.window.wall_now, + guard.window.pending_join_skew_ms, + ) { + return Err(AbortReason::PendingJoin); + } + + if guard.preflight.media_holds(guard.target) { + return Err(AbortReason::PreflightDisagreed); + } + + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MediaRemovalGuard<'a> { + pub room: RoomKey, + pub target: &'a ConnectionId, + pub user_id: UserId, + pub dm_posture: DmPosture, + pub divergence_since: Millis, + pub participant_joined_at: Millis, + pub repair_verdict: Option, + pub server_health: ServerHealth, + pub preflight: &'a GatewayPreflight, + pub window: PreflightWindow, +} + +pub fn media_removal_preflight(guard: &MediaRemovalGuard<'_>) -> Result<(), AbortReason> { + if !guard + .preflight + .is_fresh(guard.window.now, guard.window.preflight_max_age_ms) + { + return Err(AbortReason::PreflightStale); + } + + if !guard.dm_posture.allows_media_removal(guard.room) { + return Err(AbortReason::DmPosture); + } + + if guard + .preflight + .states_for(guard.user_id) + .any(|state| state.connection.is_none()) + { + return Err(AbortReason::NoConnectionId); + } + + if guard.participant_joined_at > guard.divergence_since { + return Err(AbortReason::JoinedAfterDivergence); + } + + if matches!(guard.room, RoomKey::Guild { .. }) + && guard.repair_verdict != Some(RepairVerdict::NotRepairable) + { + return Err(AbortReason::PreflightDisagreed); + } + + if guard.preflight.gateway_holds(guard.target) { + return Err(AbortReason::PreflightDisagreed); + } + + if guard.preflight.user_has_pending_join( + guard.user_id, + guard.window.wall_now, + guard.window.pending_join_skew_ms, + ) { + return Err(AbortReason::PendingJoin); + } + + if !guard.server_health.is_healthy() { + return Err(AbortReason::ServerUnhealthy); + } + + Ok(()) +} diff --git a/fluxer_recon/src/health.rs b/fluxer_recon/src/health.rs new file mode 100644 index 000000000..4292607df --- /dev/null +++ b/fluxer_recon/src/health.rs @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; + +use crate::ids::{Location, Millis}; + +pub const DEGRADED_AT_FAILURES: u32 = 3; +pub const UNREACHABLE_AT_FAILURES: u32 = 10; +pub const QUARANTINED_AT_FAILURES: u32 = 30; +pub const SUCCESSES_TO_RECOVER: u32 = 3; +pub const UNREACHABLE_PROBE_INTERVAL_MS: u64 = 10_000; +pub const QUARANTINED_PROBE_INTERVAL_MS: u64 = 60_000; +pub const MUTATIONS_PAUSED_BACKOFF_MS: u64 = 15_000; +pub const MUTATIONS_PAUSED_MAX_BACKOFF_MS: u64 = 120_000; +pub const MUTATIONS_PAUSED_QUIET_MS: u64 = 300_000; +pub const MUTATIONS_PAUSED_MAX_DOUBLINGS: u32 = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ReconMode { + Halted, + Observing, + Constructive, + Enforcing, +} + +impl ReconMode { + pub fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "halted" => Some(Self::Halted), + "observing" => Some(Self::Observing), + "constructive" => Some(Self::Constructive), + "enforcing" => Some(Self::Enforcing), + _ => None, + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Halted => "halted", + Self::Observing => "observing", + Self::Constructive => "constructive", + Self::Enforcing => "enforcing", + } + } + + pub const fn allows_constructive(self) -> bool { + matches!(self, Self::Constructive | Self::Enforcing) + } + + pub const fn allows_destructive(self) -> bool { + matches!(self, Self::Enforcing) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ModeClamp { + Warmup, + ColdStart, + BreakerTripped, + SingletonConflict, + TopologyStale, + NatsReconnect, + MutationsPaused, + CoverageOverrun, +} + +impl ModeClamp { + pub const ALL: [Self; 8] = [ + Self::Warmup, + Self::ColdStart, + Self::BreakerTripped, + Self::SingletonConflict, + Self::TopologyStale, + Self::NatsReconnect, + Self::MutationsPaused, + Self::CoverageOverrun, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::Warmup => "warmup", + Self::ColdStart => "cold_start", + Self::BreakerTripped => "breaker_tripped", + Self::SingletonConflict => "singleton_conflict", + Self::TopologyStale => "topology_stale", + Self::NatsReconnect => "nats_reconnect", + Self::MutationsPaused => "mutations_paused", + Self::CoverageOverrun => "coverage_overrun", + } + } + + pub fn parse(value: &str) -> Option { + let wanted = value.trim().to_ascii_lowercase(); + Self::ALL.into_iter().find(|clamp| clamp.label() == wanted) + } + + pub const fn is_self_clearing(self) -> bool { + match self { + Self::Warmup + | Self::ColdStart + | Self::SingletonConflict + | Self::TopologyStale + | Self::NatsReconnect + | Self::MutationsPaused + | Self::CoverageOverrun => true, + Self::BreakerTripped => false, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModeClamps { + active: BTreeMap>, +} + +impl ModeClamps { + pub fn new() -> Self { + Self::default() + } + + pub fn set(&mut self, clamp: ModeClamp, engaged: bool) { + if engaged { + self.active.insert(clamp, None); + } else { + self.active.remove(&clamp); + } + } + + pub fn engage_until(&mut self, clamp: ModeClamp, until: Millis) { + self.active.insert(clamp, Some(until)); + } + + pub fn release_expired(&mut self, now: Millis) { + self.active + .retain(|_, until| until.is_none_or(|deadline| now < deadline)); + } + + pub fn is_engaged(&self, clamp: ModeClamp) -> bool { + self.active.contains_key(&clamp) + } + + pub fn deadline(&self, clamp: ModeClamp) -> Option { + self.active.get(&clamp).copied().flatten() + } + + pub fn release(&mut self, clamp: ModeClamp) -> bool { + self.active.remove(&clamp).is_some() + } + + pub fn engaged(&self) -> impl Iterator + '_ { + self.active.keys().copied() + } + + pub fn is_empty(&self) -> bool { + self.active.is_empty() + } + + pub fn ceiling(&self) -> ReconMode { + if self.active.is_empty() { + ReconMode::Enforcing + } else { + ReconMode::Constructive + } + } +} + +pub fn effective_mode( + configured: ReconMode, + runtime_override: Option, + clamps: &ModeClamps, +) -> ReconMode { + let requested = match runtime_override { + None => configured, + Some(override_mode) => configured.min(override_mode), + }; + requested.min(clamps.ceiling()) +} + +pub const fn mutations_paused_hold_ms(windows: u32) -> u64 { + if windows == 0 { + return 0; + } + let doublings = if windows - 1 > MUTATIONS_PAUSED_MAX_DOUBLINGS { + MUTATIONS_PAUSED_MAX_DOUBLINGS + } else { + windows - 1 + }; + let hold = MUTATIONS_PAUSED_BACKOFF_MS.saturating_mul(1u64 << doublings); + if hold > MUTATIONS_PAUSED_MAX_BACKOFF_MS { + MUTATIONS_PAUSED_MAX_BACKOFF_MS + } else { + hold + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PauseBackoff { + windows: u32, + until: Option, +} + +impl PauseBackoff { + pub const fn new() -> Self { + Self { + windows: 0, + until: None, + } + } + + pub const fn windows(&self) -> u32 { + self.windows + } + + pub const fn until(&self) -> Option { + self.until + } + + pub const fn hold_ms(&self) -> u64 { + mutations_paused_hold_ms(self.windows) + } + + pub const fn is_holding(&self, now: Millis) -> bool { + match self.until { + None => false, + Some(until) => now.get() < until.get(), + } + } + + pub const fn remaining_ms(&self, now: Millis) -> u64 { + match self.until { + None => 0, + Some(until) => until.saturating_since(now), + } + } + + pub const fn note_refused(&mut self, now: Millis) -> Option { + if self.is_holding(now) { + return None; + } + self.windows = self.windows.saturating_add(1); + let until = now.saturating_add_millis(mutations_paused_hold_ms(self.windows)); + self.until = Some(until); + Some(until) + } + + pub const fn clear(&mut self) { + self.windows = 0; + self.until = None; + } + + pub const fn decay(&mut self, now: Millis) -> bool { + let Some(until) = self.until else { + return false; + }; + if now.get() < until.get() { + return false; + } + if now.saturating_since(until) < MUTATIONS_PAUSED_QUIET_MS { + return false; + } + self.clear(); + true + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WarmupGate { + started_at: Millis, + warmup_ms: u64, + coverage_pass_complete: bool, +} + +impl WarmupGate { + pub const fn new(started_at: Millis, warmup_ms: u64) -> Self { + Self { + started_at, + warmup_ms, + coverage_pass_complete: false, + } + } + + pub const fn note_coverage_pass(&mut self) { + self.coverage_pass_complete = true; + } + + pub const fn coverage_pass_complete(&self) -> bool { + self.coverage_pass_complete + } + + pub const fn elapsed_complete(&self, now: Millis) -> bool { + now.saturating_since(self.started_at) >= self.warmup_ms + } + + pub const fn is_complete(&self, now: Millis) -> bool { + self.elapsed_complete(now) && self.coverage_pass_complete + } + + pub const fn force_complete(&mut self) { + self.warmup_ms = 0; + self.coverage_pass_complete = true; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ServerHealth { + Healthy, + Degraded, + Unreachable, + Quarantined, +} + +impl ServerHealth { + pub const fn label(self) -> &'static str { + match self { + Self::Healthy => "healthy", + Self::Degraded => "degraded", + Self::Unreachable => "unreachable", + Self::Quarantined => "quarantined", + } + } + + pub const fn is_healthy(self) -> bool { + matches!(self, Self::Healthy) + } + + pub const fn is_written_off(self) -> bool { + matches!(self, Self::Unreachable | Self::Quarantined) + } + + pub const fn severity(self) -> u8 { + match self { + Self::Healthy => 0, + Self::Degraded => 1, + Self::Unreachable => 2, + Self::Quarantined => 3, + } + } + + pub const fn worse_of(self, other: Self) -> Self { + if self.severity() >= other.severity() { + self + } else { + other + } + } + + pub const fn probe_interval_ms(self) -> Option { + match self { + Self::Healthy | Self::Degraded => None, + Self::Unreachable => Some(UNREACHABLE_PROBE_INTERVAL_MS), + Self::Quarantined => Some(QUARANTINED_PROBE_INTERVAL_MS), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProbeOutcome { + Success, + Failure, + AuthFailure, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ServerHealthTracker { + state: ServerHealth, + consecutive_failures: u32, + consecutive_successes: u32, + last_probe_at: Option, + ever_answered: bool, +} + +impl Default for ServerHealthTracker { + fn default() -> Self { + Self::new() + } +} + +impl ServerHealthTracker { + pub const fn new() -> Self { + Self { + state: ServerHealth::Healthy, + consecutive_failures: 0, + consecutive_successes: 0, + last_probe_at: None, + ever_answered: false, + } + } + + pub const fn state(&self) -> ServerHealth { + self.state + } + + pub const fn consecutive_failures(&self) -> u32 { + self.consecutive_failures + } + + pub const fn ever_answered(&self) -> bool { + self.ever_answered + } + + pub const fn record(&mut self, outcome: ProbeOutcome, now: Millis) { + self.last_probe_at = Some(now); + match outcome { + ProbeOutcome::Success => { + self.ever_answered = true; + self.consecutive_failures = 0; + self.consecutive_successes = self.consecutive_successes.saturating_add(1); + if self.consecutive_successes >= SUCCESSES_TO_RECOVER { + self.state = ServerHealth::Healthy; + } + } + ProbeOutcome::Failure => { + self.consecutive_successes = 0; + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.state = self + .state + .worse_of(failure_state(self.consecutive_failures)); + } + ProbeOutcome::AuthFailure => { + self.consecutive_successes = 0; + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.state = ServerHealth::Quarantined; + } + } + } + + pub const fn may_probe(&self, now: Millis) -> bool { + match (self.state.probe_interval_ms(), self.last_probe_at) { + (None, _) => true, + (Some(_), None) => true, + (Some(interval), Some(last)) => now.saturating_since(last) >= interval, + } + } + + pub const fn last_probe_at(&self) -> Option { + self.last_probe_at + } +} + +const fn failure_state(consecutive_failures: u32) -> ServerHealth { + if consecutive_failures >= QUARANTINED_AT_FAILURES { + ServerHealth::Quarantined + } else if consecutive_failures >= UNREACHABLE_AT_FAILURES { + ServerHealth::Unreachable + } else if consecutive_failures >= DEGRADED_AT_FAILURES { + ServerHealth::Degraded + } else { + ServerHealth::Healthy + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ServerHealthMap { + servers: BTreeMap, +} + +impl ServerHealthMap { + pub fn new() -> Self { + Self::default() + } + + pub fn record(&mut self, location: &Location, outcome: ProbeOutcome, now: Millis) { + self.servers + .entry(location.clone()) + .or_default() + .record(outcome, now); + } + + pub fn health(&self, location: &Location) -> ServerHealth { + self.servers + .get(location) + .map_or(ServerHealth::Healthy, ServerHealthTracker::state) + } + + pub fn ever_answered(&self, location: &Location) -> bool { + self.servers + .get(location) + .is_some_and(ServerHealthTracker::ever_answered) + } + + pub fn may_probe(&self, location: &Location, now: Millis) -> bool { + self.servers + .get(location) + .is_none_or(|tracker| tracker.may_probe(now)) + } + + pub fn tracked(&self) -> impl Iterator { + self.servers.iter() + } + + pub fn forget(&mut self, location: &Location) { + self.servers.remove(location); + } + + pub fn len(&self) -> usize { + self.servers.len() + } + + pub fn is_empty(&self) -> bool { + self.servers.is_empty() + } +} diff --git a/fluxer_recon/src/ids.rs b/fluxer_recon/src/ids.rs new file mode 100644 index 000000000..155874ff2 --- /dev/null +++ b/fluxer_recon/src/ids.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::fmt; + +const REDACTED: &str = "***"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum IdError { + #[error("identifier is empty")] + Empty, + #[error("identifier contains whitespace")] + ContainsWhitespace, + #[error("identifier contains an underscore")] + ContainsUnderscore, +} + +macro_rules! u64_newtype { + ($name:ident) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct $name(u64); + + impl $name { + pub const fn new(value: u64) -> Self { + Self(value) + } + + pub const fn get(self) -> u64 { + self.0 + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } + } + }; +} + +u64_newtype!(GuildId); +u64_newtype!(ChannelId); +u64_newtype!(UserId); +u64_newtype!(TurnId); +u64_newtype!(Epoch); +u64_newtype!(Millis); +u64_newtype!(WallMillis); + +impl TurnId { + pub const FIRST: Self = Self(0); + + pub const fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl Epoch { + pub const FIRST: Self = Self(0); + + pub const fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl Millis { + pub const ZERO: Self = Self(0); + + pub const fn saturating_since(self, earlier: Self) -> u64 { + self.0.saturating_sub(earlier.0) + } + + pub const fn saturating_add_millis(self, delta: u64) -> Self { + Self(self.0.saturating_add(delta)) + } +} + +impl WallMillis { + pub const ZERO: Self = Self(0); + + pub const fn saturating_since(self, earlier: Self) -> u64 { + self.0.saturating_sub(earlier.0) + } + + pub const fn saturating_add_millis(self, delta: u64) -> Self { + Self(self.0.saturating_add(delta)) + } +} + +fn checked_text(value: &str) -> Result, IdError> { + if value.is_empty() { + return Err(IdError::Empty); + } + if value.contains(char::is_whitespace) { + return Err(IdError::ContainsWhitespace); + } + Ok(Box::from(value)) +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ConnectionId(Box); + +impl ConnectionId { + pub fn new(value: &str) -> Result { + let text = checked_text(value)?; + if text.contains('_') { + return Err(IdError::ContainsUnderscore); + } + Ok(Self(text)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ConnectionId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RegionId(Box); + +impl RegionId { + pub fn new(value: &str) -> Result { + checked_text(value).map(Self) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for RegionId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ServerId(Box); + +impl ServerId { + pub fn new(value: &str) -> Result { + checked_text(value).map(Self) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ServerId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Clone, PartialEq, Eq)] +pub struct ApiSecret(Box); + +impl ApiSecret { + pub fn new(value: &str) -> Self { + Self(Box::from(value)) + } + + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for ApiSecret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(REDACTED) + } +} + +impl fmt::Display for ApiSecret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(REDACTED) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Scope { + Guild, + Dm, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum RoomKey { + Guild { + guild_id: GuildId, + channel_id: ChannelId, + }, + Dm { + channel_id: ChannelId, + }, +} + +impl RoomKey { + pub const fn scope(self) -> Scope { + match self { + Self::Guild { .. } => Scope::Guild, + Self::Dm { .. } => Scope::Dm, + } + } + + pub const fn channel_id(self) -> ChannelId { + match self { + Self::Guild { channel_id, .. } => channel_id, + Self::Dm { channel_id } => channel_id, + } + } + + pub const fn guild_id(self) -> Option { + match self { + Self::Guild { guild_id, .. } => Some(guild_id), + Self::Dm { .. } => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Location { + pub region: RegionId, + pub server: ServerId, +} + +impl Location { + pub const fn new(region: RegionId, server: ServerId) -> Self { + Self { region, server } + } +} + +impl fmt::Display for Location { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.region, self.server) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ConnectionKey { + pub room: RoomKey, + pub connection: ConnectionId, +} + +impl ConnectionKey { + pub const fn new(room: RoomKey, connection: ConnectionId) -> Self { + Self { room, connection } + } +} diff --git a/fluxer_recon/src/ledger.rs b/fluxer_recon/src/ledger.rs new file mode 100644 index 000000000..4d15a0e02 --- /dev/null +++ b/fluxer_recon/src/ledger.rs @@ -0,0 +1,978 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; +use std::mem::size_of; + +use crate::decide::DecideParams; +use crate::evidence::{CliffWindow, PriorConnection}; +use crate::gateway::Nonce; +use crate::ids::{ConnectionId, ConnectionKey, Epoch, Location, Millis, RoomKey, TurnId, UserId}; + +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; +const CONNECTION_ID_HEAP_BYTES: usize = 40; +pub const MAX_ROSTER_STAMPS: usize = 16; +const ROSTER_STAMP_BYTES: usize = 16; +const ROOM_OVERHEAD_BYTES: usize = 256 + MAX_ROSTER_STAMPS * ROSTER_STAMP_BYTES; + +pub const MAP_ENTRY_OVERHEAD_BYTES: usize = 48; +pub const LOCATION_MAP_BYTES: u64 = 640; +pub const LOCATION_ENTRY_BYTES: u64 = 64; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Fingerprint(u64); + +impl Fingerprint { + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Hasher(u64); + +impl Hasher { + const fn new() -> Self { + Self(FNV_OFFSET) + } + + const fn byte(&mut self, value: u8) { + self.0 ^= value as u64; + self.0 = self.0.wrapping_mul(FNV_PRIME); + } + + const fn u64(&mut self, value: u64) { + let bytes = value.to_le_bytes(); + let mut index = 0; + while index < bytes.len() { + self.byte(bytes[index]); + index += 1; + } + } + + fn text(&mut self, value: &str) { + for byte in value.as_bytes() { + self.byte(*byte); + } + self.byte(0); + } + + const fn finish(self) -> Fingerprint { + Fingerprint(self.0) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FingerprintInput<'a> { + pub room: RoomKey, + pub connection: &'a ConnectionId, + pub user_id: UserId, + pub gateway_siblings: &'a [ConnectionId], + pub media_locations: &'a [Location], + pub participant_joined_at: Option, + pub census_epoch: Epoch, + pub topology_epoch: Epoch, +} + +pub fn fingerprint(input: &FingerprintInput<'_>) -> Fingerprint { + let mut hasher = Hasher::new(); + + match input.room { + RoomKey::Guild { + guild_id, + channel_id, + } => { + hasher.byte(1); + hasher.u64(guild_id.get()); + hasher.u64(channel_id.get()); + } + RoomKey::Dm { channel_id } => { + hasher.byte(2); + hasher.u64(channel_id.get()); + } + } + + hasher.text(input.connection.as_str()); + hasher.u64(input.user_id.get()); + + hasher.u64(input.gateway_siblings.len() as u64); + for sibling in input.gateway_siblings { + hasher.text(sibling.as_str()); + } + + hasher.u64(input.media_locations.len() as u64); + for location in input.media_locations { + hasher.text(location.region.as_str()); + hasher.text(location.server.as_str()); + } + + match input.participant_joined_at { + None => hasher.byte(0), + Some(joined_at) => { + hasher.byte(1); + hasher.u64(joined_at.get()); + } + } + + hasher.u64(input.census_epoch.get()); + hasher.u64(input.topology_epoch.get()); + hasher.finish() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LocationIndex(u16); + +impl LocationIndex { + pub const fn get(self) -> u16 { + self.0 + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LocationTable { + entries: Vec, +} + +impl LocationTable { + pub const fn new() -> Self { + Self { + entries: Vec::new(), + } + } + + pub fn intern(&mut self, location: &Location) -> Option { + if let Some(position) = self.entries.iter().position(|entry| entry == location) { + return u16::try_from(position).ok().map(LocationIndex); + } + let index = u16::try_from(self.entries.len()).ok()?; + self.entries.push(location.clone()); + Some(LocationIndex(index)) + } + + pub fn get(&self, index: LocationIndex) -> Option<&Location> { + self.entries.get(usize::from(index.0)) + } + + pub fn index_of(&self, location: &Location) -> Option { + self.entries + .iter() + .position(|entry| entry == location) + .and_then(|position| u16::try_from(position).ok()) + .map(LocationIndex) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepairVerdict { + Repaired, + NoChange, + NotRepairable, +} + +impl RepairVerdict { + pub const fn label(self) -> &'static str { + match self { + Self::Repaired => "repaired", + Self::NoChange => "no_change", + Self::NotRepairable => "not_repairable", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ActionKind { + RemoveGatewayState, + RemoveParticipant, +} + +impl ActionKind { + pub const fn label(self) -> &'static str { + match self { + Self::RemoveGatewayState => "remove_gateway_state", + Self::RemoveParticipant => "remove_participant", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PriorDivergence { + GatewayOnly, + MediaOnly, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WedgedReason { + RepairExhausted, + ActionIneffective, +} + +impl WedgedReason { + pub const fn label(self) -> &'static str { + match self { + Self::RepairExhausted => "repair_exhausted", + Self::ActionIneffective => "action_ineffective", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionState { + Nascent, + Consistent, + PendingJoin { + expires_at: crate::ids::WallMillis, + nonce: Nonce, + last_confirm: Millis, + }, + GatewayOnly { + since: Millis, + corroborations: u32, + fingerprint: Fingerprint, + last_corroboration: Millis, + }, + MediaOnly { + since: Millis, + corroborations: u32, + fingerprint: Fingerprint, + last_corroboration: Millis, + location: LocationIndex, + participant_joined_at: Millis, + repair_verdict: Option, + }, + Repairing { + since: Millis, + last_attempt: Millis, + }, + ActionTaken { + kind: ActionKind, + at: Millis, + authorizing_turn: TurnId, + prior: PriorDivergence, + since: Millis, + }, + Wedged { + reason: WedgedReason, + at: Millis, + fingerprint: Fingerprint, + }, + Retired { + at: Millis, + }, +} + +impl ConnectionState { + pub const fn label(&self) -> &'static str { + match self { + Self::Nascent => "nascent", + Self::Consistent => "consistent", + Self::PendingJoin { .. } => "pending_join", + Self::GatewayOnly { .. } => "gateway_only", + Self::MediaOnly { .. } => "media_only", + Self::Repairing { .. } => "repairing", + Self::ActionTaken { .. } => "action_taken", + Self::Wedged { .. } => "wedged", + Self::Retired { .. } => "retired", + } + } + + pub const fn stored_fingerprint(&self) -> Option { + match self { + Self::GatewayOnly { fingerprint, .. } + | Self::MediaOnly { fingerprint, .. } + | Self::Wedged { fingerprint, .. } => Some(*fingerprint), + Self::Nascent + | Self::Consistent + | Self::PendingJoin { .. } + | Self::Repairing { .. } + | Self::ActionTaken { .. } + | Self::Retired { .. } => None, + } + } + + pub const fn corroborations(&self) -> u32 { + match self { + Self::GatewayOnly { corroborations, .. } | Self::MediaOnly { corroborations, .. } => { + *corroborations + } + Self::Nascent + | Self::Consistent + | Self::PendingJoin { .. } + | Self::Repairing { .. } + | Self::ActionTaken { .. } + | Self::Wedged { .. } + | Self::Retired { .. } => 0, + } + } + + pub const fn divergence_since(&self) -> Option { + match self { + Self::GatewayOnly { since, .. } + | Self::MediaOnly { since, .. } + | Self::Repairing { since, .. } + | Self::ActionTaken { since, .. } => Some(*since), + Self::Nascent + | Self::Consistent + | Self::PendingJoin { .. } + | Self::Wedged { .. } + | Self::Retired { .. } => None, + } + } + + pub const fn is_divergent(&self) -> bool { + match self { + Self::GatewayOnly { .. } + | Self::MediaOnly { .. } + | Self::Repairing { .. } + | Self::ActionTaken { .. } + | Self::Wedged { .. } => true, + Self::Nascent | Self::Consistent | Self::PendingJoin { .. } | Self::Retired { .. } => { + false + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConnectionLedger { + pub connection: ConnectionId, + pub user_id: UserId, + pub state: ConnectionState, + pub last_seen: Millis, + pub action_attempts: u8, + pub repair_attempts: u8, + pub last_action_at: Option, + pub last_repair_verdict: Option, + pub last_location: Option, +} + +impl ConnectionLedger { + pub fn new(connection: ConnectionId, user_id: UserId, now: Millis) -> Self { + Self { + connection, + user_id, + state: ConnectionState::Nascent, + last_seen: now, + action_attempts: 0, + repair_attempts: 0, + last_action_at: None, + last_repair_verdict: None, + last_location: None, + } + } + + pub const fn is_divergent(&self) -> bool { + self.state.is_divergent() + } + + pub const fn believed_home(&self) -> Option { + match self.state { + ConnectionState::Retired { .. } => None, + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::GatewayOnly { .. } + | ConnectionState::MediaOnly { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::ActionTaken { .. } + | ConnectionState::Wedged { .. } => self.last_location, + } + } + + pub fn clear_evidence(&mut self) { + self.state = ConnectionState::Nascent; + self.last_repair_verdict = None; + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomLedger { + room: RoomKey, + connections: Vec, + last_seen: Millis, + last_turn: TurnId, + gateway_cliff: CliffWindow, + media_cliff: CliffWindow, + partially_unreadable: bool, + no_candidates: bool, + last_known_locations: Vec, + roster_stamps: Vec<(LocationIndex, Millis)>, +} + +impl RoomLedger { + fn new(room: RoomKey, now: Millis) -> Self { + Self { + room, + connections: Vec::new(), + last_seen: now, + last_turn: TurnId::FIRST, + gateway_cliff: CliffWindow::clear(), + media_cliff: CliffWindow::clear(), + partially_unreadable: false, + no_candidates: false, + last_known_locations: Vec::new(), + roster_stamps: Vec::new(), + } + } + + pub const fn room(&self) -> RoomKey { + self.room + } + + pub fn connections(&self) -> &[ConnectionLedger] { + &self.connections + } + + pub fn connection(&self, connection: &ConnectionId) -> Option<&ConnectionLedger> { + self.connections + .iter() + .find(|entry| &entry.connection == connection) + } + + pub fn connection_mut(&mut self, connection: &ConnectionId) -> Option<&mut ConnectionLedger> { + self.connections + .iter_mut() + .find(|entry| &entry.connection == connection) + } + + pub const fn last_seen(&self) -> Millis { + self.last_seen + } + + pub const fn last_turn(&self) -> TurnId { + self.last_turn + } + + pub const fn note_turn(&mut self, turn: TurnId, now: Millis) { + self.last_turn = turn; + self.last_seen = now; + } + + pub const fn gateway_cliff(&self) -> CliffWindow { + self.gateway_cliff + } + + pub const fn gateway_cliff_mut(&mut self) -> &mut CliffWindow { + &mut self.gateway_cliff + } + + pub const fn media_cliff(&self) -> CliffWindow { + self.media_cliff + } + + pub const fn media_cliff_mut(&mut self) -> &mut CliffWindow { + &mut self.media_cliff + } + + pub const fn partially_unreadable(&self) -> bool { + self.partially_unreadable + } + + pub const fn set_partially_unreadable(&mut self, value: bool) { + self.partially_unreadable = value; + } + + pub const fn no_candidates(&self) -> bool { + self.no_candidates + } + + pub const fn set_no_candidates(&mut self, value: bool) { + self.no_candidates = value; + } + + pub fn last_known_locations(&self) -> &[LocationIndex] { + &self.last_known_locations + } + + pub fn note_location(&mut self, index: LocationIndex) { + if !self.last_known_locations.contains(&index) { + self.last_known_locations.push(index); + } + } + + pub fn note_roster_read(&mut self, index: LocationIndex, at: Millis) { + if let Some(stamp) = self + .roster_stamps + .iter_mut() + .find(|(stamped, _)| *stamped == index) + { + stamp.1 = at; + return; + } + if self.roster_stamps.len() >= MAX_ROSTER_STAMPS { + let oldest = self + .roster_stamps + .iter() + .enumerate() + .min_by_key(|(_, (_, at))| at.get()) + .map(|(position, _)| position); + if let Some(position) = oldest { + self.roster_stamps.remove(position); + } + } + self.roster_stamps.push((index, at)); + } + + pub fn roster_read_at(&self, index: LocationIndex) -> Option { + self.roster_stamps + .iter() + .find(|(stamped, _)| *stamped == index) + .map(|(_, at)| *at) + } + + pub fn retain_roster_stamps(&mut self, live: &[LocationIndex]) { + self.roster_stamps.retain(|(index, _)| live.contains(index)); + } + + pub fn roster_stamps(&self) -> &[(LocationIndex, Millis)] { + &self.roster_stamps + } + + pub fn believed_homes(&self) -> Vec { + let mut homes: Vec = self + .connections + .iter() + .filter_map(ConnectionLedger::believed_home) + .collect(); + homes.sort_by_key(|index| index.get()); + homes.dedup(); + homes + } + + pub fn believed_home_counts(&self) -> Vec<(LocationIndex, u32)> { + let mut counts: Vec<(LocationIndex, u32)> = Vec::new(); + for home in self + .connections + .iter() + .filter_map(ConnectionLedger::believed_home) + { + match counts.iter_mut().find(|(index, _)| *index == home) { + Some((_, count)) => *count = count.saturating_add(1), + None => counts.push((home, 1)), + } + } + counts.sort_by_key(|(index, _)| index.get()); + counts + } + + pub fn prior_connections(&self) -> Vec { + self.connections + .iter() + .map(|entry| PriorConnection { + connection: entry.connection.clone(), + was_consistent: matches!(entry.state, ConnectionState::Consistent), + }) + .collect() + } + + pub fn reset_corroborations(&mut self) { + for entry in &mut self.connections { + if entry.is_divergent() { + entry.clear_evidence(); + } + } + } + + pub fn divergent_connections(&self) -> usize { + self.connections + .iter() + .filter(|entry| entry.is_divergent()) + .count() + } + + fn is_cold(&self) -> bool { + self.connections.iter().all(|entry| { + matches!( + entry.state, + ConnectionState::Consistent + | ConnectionState::Nascent + | ConnectionState::Retired { .. } + ) + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LedgerLimits { + pub max_tracked_rooms: usize, + pub max_tracked_connections: usize, + pub max_connections_per_room: usize, + pub memory_budget_bytes: u64, + pub retired_grace_ms: u64, +} + +impl Default for LedgerLimits { + fn default() -> Self { + Self { + max_tracked_rooms: 8_192, + max_tracked_connections: 65_536, + max_connections_per_room: 512, + memory_budget_bytes: 16_777_216, + retired_grace_ms: 60_000, + } + } +} + +pub const CONNECTION_ENTRY_BYTES: u64 = + (size_of::() + CONNECTION_ID_HEAP_BYTES) as u64; +pub const ROOM_ENTRY_BYTES: u64 = (size_of::() + ROOM_OVERHEAD_BYTES) as u64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdmissionRefusal { + RoomCapacity, + ConnectionCapacity, + RoomConnectionCapacity, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Ledger { + rooms: BTreeMap, + locations: LocationTable, + limits: LedgerLimits, + params: DecideParams, + tracked_connections: usize, + evicted_rooms_total: u64, + evicted_connections_total: u64, +} + +impl Ledger { + pub fn new(limits: LedgerLimits, params: DecideParams) -> Self { + Self { + rooms: BTreeMap::new(), + locations: LocationTable::new(), + limits, + params, + tracked_connections: 0, + evicted_rooms_total: 0, + evicted_connections_total: 0, + } + } + + pub const fn limits(&self) -> LedgerLimits { + self.limits + } + + pub const fn params(&self) -> &DecideParams { + &self.params + } + + pub const fn locations(&self) -> &LocationTable { + &self.locations + } + + pub fn intern_location(&mut self, location: &Location) -> Option { + self.locations.intern(location) + } + + pub fn location(&self, index: LocationIndex) -> Option<&Location> { + self.locations.get(index) + } + + pub fn location_index(&self, location: &Location) -> Option { + self.locations.index_of(location) + } + + pub fn rooms(&self) -> impl Iterator { + self.rooms.iter() + } + + pub fn room(&self, room: &RoomKey) -> Option<&RoomLedger> { + self.rooms.get(room) + } + + pub fn room_mut(&mut self, room: &RoomKey) -> Option<&mut RoomLedger> { + self.rooms.get_mut(room) + } + + pub fn connection(&self, key: &ConnectionKey) -> Option<&ConnectionLedger> { + self.rooms + .get(&key.room) + .and_then(|room| room.connection(&key.connection)) + } + + pub fn connection_mut(&mut self, key: &ConnectionKey) -> Option<&mut ConnectionLedger> { + self.rooms + .get_mut(&key.room) + .and_then(|room| room.connection_mut(&key.connection)) + } + + pub fn ensure_room( + &mut self, + room: RoomKey, + now: Millis, + ) -> Result<&mut RoomLedger, AdmissionRefusal> { + if !self.rooms.contains_key(&room) && self.rooms.len() >= self.limits.max_tracked_rooms { + return Err(AdmissionRefusal::RoomCapacity); + } + Ok(self + .rooms + .entry(room) + .or_insert_with(|| RoomLedger::new(room, now))) + } + + pub fn upsert_connection( + &mut self, + room: RoomKey, + connection: &ConnectionId, + user_id: UserId, + now: Millis, + ) -> Result<&mut ConnectionLedger, AdmissionRefusal> { + let limits = self.limits; + let tracked = self.tracked_connections; + let room_entry = self.ensure_room(room, now)?; + + let existing = room_entry + .connections + .iter() + .position(|entry| &entry.connection == connection); + + let index = match existing { + Some(index) => index, + None => { + if room_entry.connections.len() >= limits.max_connections_per_room { + room_entry.partially_unreadable = true; + return Err(AdmissionRefusal::RoomConnectionCapacity); + } + if tracked >= limits.max_tracked_connections { + return Err(AdmissionRefusal::ConnectionCapacity); + } + room_entry.connections.push(ConnectionLedger::new( + connection.clone(), + user_id, + now, + )); + self.tracked_connections = tracked.saturating_add(1); + self.rooms + .get(&room) + .map(|entry| entry.connections.len().saturating_sub(1)) + .unwrap_or(0) + } + }; + + let entry = self + .rooms + .get_mut(&room) + .and_then(|room_ledger| room_ledger.connections.get_mut(index)) + .expect("the connection was just admitted"); + entry.user_id = user_id; + entry.last_seen = now; + Ok(entry) + } + + pub fn tracked_rooms(&self) -> usize { + self.rooms.len() + } + + pub const fn tracked_connections(&self) -> usize { + self.tracked_connections + } + + pub fn tracked_bytes(&self) -> u64 { + (self.rooms.len() as u64) + .saturating_mul(ROOM_ENTRY_BYTES) + .saturating_add( + (self.tracked_connections as u64).saturating_mul(CONNECTION_ENTRY_BYTES), + ) + } + + pub const fn evicted_rooms_total(&self) -> u64 { + self.evicted_rooms_total + } + + pub const fn evicted_connections_total(&self) -> u64 { + self.evicted_connections_total + } + + pub fn divergent_connections(&self) -> usize { + self.rooms + .values() + .map(RoomLedger::divergent_connections) + .sum() + } + + pub fn divergent_fraction(&self) -> f64 { + if self.tracked_connections == 0 { + return 0.0; + } + self.divergent_connections() as f64 / self.tracked_connections as f64 + } + + pub fn reset_all_corroborations(&mut self) { + for room in self.rooms.values_mut() { + room.reset_corroborations(); + } + } + + pub fn prune_retired(&mut self, now: Millis) -> usize { + let grace = self.limits.retired_grace_ms; + let mut dropped = 0; + + for room in self.rooms.values_mut() { + room.connections.retain(|entry| match entry.state { + ConnectionState::Retired { at } => { + let expired = now.saturating_since(at) >= grace; + if expired { + dropped += 1; + } + !expired + } + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::GatewayOnly { .. } + | ConnectionState::MediaOnly { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::ActionTaken { .. } + | ConnectionState::Wedged { .. } => true, + }); + } + + self.tracked_connections = self.tracked_connections.saturating_sub(dropped); + self.evicted_connections_total = self + .evicted_connections_total + .saturating_add(dropped as u64); + dropped + } + + pub fn drop_empty_rooms(&mut self) -> usize { + let before = self.rooms.len(); + self.rooms.retain(|_, room| !room.connections.is_empty()); + before.saturating_sub(self.rooms.len()) + } + + pub fn evict_to_budget(&mut self) -> usize { + let mut evicted = 0; + + while self.tracked_bytes() > self.limits.memory_budget_bytes + || self.rooms.len() > self.limits.max_tracked_rooms + || self.tracked_connections > self.limits.max_tracked_connections + { + let Some(victim) = self.coldest_room() else { + break; + }; + if self.evict_room(&victim) { + evicted += 1; + } else { + break; + } + } + + evicted + } + + pub fn evict_room(&mut self, room: &RoomKey) -> bool { + match self.rooms.remove(room) { + None => false, + Some(entry) => { + self.tracked_connections = self + .tracked_connections + .saturating_sub(entry.connections.len()); + self.evicted_connections_total = self + .evicted_connections_total + .saturating_add(entry.connections.len() as u64); + self.evicted_rooms_total = self.evicted_rooms_total.saturating_add(1); + true + } + } + } + + pub fn coldest_room(&self) -> Option { + let cold = self + .rooms + .values() + .filter(|room| room.is_cold()) + .min_by_key(|room| (room.last_seen(), room.room())) + .map(RoomLedger::room); + + cold.or_else(|| { + self.rooms + .values() + .min_by_key(|room| (room.last_seen(), room.room())) + .map(RoomLedger::room) + }) + } + + pub fn record_action_issued( + &mut self, + key: &ConnectionKey, + kind: ActionKind, + at: Millis, + authorizing_turn: TurnId, + ) -> bool { + let Some(entry) = self.connection_mut(key) else { + return false; + }; + + let (prior, since) = match &entry.state { + ConnectionState::GatewayOnly { since, .. } => (PriorDivergence::GatewayOnly, *since), + ConnectionState::MediaOnly { since, .. } => (PriorDivergence::MediaOnly, *since), + ConnectionState::Nascent + | ConnectionState::Consistent + | ConnectionState::PendingJoin { .. } + | ConnectionState::Repairing { .. } + | ConnectionState::ActionTaken { .. } + | ConnectionState::Wedged { .. } + | ConnectionState::Retired { .. } => return false, + }; + + entry.action_attempts = entry.action_attempts.saturating_add(1); + entry.last_action_at = Some(at); + entry.state = ConnectionState::ActionTaken { + kind, + at, + authorizing_turn, + prior, + since, + }; + true + } + + pub fn refund_action_attempt(&mut self, key: &ConnectionKey) -> bool { + let Some(entry) = self.connection_mut(key) else { + return false; + }; + if entry.action_attempts == 0 { + return false; + } + entry.action_attempts = entry.action_attempts.saturating_sub(1); + true + } + + pub fn record_repair_issued(&mut self, key: &ConnectionKey, at: Millis) -> bool { + let Some(entry) = self.connection_mut(key) else { + return false; + }; + entry.repair_attempts = entry.repair_attempts.saturating_add(1); + entry.last_action_at = Some(at); + if let ConnectionState::Repairing { last_attempt, .. } = &mut entry.state { + *last_attempt = at; + } + true + } + + pub fn record_repair_verdict(&mut self, key: &ConnectionKey, verdict: RepairVerdict) -> bool { + let Some(entry) = self.connection_mut(key) else { + return false; + }; + entry.last_repair_verdict = Some(verdict); + true + } + + pub fn record_confirm_issued(&mut self, key: &ConnectionKey, at: Millis) -> bool { + let Some(entry) = self.connection_mut(key) else { + return false; + }; + entry.last_action_at = Some(at); + if let ConnectionState::PendingJoin { last_confirm, .. } = &mut entry.state { + *last_confirm = at; + } + true + } +} diff --git a/fluxer_recon/src/lib.rs b/fluxer_recon/src/lib.rs new file mode 100644 index 000000000..c99b69f1a --- /dev/null +++ b/fluxer_recon/src/lib.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub mod actuate; +pub mod budget; +pub mod census; +pub mod clock; +pub mod config; +pub mod control; +pub mod decide; +pub mod discovery; +pub mod evidence; +pub mod gateway; +pub mod guards; +pub mod health; +pub mod ids; +pub mod ledger; +pub mod livekit; +pub mod metrics; +pub mod names; +pub mod observe; +pub mod runtime; +pub mod schedule; +pub mod service; +pub mod singleton; +pub mod suspicion; +pub mod topology; +pub mod turn; +pub mod webhook; diff --git a/fluxer_recon/src/livekit/auth.rs b/fluxer_recon/src/livekit/auth.rs new file mode 100644 index 000000000..61dccec42 --- /dev/null +++ b/fluxer_recon/src/livekit/auth.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use base64::prelude::*; +use hmac::{Hmac, KeyInit, Mac}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; + +use crate::ids::ApiSecret; + +type HmacSha256 = Hmac; + +pub const ADMIN_TOKEN_TTL_SECONDS: u64 = 600; + +const HEADER_JSON: &str = r#"{"alg":"HS256","typ":"JWT"}"#; +const HEADER_ALG: &str = "HS256"; +const HEADER_TYP: &str = "JWT"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum TokenError { + #[error("token is not three dot-separated parts")] + Malformed, + #[error("token header is not an HS256 JWT header")] + UnsupportedHeader, + #[error("token carries invalid base64url")] + BadBase64, + #[error("token carries invalid json")] + BadJson, + #[error("token signature does not verify")] + BadSignature, + #[error("token issuer does not match the api key")] + IssuerMismatch, + #[error("token has expired")] + Expired, + #[error("token is not valid yet")] + NotYetValid, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum VideoGrant { + RoomList, + RoomAdmin { room: String }, +} + +impl VideoGrant { + pub fn claim(&self) -> VideoGrantClaim { + match self { + Self::RoomList => VideoGrantClaim { + room_list: Some(true), + room_admin: None, + room: None, + }, + Self::RoomAdmin { room } => VideoGrantClaim { + room_list: None, + room_admin: Some(true), + room: Some(room.clone()), + }, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VideoGrantClaim { + #[serde(rename = "roomList", default, skip_serializing_if = "Option::is_none")] + pub room_list: Option, + #[serde(rename = "roomAdmin", default, skip_serializing_if = "Option::is_none")] + pub room_admin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub room: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminClaims { + pub iss: String, + pub exp: u64, + pub nbf: u64, + pub video: VideoGrantClaim, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +struct TokenHeader { + alg: String, + typ: String, +} + +pub fn admin_claims(api_key: &str, grant: &VideoGrant, now_unix_seconds: u64) -> AdminClaims { + AdminClaims { + iss: api_key.to_owned(), + exp: now_unix_seconds.saturating_add(ADMIN_TOKEN_TTL_SECONDS), + nbf: 0, + video: grant.claim(), + } +} + +pub fn mint_admin_token( + api_key: &str, + api_secret: &ApiSecret, + grant: &VideoGrant, + now_unix_seconds: u64, +) -> String { + encode_admin_token(&admin_claims(api_key, grant, now_unix_seconds), api_secret) +} + +pub fn encode_admin_token(claims: &AdminClaims, api_secret: &ApiSecret) -> String { + let header = BASE64_URL_SAFE_NO_PAD.encode(HEADER_JSON.as_bytes()); + let payload = BASE64_URL_SAFE_NO_PAD + .encode(serde_json::to_vec(claims).expect("admin claims serialise to json")); + let signing_input = format!("{header}.{payload}"); + let signature = sign(signing_input.as_bytes(), api_secret); + format!( + "{signing_input}.{}", + BASE64_URL_SAFE_NO_PAD.encode(signature) + ) +} + +pub fn verify_admin_token( + token: &str, + api_key: &str, + api_secret: &ApiSecret, + now_unix_seconds: u64, +) -> Result { + let mut parts = token.split('.'); + let (Some(header_b64), Some(payload_b64), Some(signature_b64), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(TokenError::Malformed); + }; + + let header_bytes = decode_part(header_b64)?; + let header: TokenHeader = + serde_json::from_slice(&header_bytes).map_err(|_| TokenError::BadJson)?; + if header.alg != HEADER_ALG || header.typ != HEADER_TYP { + return Err(TokenError::UnsupportedHeader); + } + + let signature = decode_part(signature_b64)?; + let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) + .expect("hmac accepts any key length"); + mac.update(header_b64.as_bytes()); + mac.update(b"."); + mac.update(payload_b64.as_bytes()); + mac.verify_slice(&signature) + .map_err(|_| TokenError::BadSignature)?; + + let payload_bytes = decode_part(payload_b64)?; + let claims: AdminClaims = + serde_json::from_slice(&payload_bytes).map_err(|_| TokenError::BadJson)?; + if claims.iss != api_key { + return Err(TokenError::IssuerMismatch); + } + if now_unix_seconds >= claims.exp { + return Err(TokenError::Expired); + } + if now_unix_seconds < claims.nbf { + return Err(TokenError::NotYetValid); + } + Ok(claims) +} + +fn decode_part(part: &str) -> Result, TokenError> { + BASE64_URL_SAFE_NO_PAD + .decode(part.as_bytes()) + .map_err(|_| TokenError::BadBase64) +} + +fn sign(input: &[u8], api_secret: &ApiSecret) -> Vec { + let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) + .expect("hmac accepts any key length"); + mac.update(input); + mac.finalize().into_bytes().to_vec() +} diff --git a/fluxer_recon/src/livekit/mod.rs b/fluxer_recon/src/livekit/mod.rs new file mode 100644 index 000000000..8b5c72fe7 --- /dev/null +++ b/fluxer_recon/src/livekit/mod.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub mod auth; +pub mod twirp; + +use std::future::Future; + +use crate::ids::{ApiSecret, Location, RoomKey}; +use crate::names::ParticipantIdentity; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LiveKitFault { + Retryable { status: u16 }, + NotFound, + AuthFailed, + Other { status: u16 }, + Transport { timeout: bool }, + ServerMissing, + MalformedEndpoint, +} + +impl LiveKitFault { + pub const fn label(self) -> &'static str { + match self { + Self::Retryable { .. } => "retryable", + Self::NotFound => "not_found", + Self::AuthFailed => "auth_failed", + Self::Other { .. } => "other", + Self::Transport { timeout: true } => "transport_timeout", + Self::Transport { timeout: false } => "transport_refused", + Self::ServerMissing => "server_missing", + Self::MalformedEndpoint => "malformed_endpoint", + } + } + + pub const fn is_retryable(self) -> bool { + match self { + Self::Retryable { .. } | Self::Transport { .. } => true, + Self::NotFound + | Self::AuthFailed + | Self::Other { .. } + | Self::ServerMissing + | Self::MalformedEndpoint => false, + } + } + + pub const fn is_auth_failure(self) -> bool { + matches!(self, Self::AuthFailed) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReadResult { + Read(T), + Unreadable(LiveKitFault), +} + +impl ReadResult { + pub fn is_readable(&self) -> bool { + matches!(self, Self::Read(_)) + } + + pub fn as_read(&self) -> Option<&T> { + match self { + Self::Read(value) => Some(value), + Self::Unreadable(_) => None, + } + } + + pub fn into_read(self) -> Option { + match self { + Self::Read(value) => Some(value), + Self::Unreadable(_) => None, + } + } + + pub fn fault(&self) -> Option { + match self { + Self::Read(_) => None, + Self::Unreadable(fault) => Some(*fault), + } + } + + pub fn map(self, transform: impl FnOnce(T) -> U) -> ReadResult { + match self { + Self::Read(value) => ReadResult::Read(transform(value)), + Self::Unreadable(fault) => ReadResult::Unreadable(fault), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum RemoveOutcome { + Removed, + AlreadyGone, + Failed(LiveKitFault), +} + +impl RemoveOutcome { + pub const fn is_success(self) -> bool { + match self { + Self::Removed | Self::AlreadyGone => true, + Self::Failed(_) => false, + } + } + + pub const fn fault(self) -> Option { + match self { + Self::Removed | Self::AlreadyGone => None, + Self::Failed(fault) => Some(fault), + } + } + + pub const fn label(self) -> &'static str { + match self { + Self::Removed => "removed", + Self::AlreadyGone => "already_gone", + Self::Failed(_) => "failed", + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ParticipantState { + #[default] + Joining, + Joined, + Active, + Disconnected, + Unrecognised, +} + +impl ParticipantState { + pub const fn label(self) -> &'static str { + match self { + Self::Joining => "joining", + Self::Joined => "joined", + Self::Active => "active", + Self::Disconnected => "disconnected", + Self::Unrecognised => "unrecognised", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParticipantRecord { + pub identity: Box, + pub joined_at_unix_seconds: u64, + pub state: ParticipantState, + pub is_publisher: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServerCredentials { + pub location: Location, + pub endpoint: Box, + pub api_key: Box, + pub api_secret: ApiSecret, +} + +pub trait LiveKitApi { + fn list_rooms( + &self, + location: &Location, + ) -> impl Future>>> + Send; + + fn list_participants( + &self, + location: &Location, + room: RoomKey, + ) -> impl Future>> + Send; + + fn remove_participant( + &self, + location: &Location, + room: RoomKey, + identity: &ParticipantIdentity, + ) -> impl Future + Send; +} diff --git a/fluxer_recon/src/livekit/twirp.rs b/fluxer_recon/src/livekit/twirp.rs new file mode 100644 index 000000000..76e8a4cbe --- /dev/null +++ b/fluxer_recon/src/livekit/twirp.rs @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::HashMap; +use std::fmt; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::de::{self, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::ids::{ApiSecret, Location, RoomKey}; +use crate::livekit::auth::{VideoGrant, mint_admin_token}; +use crate::livekit::{ + LiveKitApi, LiveKitFault, ParticipantRecord, ParticipantState, ReadResult, RemoveOutcome, + ServerCredentials, +}; +use crate::names::{ParticipantIdentity, format_participant_identity, format_room_name}; + +pub const TWIRP_PREFIX: &str = "/twirp/livekit.RoomService/"; +pub const METHOD_LIST_ROOMS: &str = "ListRooms"; +pub const METHOD_LIST_PARTICIPANTS: &str = "ListParticipants"; +pub const METHOD_REMOVE_PARTICIPANT: &str = "RemoveParticipant"; + +const MAX_IDLE_CONNECTIONS_PER_HOST: usize = 1; +const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(90); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TwirpTimeouts { + pub connect: Duration, + pub request: Duration, +} + +impl TwirpTimeouts { + pub const DEFAULT: Self = Self { + connect: Duration::from_secs(3), + request: Duration::from_secs(5), + }; +} + +impl Default for TwirpTimeouts { + fn default() -> Self { + Self::DEFAULT + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum EndpointError { + #[error("endpoint scheme is not ws, wss, http or https")] + UnsupportedScheme, + #[error("endpoint names no host")] + MissingHost, + #[error("endpoint carries a query string or fragment")] + QueryOrFragment, + #[error("endpoint contains whitespace")] + ContainsWhitespace, +} + +pub fn twirp_base(endpoint: &str) -> Result { + if endpoint.contains(char::is_whitespace) { + return Err(EndpointError::ContainsWhitespace); + } + if endpoint.contains('?') || endpoint.contains('#') { + return Err(EndpointError::QueryOrFragment); + } + + let rewritten = if let Some(rest) = endpoint.strip_prefix("wss://") { + format!("https://{rest}") + } else if let Some(rest) = endpoint.strip_prefix("ws://") { + format!("http://{rest}") + } else if endpoint.starts_with("https://") || endpoint.starts_with("http://") { + endpoint.to_owned() + } else { + return Err(EndpointError::UnsupportedScheme); + }; + + let authority_and_path = rewritten + .split_once("://") + .expect("a rewritten endpoint always carries a scheme separator") + .1; + let host = authority_and_path.split('/').next().unwrap_or_default(); + if host.is_empty() { + return Err(EndpointError::MissingHost); + } + + Ok(rewritten.trim_end_matches('/').to_owned()) +} + +pub fn twirp_url(endpoint: &str, method: &str) -> Result { + twirp_base(endpoint).map(|base| format!("{base}{TWIRP_PREFIX}{method}")) +} + +pub const fn classify_status(status: u16) -> LiveKitFault { + match status { + 401 | 403 => LiveKitFault::AuthFailed, + 404 => LiveKitFault::NotFound, + 500..=599 => LiveKitFault::Retryable { status }, + other => LiveKitFault::Other { status: other }, + } +} + +#[derive(Serialize)] +struct ListRoomsRequest { + names: [&'static str; 0], +} + +#[derive(Serialize)] +struct RoomRequest<'a> { + room: &'a str, +} + +#[derive(Serialize)] +struct RemoveParticipantRequest<'a> { + room: &'a str, + identity: &'a str, +} + +#[derive(Debug, Deserialize)] +struct ListRoomsResponse { + #[serde(default)] + rooms: Vec, +} + +#[derive(Debug, Deserialize)] +struct RoomWire { + #[serde(default)] + name: String, +} + +#[derive(Debug, Deserialize)] +struct ListParticipantsResponse { + #[serde(default)] + participants: Vec, +} + +#[derive(Debug, Deserialize)] +struct ParticipantWire { + #[serde(default)] + identity: String, + #[serde( + rename = "joinedAt", + alias = "joined_at", + default, + deserialize_with = "protojson_int64" + )] + joined_at: u64, + #[serde(default)] + state: ParticipantState, + #[serde(rename = "isPublisher", alias = "is_publisher", default)] + is_publisher: bool, +} + +fn protojson_int64<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + struct Int64Visitor; + + impl Visitor<'_> for Int64Visitor { + type Value = u64; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a protojson int64 as a json number or a decimal string") + } + + fn visit_u64(self, value: u64) -> Result { + Ok(value) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(u64::try_from(value).unwrap_or_default()) + } + + fn visit_f64(self, value: f64) -> Result { + Err(E::custom(format!("int64 encoded as a float: {value}"))) + } + + fn visit_str(self, value: &str) -> Result { + value + .parse::() + .map(|seconds| u64::try_from(seconds).unwrap_or_default()) + .map_err(|_| E::custom(format!("int64 string is not decimal: {value}"))) + } + } + + deserializer.deserialize_any(Int64Visitor) +} + +impl<'de> Deserialize<'de> for ParticipantState { + fn deserialize>(deserializer: D) -> Result { + struct StateVisitor; + + impl Visitor<'_> for StateVisitor { + type Value = ParticipantState; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a protobuf enum as its name or its number") + } + + fn visit_u64(self, value: u64) -> Result { + Ok(match value { + 0 => ParticipantState::Joining, + 1 => ParticipantState::Joined, + 2 => ParticipantState::Active, + 3 => ParticipantState::Disconnected, + _ => ParticipantState::Unrecognised, + }) + } + + fn visit_i64(self, value: i64) -> Result { + match u64::try_from(value) { + Ok(value) => self.visit_u64(value), + Err(_) => Ok(ParticipantState::Unrecognised), + } + } + + fn visit_str(self, value: &str) -> Result { + Ok(match value { + "JOINING" => ParticipantState::Joining, + "JOINED" => ParticipantState::Joined, + "ACTIVE" => ParticipantState::Active, + "DISCONNECTED" => ParticipantState::Disconnected, + _ => ParticipantState::Unrecognised, + }) + } + } + + deserializer.deserialize_any(StateVisitor) + } +} + +fn decode_rooms(body: &str) -> Result>, serde_json::Error> { + let response: ListRoomsResponse = serde_json::from_str(body)?; + Ok(response + .rooms + .into_iter() + .filter(|room| !room.name.is_empty()) + .map(|room| Box::from(room.name.as_str())) + .collect()) +} + +fn decode_participants(body: &str) -> Result, serde_json::Error> { + let response: ListParticipantsResponse = serde_json::from_str(body)?; + Ok(response + .participants + .into_iter() + .map(|participant| ParticipantRecord { + identity: Box::from(participant.identity.as_str()), + joined_at_unix_seconds: participant.joined_at, + state: participant.state, + is_publisher: participant.is_publisher, + }) + .collect()) +} + +#[derive(Debug)] +enum HttpOutcome { + Success { status: u16, body: String }, + Status { status: u16 }, + Transport { fault: LiveKitFault }, +} + +fn transport_fault(error: &reqwest::Error) -> LiveKitFault { + if error.is_connect() { + return LiveKitFault::Transport { timeout: false }; + } + if error.is_timeout() { + return LiveKitFault::Transport { timeout: true }; + } + LiveKitFault::Transport { timeout: false } +} + +fn current_unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +#[derive(Debug)] +struct ServerClient { + base: String, + api_key: Box, + api_secret: ApiSecret, + http: reqwest::Client, +} + +impl ServerClient { + fn url(&self, method: &str) -> String { + format!("{}{TWIRP_PREFIX}{method}", self.base) + } + + async fn post(&self, method: &str, grant: &VideoGrant, body: &impl Serialize) -> HttpOutcome { + let token = mint_admin_token( + &self.api_key, + &self.api_secret, + grant, + current_unix_seconds(), + ); + let response = match self + .http + .post(self.url(method)) + .bearer_auth(token) + .json(body) + .send() + .await + { + Ok(response) => response, + Err(error) => { + return HttpOutcome::Transport { + fault: transport_fault(&error), + }; + } + }; + + let status = response.status(); + if !status.is_success() { + return HttpOutcome::Status { + status: status.as_u16(), + }; + } + match response.text().await { + Ok(body) => HttpOutcome::Success { + status: status.as_u16(), + body, + }, + Err(error) => HttpOutcome::Transport { + fault: transport_fault(&error), + }, + } + } +} + +#[derive(Debug)] +enum ServerEntry { + Ready(Box), + Unusable(LiveKitFault), +} + +#[derive(Debug)] +pub struct TwirpLiveKit { + servers: HashMap, + timeouts: TwirpTimeouts, +} + +impl TwirpLiveKit { + pub fn new(servers: Vec, timeouts: TwirpTimeouts) -> anyhow::Result { + let mut adapter = Self { + servers: HashMap::new(), + timeouts, + }; + adapter.replace_servers(servers)?; + Ok(adapter) + } + + pub fn replace_servers(&mut self, servers: Vec) -> anyhow::Result<()> { + let mut replacement: HashMap = HashMap::with_capacity(servers.len()); + for credentials in servers { + let entry = match twirp_base(&credentials.endpoint) { + Ok(base) => ServerEntry::Ready(Box::new(ServerClient { + base, + api_key: credentials.api_key, + api_secret: credentials.api_secret, + http: self.build_http_client()?, + })), + Err(error) => { + tracing::warn!( + location = %credentials.location, + %error, + "livekit endpoint is unusable" + ); + ServerEntry::Unusable(LiveKitFault::MalformedEndpoint) + } + }; + replacement.insert(credentials.location, entry); + } + self.servers = replacement; + Ok(()) + } + + fn build_http_client(&self) -> anyhow::Result { + Ok(reqwest::Client::builder() + .connect_timeout(self.timeouts.connect) + .timeout(self.timeouts.request) + .pool_max_idle_per_host(MAX_IDLE_CONNECTIONS_PER_HOST) + .pool_idle_timeout(POOL_IDLE_TIMEOUT) + .build()?) + } + + fn client(&self, location: &Location) -> Result<&ServerClient, LiveKitFault> { + match self.servers.get(location) { + None => Err(LiveKitFault::ServerMissing), + Some(ServerEntry::Unusable(fault)) => Err(*fault), + Some(ServerEntry::Ready(client)) => Ok(client), + } + } +} + +impl LiveKitApi for TwirpLiveKit { + async fn list_rooms(&self, location: &Location) -> ReadResult>> { + let client = match self.client(location) { + Ok(client) => client, + Err(fault) => return ReadResult::Unreadable(fault), + }; + + let outcome = client + .post( + METHOD_LIST_ROOMS, + &VideoGrant::RoomList, + &ListRoomsRequest { names: [] }, + ) + .await; + + match outcome { + HttpOutcome::Transport { fault } => ReadResult::Unreadable(fault), + HttpOutcome::Status { status } => ReadResult::Unreadable(classify_status(status)), + HttpOutcome::Success { status, body } => match decode_rooms(&body) { + Ok(rooms) => ReadResult::Read(rooms), + Err(_) => ReadResult::Unreadable(LiveKitFault::Other { status }), + }, + } + } + + async fn list_participants( + &self, + location: &Location, + room: RoomKey, + ) -> ReadResult> { + let client = match self.client(location) { + Ok(client) => client, + Err(fault) => return ReadResult::Unreadable(fault), + }; + + let room_name = format_room_name(room); + let outcome = client + .post( + METHOD_LIST_PARTICIPANTS, + &VideoGrant::RoomAdmin { + room: room_name.clone(), + }, + &RoomRequest { room: &room_name }, + ) + .await; + + match outcome { + HttpOutcome::Transport { fault } => ReadResult::Unreadable(fault), + HttpOutcome::Status { status } => ReadResult::Unreadable(classify_status(status)), + HttpOutcome::Success { status, body } => match decode_participants(&body) { + Ok(participants) => ReadResult::Read(participants), + Err(_) => ReadResult::Unreadable(LiveKitFault::Other { status }), + }, + } + } + + async fn remove_participant( + &self, + location: &Location, + room: RoomKey, + identity: &ParticipantIdentity, + ) -> RemoveOutcome { + let client = match self.client(location) { + Ok(client) => client, + Err(fault) => return RemoveOutcome::Failed(fault), + }; + + let room_name = format_room_name(room); + let participant_identity = format_participant_identity(identity); + let outcome = client + .post( + METHOD_REMOVE_PARTICIPANT, + &VideoGrant::RoomAdmin { + room: room_name.clone(), + }, + &RemoveParticipantRequest { + room: &room_name, + identity: &participant_identity, + }, + ) + .await; + + match outcome { + HttpOutcome::Transport { fault } => RemoveOutcome::Failed(fault), + HttpOutcome::Success { .. } => RemoveOutcome::Removed, + HttpOutcome::Status { status: 404 } => RemoveOutcome::AlreadyGone, + HttpOutcome::Status { status } => RemoveOutcome::Failed(classify_status(status)), + } + } +} diff --git a/fluxer_recon/src/main.rs b/fluxer_recon/src/main.rs new file mode 100644 index 000000000..e09ab441f --- /dev/null +++ b/fluxer_recon/src/main.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::net::SocketAddr; +use std::sync::Arc; + +use tokio::task::JoinSet; + +use fluxer_recon::clock::{SharedClock, SystemClock}; +use fluxer_recon::config::ReconConfig; +use fluxer_recon::control::{CONTROL_SUBJECT, INSTANCE_SUBJECT, run_control}; +use fluxer_recon::gateway::nats::NatsGateway; +use fluxer_recon::metrics::ReconMetrics; +use fluxer_recon::runtime::{ + InstanceIdentity, Shared, boot_assertions, monotonic_now, run_singleton, runtime_tasks, +}; +use fluxer_recon::service::{ + GuardedClock, ReconEngine, SharedFleet, build_livekit, connect_topology, internal_endpoint, + run_census, run_discovery, run_engine, +}; +use fluxer_recon::webhook::run_webhook; +use fluxer_svc::config::ServiceConfig; +use fluxer_svc::metrics::ServiceMetrics; +use fluxer_svc::shutdown::{DEFAULT_DRAIN_TIMEOUT, drain_with_timeout, wait_for_shutdown}; +use fluxer_svc::transport::NatsTransport; + +enum Exit { + Task(anyhow::Result<()>), + Signal, +} + +fn main() -> anyhow::Result<()> { + fluxer_svc::init_tracing(); + + let service = ServiceConfig::from_env()?; + let recon = ReconConfig::from_env()?; + boot_assertions(&service, &recon)?; + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(recon.worker_threads) + .max_blocking_threads(1) + .thread_name("recon") + .enable_all() + .build()?; + + runtime.block_on(serve(service, recon)) +} + +async fn serve(service: ServiceConfig, recon: ReconConfig) -> anyhow::Result<()> { + let transport = + NatsTransport::connect(&service.nats_url, service.nats_auth_token.as_deref()).await?; + + let clock: SharedClock = GuardedClock::shared(SystemClock::shared()); + let instance = InstanceIdentity::generate(); + let shared = Shared::new(recon.clone(), instance, monotonic_now()); + + let metrics = Arc::new(ServiceMetrics::with_additional_renderer( + ReconMetrics::renderer(shared.metrics()), + )); + metrics.init(); + + let topology = connect_topology(&service).await?; + let internal = internal_endpoint(&recon); + let fleet = SharedFleet::new(build_livekit(&[], &internal)?); + + tracing::info!( + service = service.service_name, + listen_addr = %service.listen_addr, + instance_id = shared.instance().id, + mode = recon.mode.as_str(), + worker_threads = recon.worker_threads, + tick_ms = recon.tick_ms, + coverage_target_ms = recon.coverage_target_ms, + topology_backend = topology.backend(), + livekit_internal_endpoint = internal.url.as_deref().unwrap_or("none"), + livekit_default_region = internal.default_region_id.as_deref().unwrap_or("none"), + control_subject = CONTROL_SUBJECT, + instance_subject = INSTANCE_SUBJECT, + tasks = ?runtime_tasks(&recon), + "starting recon service" + ); + + let mut tasks: JoinSet> = JoinSet::new(); + + let health_addr = service.listen_addr; + let health_serving = shared.readiness().flag(); + let health_metrics = metrics.clone(); + let health_name = service.service_name.clone(); + tasks.spawn(async move { + fluxer_svc::server::run_http(health_addr, health_serving, health_metrics, health_name).await + }); + + let engine = ReconEngine::new( + shared.clone(), + NatsGateway::new(transport.clone()), + fleet.clone(), + clock.clone(), + ); + tasks.spawn(async move { run_engine(engine).await }); + + let census_shared = shared.clone(); + let census_gateway = NatsGateway::new(transport.clone()); + let census_clock = clock.clone(); + tasks.spawn(async move { run_census(census_shared, census_gateway, census_clock).await }); + + let discovery_shared = shared.clone(); + let discovery_fleet = fleet.clone(); + let discovery_clock = clock.clone(); + tasks.spawn(async move { + run_discovery(discovery_shared, discovery_fleet, topology, discovery_clock).await + }); + + let control_shared = shared.clone(); + let control_transport = transport.clone(); + tasks.spawn(async move { run_control(control_shared, control_transport).await }); + + let singleton_shared = shared.clone(); + let singleton_transport = transport.clone(); + tasks.spawn(async move { run_singleton(singleton_shared, singleton_transport).await }); + + if recon.webhook_enabled { + let webhook_shared = shared.clone(); + let webhook_clock = clock.clone(); + let webhook_addr = SocketAddr::from((service.listen_addr.ip(), recon.webhook_port)); + tasks.spawn(async move { run_webhook(webhook_shared, webhook_clock, webhook_addr).await }); + } + + let exit = tokio::select! { + result = tasks.join_next() => Exit::Task(match result { + Some(Ok(Ok(()))) => Ok(()), + Some(Ok(Err(error))) => Err(error), + Some(Err(error)) => Err(error.into()), + None => Ok(()), + }), + () = wait_for_shutdown() => Exit::Signal, + }; + + shared.readiness().begin_shutdown(); + tasks.abort_all(); + drain_with_timeout( + async { while tasks.join_next().await.is_some() {} }, + DEFAULT_DRAIN_TIMEOUT, + ) + .await; + + match exit { + Exit::Task(result) => { + tracing::warn!("a recon task ended, shutting the service down"); + result + } + Exit::Signal => { + tracing::info!("recon shutdown complete"); + Ok(()) + } + } +} diff --git a/fluxer_recon/src/metrics.rs b/fluxer_recon/src/metrics.rs new file mode 100644 index 000000000..d8e4ad274 --- /dev/null +++ b/fluxer_recon/src/metrics.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; +use std::fmt::Write; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use fluxer_svc::metrics::AdditionalMetricsRenderer; + +use crate::decide::{Decision, DecisionAction}; +use crate::evidence::SideKind; +use crate::guards::{AbortReason, UnknownMediaCause}; +use crate::health::{ModeClamp, ReconMode, ServerHealth}; +use crate::ids::{Location, Scope}; +use crate::ledger::{ActionKind, RepairVerdict}; + +const PREFIX: &str = "fluxer_recon"; +const ORDERING: Ordering = Ordering::Relaxed; + +const DURATION_BUCKETS_MS: [u64; 13] = [ + 1, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, +]; + +pub const DECISION_LABELS: [&str; 5] = [ + "hold", + "confirm_connection", + "repair_state", + "remove_gateway_state", + "remove_participant", +]; + +pub const REPAIR_OUTCOMES: [&str; 4] = ["repaired", "no_change", "not_repairable", "failed"]; + +pub const CONFIRM_OUTCOMES: [&str; 4] = + ["confirmed", "already_confirmed", "call_not_found", "failed"]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MutationOutcome { + Issued, + Succeeded, + NoChange, + Failed, +} + +impl MutationOutcome { + pub const ALL: [Self; 4] = [Self::Issued, Self::Succeeded, Self::NoChange, Self::Failed]; + + pub const fn label(self) -> &'static str { + match self { + Self::Issued => "issued", + Self::Succeeded => "succeeded", + Self::NoChange => "no_change", + Self::Failed => "failed", + } + } +} + +pub const fn scope_label(scope: Scope) -> &'static str { + match scope { + Scope::Guild => "guild", + Scope::Dm => "dm", + } +} + +#[derive(Debug, Default)] +struct Counter(AtomicU64); + +impl Counter { + fn increment(&self) { + self.0.fetch_add(1, ORDERING); + } + + fn set(&self, value: u64) { + self.0.store(value, ORDERING); + } + + fn get(&self) -> u64 { + self.0.load(ORDERING) + } +} + +#[derive(Debug, Default)] +struct Gauge(AtomicU64); + +impl Gauge { + fn set(&self, value: f64) { + self.0.store(value.to_bits(), ORDERING); + } + + fn get(&self) -> f64 { + f64::from_bits(self.0.load(ORDERING)) + } +} + +#[derive(Debug, Default)] +struct LabelledSeries { + values: Mutex>, +} + +impl LabelledSeries { + fn seeded>(labels: I) -> Self { + let series = Self::default(); + for label in labels { + series.set(label, 0.0); + } + series + } + + fn lock(&self) -> std::sync::MutexGuard<'_, BTreeMap> { + self.values.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn increment(&self, labels: String) { + *self.lock().entry(labels).or_insert(0.0) += 1.0; + } + + fn set(&self, labels: String, value: f64) { + self.lock().insert(labels, value); + } + + fn get(&self, labels: &str) -> Option { + self.lock().get(labels).copied() + } + + fn render(&self, out: &mut String, name: &str, kind: &str) { + let values = self.lock(); + let _ = writeln!(out, "# TYPE {PREFIX}_{name} {kind}"); + for (labels, value) in values.iter() { + let _ = writeln!(out, "{PREFIX}_{name}{{{labels}}} {value}"); + } + } +} + +#[derive(Debug)] +struct DurationHistogram { + buckets: [AtomicU64; DURATION_BUCKETS_MS.len()], + count: AtomicU64, + sum_ms: AtomicU64, +} + +impl Default for DurationHistogram { + fn default() -> Self { + Self { + buckets: [const { AtomicU64::new(0) }; DURATION_BUCKETS_MS.len()], + count: AtomicU64::new(0), + sum_ms: AtomicU64::new(0), + } + } +} + +impl DurationHistogram { + fn observe(&self, ms: u64) { + for (index, upper) in DURATION_BUCKETS_MS.iter().copied().enumerate() { + if ms <= upper { + self.buckets[index].fetch_add(1, ORDERING); + break; + } + } + self.count.fetch_add(1, ORDERING); + self.sum_ms.fetch_add(ms, ORDERING); + } + + fn render(&self, out: &mut String, name: &str) { + let _ = writeln!(out, "# TYPE {PREFIX}_{name} histogram"); + let mut cumulative = 0; + for (index, upper) in DURATION_BUCKETS_MS.iter().copied().enumerate() { + cumulative += self.buckets[index].load(ORDERING); + let _ = writeln!(out, "{PREFIX}_{name}_bucket{{le=\"{upper}\"}} {cumulative}"); + } + let count = self.count.load(ORDERING); + let _ = writeln!(out, "{PREFIX}_{name}_bucket{{le=\"+Inf\"}} {count}"); + let _ = writeln!(out, "{PREFIX}_{name}_sum {}", self.sum_ms.load(ORDERING)); + let _ = writeln!(out, "{PREFIX}_{name}_count {count}"); + } +} + +fn escape_label(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\\' => escaped.push_str("\\\\"), + '"' => escaped.push_str("\\\""), + '\n' => escaped.push_str("\\n"), + other => escaped.push(other), + } + } + escaped +} + +fn one_label(key: &str, value: &str) -> String { + format!("{key}=\"{}\"", escape_label(value)) +} + +fn two_labels(first: (&str, &str), second: (&str, &str)) -> String { + format!( + "{}=\"{}\",{}=\"{}\"", + first.0, + escape_label(first.1), + second.0, + escape_label(second.1) + ) +} + +fn three_labels(first: (&str, &str), second: (&str, &str), third: (&str, &str)) -> String { + format!( + "{}=\"{}\",{}=\"{}\",{}=\"{}\"", + first.0, + escape_label(first.1), + second.0, + escape_label(second.1), + third.0, + escape_label(third.1) + ) +} + +#[derive(Debug)] +pub struct ReconMetrics { + decisions: LabelledSeries, + evictions_aborted: LabelledSeries, + unknown_media_causes: LabelledSeries, + mutations: LabelledSeries, + repairs: LabelledSeries, + confirms: LabelledSeries, + unknown: LabelledSeries, + gateway_rpc: LabelledSeries, + livekit_calls: LabelledSeries, + server_health: LabelledSeries, + webhook_last_event_age_seconds: LabelledSeries, + mode: LabelledSeries, + mode_clamp: LabelledSeries, + ledger_evicted_total: Counter, + scheduler_starved_total: Counter, + tripped_total: Counter, + peer_detected_total: Counter, + rooms_unknown_ratio: Gauge, + oldest_unknown_room_seconds: Gauge, + rooms_no_candidates: Gauge, + wedged_connections: Gauge, + ledger_rooms: Gauge, + ledger_connections: Gauge, + ledger_bytes: Gauge, + coverage_period_seconds: Gauge, + room_staleness_seconds: Gauge, + topology_age_seconds: Gauge, + census_age_seconds: Gauge, + last_decision_age_seconds: Gauge, + turn_duration: DurationHistogram, + mutation_duration: DurationHistogram, +} + +impl Default for ReconMetrics { + fn default() -> Self { + Self::new() + } +} + +impl ReconMetrics { + pub fn new() -> Self { + Self { + decisions: LabelledSeries::seeded( + DECISION_LABELS + .iter() + .map(|decision| one_label("decision", decision)), + ), + evictions_aborted: LabelledSeries::seeded( + AbortReason::ALL + .iter() + .map(|reason| one_label("reason", reason.label())), + ), + unknown_media_causes: LabelledSeries::seeded( + UnknownMediaCause::ALL + .iter() + .map(|cause| one_label("cause", cause.label())), + ), + mutations: LabelledSeries::default(), + repairs: LabelledSeries::seeded( + REPAIR_OUTCOMES + .iter() + .map(|outcome| one_label("outcome", outcome)), + ), + confirms: LabelledSeries::seeded( + CONFIRM_OUTCOMES + .iter() + .map(|outcome| one_label("outcome", outcome)), + ), + unknown: LabelledSeries::default(), + gateway_rpc: LabelledSeries::default(), + livekit_calls: LabelledSeries::default(), + server_health: LabelledSeries::default(), + webhook_last_event_age_seconds: LabelledSeries::default(), + mode: LabelledSeries::seeded( + [ + ReconMode::Halted, + ReconMode::Observing, + ReconMode::Constructive, + ReconMode::Enforcing, + ] + .iter() + .map(|mode| one_label("mode", mode.as_str())), + ), + mode_clamp: LabelledSeries::seeded( + ModeClamp::ALL + .iter() + .map(|clamp| one_label("clamp", clamp.label())), + ), + ledger_evicted_total: Counter::default(), + scheduler_starved_total: Counter::default(), + tripped_total: Counter::default(), + peer_detected_total: Counter::default(), + rooms_unknown_ratio: Gauge::default(), + oldest_unknown_room_seconds: Gauge::default(), + rooms_no_candidates: Gauge::default(), + wedged_connections: Gauge::default(), + ledger_rooms: Gauge::default(), + ledger_connections: Gauge::default(), + ledger_bytes: Gauge::default(), + coverage_period_seconds: Gauge::default(), + room_staleness_seconds: Gauge::default(), + topology_age_seconds: Gauge::default(), + census_age_seconds: Gauge::default(), + last_decision_age_seconds: Gauge::default(), + turn_duration: DurationHistogram::default(), + mutation_duration: DurationHistogram::default(), + } + } + + pub fn renderer(metrics: &Arc) -> AdditionalMetricsRenderer { + let metrics = Arc::clone(metrics); + Arc::new(move |out: &mut String| metrics.render(out)) + } + + pub fn record_decision(&self, decision: &Decision) { + self.decisions + .increment(one_label("decision", decision.action.label())); + if let Some(reason) = decision.blocked_by + && decision.is_destructive() + { + self.evictions_aborted + .increment(one_label("reason", reason.label())); + } + } + + pub fn record_eviction_aborted(&self, reason: AbortReason) { + self.evictions_aborted + .increment(one_label("reason", reason.label())); + } + + pub fn record_unknown_media_cause(&self, cause: UnknownMediaCause) { + self.unknown_media_causes + .increment(one_label("cause", cause.label())); + } + + pub fn unknown_media_cause_count(&self, cause: UnknownMediaCause) -> u64 { + self.unknown_media_causes + .get(&one_label("cause", cause.label())) + .unwrap_or(0.0) as u64 + } + + pub fn record_mutation(&self, kind: ActionKind, scope: Scope, outcome: MutationOutcome) { + self.mutations.increment(three_labels( + ("kind", kind.label()), + ("scope", scope_label(scope)), + ("outcome", outcome.label()), + )); + } + + pub fn record_repair(&self, outcome: &str) { + self.repairs.increment(one_label("outcome", outcome)); + } + + pub fn record_repair_verdict(&self, verdict: RepairVerdict) { + self.record_repair(verdict.label()); + } + + pub fn record_confirm(&self, outcome: &str) { + self.confirms.increment(one_label("outcome", outcome)); + } + + pub fn record_unknown(&self, side: SideKind, reason: &str) { + self.unknown + .increment(two_labels(("side", side.label()), ("reason", reason))); + } + + pub fn record_gateway_rpc(&self, method: &str, outcome: &str) { + self.gateway_rpc + .increment(two_labels(("method", method), ("outcome", outcome))); + } + + pub fn record_livekit_call(&self, method: &str, outcome: &str) { + self.livekit_calls + .increment(two_labels(("method", method), ("outcome", outcome))); + } + + pub fn record_breaker_trip(&self) { + self.tripped_total.increment(); + } + + pub fn record_peer_detected(&self) { + self.peer_detected_total.increment(); + } + + pub fn record_scheduler_starvation(&self) { + self.scheduler_starved_total.increment(); + } + + pub fn observe_turn_duration(&self, ms: u64) { + self.turn_duration.observe(ms); + } + + pub fn observe_mutation_duration(&self, ms: u64) { + self.mutation_duration.observe(ms); + } + + pub fn set_mode(&self, effective: ReconMode) { + for mode in [ + ReconMode::Halted, + ReconMode::Observing, + ReconMode::Constructive, + ReconMode::Enforcing, + ] { + let value = f64::from(u8::from(mode == effective)); + self.mode.set(one_label("mode", mode.as_str()), value); + } + } + + pub fn set_mode_clamp(&self, clamp: ModeClamp, engaged: bool) { + self.mode_clamp.set( + one_label("clamp", clamp.label()), + f64::from(u8::from(engaged)), + ); + } + + pub fn set_server_health(&self, location: &Location, health: ServerHealth) { + self.server_health.set( + two_labels( + ("region", location.region.as_str()), + ("server", location.server.as_str()), + ), + f64::from(health.severity()), + ); + } + + pub fn set_webhook_last_event_age(&self, server: &str, seconds: f64) { + self.webhook_last_event_age_seconds + .set(one_label("server", server), seconds); + } + + pub fn set_ledger(&self, rooms: usize, connections: usize, bytes: u64, evicted_total: u64) { + self.ledger_rooms.set(rooms as f64); + self.ledger_connections.set(connections as f64); + self.ledger_bytes.set(bytes as f64); + self.ledger_evicted_total.set(evicted_total); + } + + pub fn set_rooms_unknown_ratio(&self, ratio: f64) { + self.rooms_unknown_ratio.set(ratio); + } + + pub fn set_oldest_unknown_room_seconds(&self, seconds: f64) { + self.oldest_unknown_room_seconds.set(seconds); + } + + pub fn set_rooms_no_candidates(&self, rooms: usize) { + self.rooms_no_candidates.set(rooms as f64); + } + + pub fn set_wedged_connections(&self, connections: usize) { + self.wedged_connections.set(connections as f64); + } + + pub fn set_coverage_period_seconds(&self, seconds: f64) { + self.coverage_period_seconds.set(seconds); + } + + pub fn set_room_staleness_seconds(&self, seconds: f64) { + self.room_staleness_seconds.set(seconds); + } + + pub fn set_topology_age_seconds(&self, seconds: f64) { + self.topology_age_seconds.set(seconds); + } + + pub fn set_census_age_seconds(&self, seconds: f64) { + self.census_age_seconds.set(seconds); + } + + pub fn set_last_decision_age_seconds(&self, seconds: f64) { + self.last_decision_age_seconds.set(seconds); + } + + pub fn eviction_aborted_count(&self, reason: AbortReason) -> u64 { + self.evictions_aborted + .get(&one_label("reason", reason.label())) + .unwrap_or(0.0) as u64 + } + + pub fn unknown_count(&self, side: SideKind, reason: &str) -> u64 { + self.unknown + .get(&two_labels(("side", side.label()), ("reason", reason))) + .unwrap_or(0.0) as u64 + } + + pub fn gateway_rpc_count(&self, method: &str, outcome: &str) -> u64 { + self.gateway_rpc + .get(&two_labels(("method", method), ("outcome", outcome))) + .unwrap_or(0.0) as u64 + } + + pub fn livekit_call_count(&self, method: &str, outcome: &str) -> u64 { + self.livekit_calls + .get(&two_labels(("method", method), ("outcome", outcome))) + .unwrap_or(0.0) as u64 + } + + pub fn decision_count(&self, action: &DecisionAction) -> u64 { + self.decisions + .get(&one_label("decision", action.label())) + .unwrap_or(0.0) as u64 + } + + pub fn peer_detected_total(&self) -> u64 { + self.peer_detected_total.get() + } + + pub fn tripped_total(&self) -> u64 { + self.tripped_total.get() + } + + pub fn scheduler_starved_total(&self) -> u64 { + self.scheduler_starved_total.get() + } + + pub fn ledger_evicted_total(&self) -> u64 { + self.ledger_evicted_total.get() + } + + pub fn render(&self, out: &mut String) { + self.decisions.render(out, "decisions_total", "counter"); + self.mutations.render(out, "mutations_total", "counter"); + self.evictions_aborted + .render(out, "evictions_aborted_total", "counter"); + self.unknown_media_causes + .render(out, "unknown_media_refusals_total", "counter"); + self.repairs.render(out, "repairs_total", "counter"); + self.confirms.render(out, "confirms_total", "counter"); + self.unknown.render(out, "unknown_total", "counter"); + self.gateway_rpc.render(out, "gateway_rpc_total", "counter"); + self.livekit_calls + .render(out, "livekit_calls_total", "counter"); + + render_counter(out, "ledger_evicted_total", &self.ledger_evicted_total); + render_counter( + out, + "scheduler_starved_total", + &self.scheduler_starved_total, + ); + render_counter(out, "tripped_total", &self.tripped_total); + render_counter(out, "peer_detected_total", &self.peer_detected_total); + + self.mode.render(out, "mode", "gauge"); + self.mode_clamp.render(out, "mode_clamp", "gauge"); + self.server_health.render(out, "server_health", "gauge"); + self.webhook_last_event_age_seconds + .render(out, "webhook_last_event_age_seconds", "gauge"); + + render_gauge(out, "rooms_unknown_ratio", &self.rooms_unknown_ratio); + render_gauge( + out, + "oldest_unknown_room_seconds", + &self.oldest_unknown_room_seconds, + ); + render_gauge(out, "rooms_no_candidates", &self.rooms_no_candidates); + render_gauge(out, "wedged_connections", &self.wedged_connections); + render_gauge(out, "ledger_rooms", &self.ledger_rooms); + render_gauge(out, "ledger_connections", &self.ledger_connections); + render_gauge(out, "ledger_bytes", &self.ledger_bytes); + render_gauge( + out, + "coverage_period_seconds", + &self.coverage_period_seconds, + ); + render_gauge(out, "room_staleness_seconds", &self.room_staleness_seconds); + render_gauge(out, "topology_age_seconds", &self.topology_age_seconds); + render_gauge(out, "census_age_seconds", &self.census_age_seconds); + render_gauge( + out, + "last_decision_age_seconds", + &self.last_decision_age_seconds, + ); + + self.turn_duration.render(out, "turn_duration_ms"); + self.mutation_duration.render(out, "mutation_duration_ms"); + } +} + +fn render_counter(out: &mut String, name: &str, counter: &Counter) { + let _ = writeln!(out, "# TYPE {PREFIX}_{name} counter"); + let _ = writeln!(out, "{PREFIX}_{name} {}", counter.get()); +} + +fn render_gauge(out: &mut String, name: &str, gauge: &Gauge) { + let _ = writeln!(out, "# TYPE {PREFIX}_{name} gauge"); + let _ = writeln!(out, "{PREFIX}_{name} {}", gauge.get()); +} diff --git a/fluxer_recon/src/names.rs b/fluxer_recon/src/names.rs new file mode 100644 index 000000000..6d9a6c845 --- /dev/null +++ b/fluxer_recon/src/names.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::ids::{ChannelId, ConnectionId, GuildId, IdError, RoomKey, UserId}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum NameError { + #[error("room name has {0} underscore-separated parts")] + RoomPartCount(usize), + #[error("room name has an unrecognised shape")] + RoomShape, + #[error("room name has a field that is not a plain decimal u64")] + RoomNumber, + #[error("participant identity has {0} underscore-separated parts")] + IdentityPartCount(usize), + #[error("participant identity has an unrecognised shape")] + IdentityShape, + #[error("participant identity has a field that is not a plain decimal u64")] + IdentityNumber, + #[error("participant identity carries an invalid connection id: {0}")] + IdentityConnection(IdError), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParticipantIdentity { + pub user_id: UserId, + pub connection_id: ConnectionId, +} + +fn parse_u64_strict(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse::().ok() +} + +pub fn format_room_name(room: RoomKey) -> String { + match room { + RoomKey::Guild { + guild_id, + channel_id, + } => format!("guild_{guild_id}_channel_{channel_id}"), + RoomKey::Dm { channel_id } => format!("dm_channel_{channel_id}"), + } +} + +pub fn parse_room_name(name: &str) -> Result { + let parts: Vec<&str> = name.split('_').collect(); + match parts.as_slice() { + ["guild", guild, "channel", channel] => Ok(RoomKey::Guild { + guild_id: GuildId::new(parse_u64_strict(guild).ok_or(NameError::RoomNumber)?), + channel_id: ChannelId::new(parse_u64_strict(channel).ok_or(NameError::RoomNumber)?), + }), + ["dm", "channel", channel] => Ok(RoomKey::Dm { + channel_id: ChannelId::new(parse_u64_strict(channel).ok_or(NameError::RoomNumber)?), + }), + [_, _, _] | [_, _, _, _] => Err(NameError::RoomShape), + other => Err(NameError::RoomPartCount(other.len())), + } +} + +pub fn format_participant_identity(identity: &ParticipantIdentity) -> String { + let ParticipantIdentity { + user_id, + connection_id, + } = identity; + format!("user_{user_id}_{connection_id}") +} + +pub fn parse_participant_identity(identity: &str) -> Result { + let parts: Vec<&str> = identity.split('_').collect(); + match parts.as_slice() { + ["user", user, connection] => Ok(ParticipantIdentity { + user_id: UserId::new(parse_u64_strict(user).ok_or(NameError::IdentityNumber)?), + connection_id: ConnectionId::new(connection).map_err(NameError::IdentityConnection)?, + }), + [_, _, _] => Err(NameError::IdentityShape), + other => Err(NameError::IdentityPartCount(other.len())), + } +} diff --git a/fluxer_recon/src/observe.rs b/fluxer_recon/src/observe.rs new file mode 100644 index 000000000..f88efa805 --- /dev/null +++ b/fluxer_recon/src/observe.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::VecDeque; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use serde::Serialize; + +use crate::decide::{ActionClass, Decision, DecisionSet}; +use crate::evidence::{ConnectionObservation, RoomObservation, SideAuthority}; +use crate::guards::AbortReason; +use crate::health::ReconMode; +use crate::ids::{Millis, RoomKey}; +use crate::ledger::ConnectionLedger; +use crate::metrics::scope_label; + +pub const DEFAULT_JOURNAL_CAPACITY: usize = 512; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Disposition { + Held, + Blocked(AbortReason), + WouldExecute, + Executed, +} + +impl Disposition { + pub const fn label(self) -> &'static str { + match self { + Self::Held => "held", + Self::Blocked(_) => "blocked", + Self::WouldExecute => "would_execute", + Self::Executed => "executed", + } + } + + pub const fn blocked_by(self) -> Option { + match self { + Self::Blocked(reason) => Some(reason), + Self::Held | Self::WouldExecute | Self::Executed => None, + } + } + + pub const fn mutates(self) -> bool { + matches!(self, Self::Executed) + } + + pub const fn is_counterfactual(self) -> bool { + matches!(self, Self::WouldExecute) + } +} + +pub const fn action_class_label(class: ActionClass) -> &'static str { + match class { + ActionClass::None => "none", + ActionClass::Constructive => "constructive", + ActionClass::Destructive => "destructive", + } +} + +pub const fn mode_allows(class: ActionClass, mode: ReconMode) -> bool { + match class { + ActionClass::None => true, + ActionClass::Constructive => mode.allows_constructive(), + ActionClass::Destructive => mode.allows_destructive(), + } +} + +pub fn disposition(decision: &Decision, mode: ReconMode) -> Disposition { + let class = decision.action.class(); + match (class, decision.blocked_by) { + (ActionClass::None, _) => Disposition::Held, + (_, Some(reason)) => Disposition::Blocked(reason), + (class, None) if mode_allows(class, mode) => Disposition::Executed, + (_, None) => Disposition::WouldExecute, + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct EvidenceChain { + pub gateway_presence: &'static str, + pub media_presence: &'static str, + pub gateway_unknown_reason: Option<&'static str>, + pub media_unknown_reason: Option<&'static str>, + pub gateway_authority: Option<&'static str>, + pub media_authority: Option<&'static str>, + pub gateway_siblings: Vec, + pub media_locations: Vec, + pub candidates: Vec, + pub gateway_state_has_connection_id: bool, + pub participant_joined_at_ms: Option, + pub user_has_pending_join: bool, + pub corroborations: u32, + pub divergence_since_ms: Option, + pub fingerprint: Option, + pub action_attempts: u32, + pub repair_attempts: u32, + pub last_repair_verdict: Option<&'static str>, +} + +const fn authority_reason(authority: &SideAuthority) -> Option<&'static str> { + match authority { + SideAuthority::Authoritative => None, + SideAuthority::Unknown(reason) => Some(reason.label()), + } +} + +pub fn evidence_chain( + observation: &RoomObservation, + connection: &ConnectionObservation, + ledger: Option<&ConnectionLedger>, + pending_join_skew_ms: u64, +) -> EvidenceChain { + EvidenceChain { + gateway_presence: connection.presence.gateway.label(), + media_presence: connection.presence.media.label(), + gateway_unknown_reason: connection + .presence + .gateway + .unknown_reason() + .map(|reason| reason.label()), + media_unknown_reason: connection + .presence + .media + .unknown_reason() + .map(|reason| reason.label()), + gateway_authority: authority_reason(&observation.authority.gateway), + media_authority: authority_reason(&observation.authority.media), + gateway_siblings: connection + .gateway_siblings + .iter() + .map(ToString::to_string) + .collect(), + media_locations: connection + .media_locations + .iter() + .map(ToString::to_string) + .collect(), + candidates: observation + .candidates + .locations() + .iter() + .map(ToString::to_string) + .collect(), + gateway_state_has_connection_id: connection.gateway_state_has_connection_id, + participant_joined_at_ms: connection.participant_joined_at.map(Millis::get), + user_has_pending_join: observation.user_has_pending_join( + connection.user_id, + observation.wall_at, + pending_join_skew_ms, + ), + corroborations: ledger.map_or(0, |entry| entry.state.corroborations()), + divergence_since_ms: ledger + .and_then(|entry| entry.state.divergence_since()) + .map(Millis::get), + fingerprint: ledger + .and_then(|entry| entry.state.stored_fingerprint()) + .map(|fingerprint| format!("{:016x}", fingerprint.get())), + action_attempts: ledger.map_or(0, |entry| u32::from(entry.action_attempts)), + repair_attempts: ledger.map_or(0, |entry| u32::from(entry.repair_attempts)), + last_repair_verdict: ledger + .and_then(|entry| entry.last_repair_verdict) + .map(|verdict| verdict.label()), + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct DecisionRecord { + #[serde(skip)] + pub room: RoomKey, + pub turn: u64, + pub at_ms: u64, + pub scope: &'static str, + pub guild_id: Option, + pub channel_id: String, + pub connection_id: String, + pub user_id: String, + pub from: &'static str, + pub to: &'static str, + pub action: &'static str, + pub action_class: &'static str, + pub reason: &'static str, + pub mode: &'static str, + pub disposition: &'static str, + pub blocked_by: Option<&'static str>, + pub would_mutate: bool, + pub evidence: EvidenceChain, +} + +impl DecisionRecord { + pub fn new( + decision: &Decision, + turn: crate::ids::TurnId, + at: Millis, + mode: ReconMode, + evidence: EvidenceChain, + ) -> Self { + let room = decision.connection.room; + let disposition = disposition(decision, mode); + Self { + room, + turn: turn.get(), + at_ms: at.get(), + scope: scope_label(room.scope()), + guild_id: room.guild_id().map(|guild| guild.to_string()), + channel_id: room.channel_id().to_string(), + connection_id: decision.connection.connection.to_string(), + user_id: decision.user_id.to_string(), + from: decision.from, + to: decision.to, + action: decision.action.label(), + action_class: action_class_label(decision.action.class()), + reason: decision.reason.label(), + mode: mode.as_str(), + disposition: disposition.label(), + blocked_by: disposition.blocked_by().map(AbortReason::label), + would_mutate: disposition.is_counterfactual(), + evidence, + } + } + + pub fn is_counterfactual(&self) -> bool { + self.would_mutate + } + + pub fn matches_room(&self, room: RoomKey) -> bool { + self.room == room + } + + pub fn to_json(&self) -> String { + serde_json::to_string(self) + .unwrap_or_else(|error| format!("{{\"serialisation_error\":\"{error}\"}}")) + } +} + +pub fn log_decision(record: &DecisionRecord) { + if record.would_mutate { + tracing::warn!( + room_scope = record.scope, + guild_id = record.guild_id.as_deref(), + channel_id = record.channel_id, + connection_id = record.connection_id, + user_id = record.user_id, + action = record.action, + reason = record.reason, + mode = record.mode, + turn = record.turn, + decision = record.to_json(), + "recon would have mutated" + ); + return; + } + + tracing::info!( + room_scope = record.scope, + guild_id = record.guild_id.as_deref(), + channel_id = record.channel_id, + connection_id = record.connection_id, + user_id = record.user_id, + action = record.action, + reason = record.reason, + mode = record.mode, + disposition = record.disposition, + turn = record.turn, + decision = record.to_json(), + "recon decision" + ); +} + +#[derive(Debug)] +pub struct DecisionJournal { + capacity: usize, + entries: Mutex>, +} + +impl Default for DecisionJournal { + fn default() -> Self { + Self::new(DEFAULT_JOURNAL_CAPACITY) + } +} + +impl DecisionJournal { + pub fn new(capacity: usize) -> Self { + Self { + capacity: capacity.max(1), + entries: Mutex::new(VecDeque::new()), + } + } + + fn lock(&self) -> MutexGuard<'_, VecDeque> { + self.entries.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub fn record(&self, record: DecisionRecord) { + let mut entries = self.lock(); + if entries.len() >= self.capacity { + entries.pop_front(); + } + entries.push_back(record); + } + + pub fn len(&self) -> usize { + self.lock().len() + } + + pub fn is_empty(&self) -> bool { + self.lock().is_empty() + } + + pub fn capacity(&self) -> usize { + self.capacity + } + + pub fn last_at_ms(&self) -> Option { + self.lock().back().map(|record| record.at_ms) + } + + pub fn recent(&self, limit: usize) -> Vec { + let entries = self.lock(); + entries.iter().rev().take(limit).cloned().collect() + } + + pub fn for_room(&self, room: RoomKey, limit: usize) -> Vec { + let entries = self.lock(); + entries + .iter() + .rev() + .filter(|record| record.matches_room(room)) + .take(limit) + .cloned() + .collect() + } + + pub fn counterfactual_count(&self) -> usize { + self.lock() + .iter() + .filter(|record| record.is_counterfactual()) + .count() + } +} + +pub struct ObservedSet<'a> { + pub set: &'a DecisionSet, + pub observation: &'a RoomObservation, + pub mode: ReconMode, + pub pending_join_skew_ms: u64, +} + +pub fn journal_decisions( + journal: &DecisionJournal, + observed: &ObservedSet<'_>, + ledger_of: impl Fn(&Decision) -> Option, +) -> Vec { + let mut recorded = Vec::with_capacity(observed.set.decisions.len()); + for decision in &observed.set.decisions { + let entry = ledger_of(decision); + let chain = match observed + .observation + .observation_for(&decision.connection.connection) + { + Some(connection) => evidence_chain( + observed.observation, + connection, + entry.as_ref(), + observed.pending_join_skew_ms, + ), + None => EvidenceChain::default(), + }; + let record = DecisionRecord::new( + decision, + observed.set.turn, + observed.set.at, + observed.mode, + chain, + ); + log_decision(&record); + journal.record(record.clone()); + recorded.push(record); + } + recorded +} diff --git a/fluxer_recon/src/runtime.rs b/fluxer_recon/src/runtime.rs new file mode 100644 index 000000000..557b5edce --- /dev/null +++ b/fluxer_recon/src/runtime.rs @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use fluxer_svc::config::ServiceConfig; + +use crate::budget::{BreakerEvent, BreakerLimits, Governor, GovernorLimits}; +use crate::census::{CensusCache, CensusLimits}; +use crate::config::ReconConfig; +use crate::decide::DecideParams; +use crate::discovery::{DirectoryLimits, PassCounters, RoomDirectory, ServerCliffWatch}; +use crate::health::{ModeClamp, PauseBackoff, ReconMode, ServerHealthMap, WarmupGate}; +use crate::ids::{Millis, RoomKey, TurnId}; +use crate::ledger::{ConnectionState, Ledger, LedgerLimits, MAP_ENTRY_OVERHEAD_BYTES}; +use crate::metrics::ReconMetrics; +use crate::observe::DecisionJournal; +use crate::schedule::{ + CoverageInputs, Scheduler, SchedulerConfig, derived_coverage_period_ms, max_auditable_rooms, +}; +use crate::suspicion::SuspicionLane; +use crate::topology::{TopologyCache, TopologyLimits}; +use crate::turn::TurnSequencer; + +pub use crate::singleton::{ + InstanceAnnounce, InstanceIdentity, InstanceProbe, PEER_CLAMP_RELEASE_AFTER_MS, PeerRecord, + PeerVerdict, SINGLETON_PROBE_DEADLINE_MS, SINGLETON_PROBE_INTERVAL_MS, SingletonLayer, + SingletonResponse, peer_verdict, run_singleton, should_answer_probe, +}; + +pub const SERVICE_NAME: &str = "recon"; +pub const RETIRED_GRACE_MS: u64 = 60_000; + +pub const RUNTIME_TASKS: [&str; 6] = [ + "http", + "engine", + "census", + "discovery", + "control", + "singleton", +]; +pub const WEBHOOK_TASK: &str = "webhook"; +pub const OPTIONAL_TASKS: [&str; 1] = [WEBHOOK_TASK]; +pub const HOUSEKEEP_INTERVAL_MS: u64 = 1_000; + +pub fn runtime_tasks(config: &ReconConfig) -> Vec<&'static str> { + let mut names = RUNTIME_TASKS.to_vec(); + if config.webhook_enabled { + names.push(WEBHOOK_TASK); + } + names +} + +pub fn monotonic_now() -> Millis { + Millis::new(fluxer_svc::metrics::now_ms().max(0) as u64) +} + +pub fn wall_now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + elapsed.as_millis().min(u128::from(u64::MAX)) as u64 + }) +} + +#[derive(Debug)] +pub struct Readiness { + flag: Arc, + has_been_ready: AtomicBool, + draining: AtomicBool, +} + +impl Default for Readiness { + fn default() -> Self { + Self::new() + } +} + +impl Readiness { + pub fn new() -> Self { + Self { + flag: Arc::new(AtomicBool::new(false)), + has_been_ready: AtomicBool::new(false), + draining: AtomicBool::new(false), + } + } + + pub fn flag(&self) -> Arc { + Arc::clone(&self.flag) + } + + pub fn mark_ready(&self) -> bool { + if self.draining.load(Ordering::SeqCst) { + return false; + } + self.flag.store(true, Ordering::SeqCst); + !self.has_been_ready.swap(true, Ordering::SeqCst) + } + + pub fn is_ready(&self) -> bool { + self.flag.load(Ordering::SeqCst) + } + + pub fn has_been_ready(&self) -> bool { + self.has_been_ready.load(Ordering::SeqCst) + } + + pub fn begin_shutdown(&self) { + self.draining.store(true, Ordering::SeqCst); + self.flag.store(false, Ordering::SeqCst); + } + + pub fn is_draining(&self) -> bool { + self.draining.load(Ordering::SeqCst) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ModeTransition { + pub configured: ReconMode, + pub previous_override: Option, + pub requested: Option, + pub effective_before: ReconMode, + pub effective_after: ReconMode, +} + +impl ModeTransition { + pub fn changed(&self) -> bool { + self.effective_before != self.effective_after + } + + pub fn escalated(&self) -> bool { + self.effective_after > self.effective_before + } +} + +pub fn governor_limits(config: &ReconConfig) -> GovernorLimits { + GovernorLimits { + gateway_read_rps: config.gateway_read_rps, + gateway_mutate_rpm: config.gateway_mutate_rpm, + livekit_read_rps: config.livekit_read_rps, + livekit_read_rps_per_server: config.livekit_read_rps_per_server, + livekit_mutate_rpm: config.livekit_mutate_rpm, + max_inflight_room_turns: config.max_inflight_room_turns, + max_inflight_gateway: config.max_inflight_gateway, + max_inflight_livekit: config.max_inflight_livekit, + max_inflight_per_server: config.max_inflight_per_server, + max_mutations_per_room_per_min: config.max_mutations_per_room_per_min, + max_mutations_per_guild_per_min: config.max_mutations_per_guild_per_min, + max_mutations_per_min: config.max_mutations_per_min, + max_action_attempts: config.max_action_attempts, + preflight_max_age_ms: config.preflight_max_age_ms, + breaker: BreakerLimits { + max_divergent_fraction: config.max_divergent_fraction, + auto_reset_ms: config.breaker_auto_reset_ms, + max_auto_resets: config.max_breaker_auto_resets, + }, + } +} + +pub fn ledger_limits(config: &ReconConfig) -> LedgerLimits { + LedgerLimits { + max_tracked_rooms: config.max_tracked_rooms, + max_tracked_connections: config.max_tracked_connections, + max_connections_per_room: config.max_connections_per_room, + memory_budget_bytes: config.memory_budget_bytes, + retired_grace_ms: RETIRED_GRACE_MS, + } +} + +#[derive(Debug)] +pub struct RuntimeState { + pub governor: Governor, + pub ledger: Ledger, + pub server_health: ServerHealthMap, + pub turn: TurnId, + pub sequencer: TurnSequencer, + pub scheduler: Scheduler, + pub directory: RoomDirectory, + pub census: CensusCache, + pub topology: TopologyCache, + pub suspicion: SuspicionLane, + pub cliffs: ServerCliffWatch, + pub counters: PassCounters, + pub pass_started_at: Millis, + pub pass_pending: BTreeSet, + pub topology_loaded_at: Option, + pub topology_servers: usize, + pub census_at: Option, + pub coverage_period_ms: u64, + pub peer: Option, + pub unknown_since: BTreeMap, + pub last_housekeep_at: Option, + pub topology_refresh_requested: bool, + pub mutations_pause: PauseBackoff, +} + +impl RuntimeState { + pub fn new(config: &ReconConfig, now: Millis) -> Self { + let warmup = WarmupGate::new(now, config.warmup_seconds.saturating_mul(1_000)); + let mut governor = Governor::new(config.mode, governor_limits(config), warmup, now); + let clamps = governor.clamps_mut(); + clamps.set(ModeClamp::Warmup, true); + clamps.set(ModeClamp::ColdStart, true); + clamps.set(ModeClamp::TopologyStale, true); + Self { + governor, + ledger: Ledger::new(ledger_limits(config), DecideParams::from_config(config)), + server_health: ServerHealthMap::new(), + turn: TurnId::FIRST, + sequencer: TurnSequencer::new(), + scheduler: Scheduler::new(SchedulerConfig::from_config(config)), + directory: RoomDirectory::new(DirectoryLimits::from_config(config)), + census: CensusCache::new(CensusLimits::from_config(config)), + topology: TopologyCache::new(TopologyLimits::from_config(config)), + suspicion: SuspicionLane::from_config(config), + cliffs: ServerCliffWatch::from_config(config), + counters: PassCounters::new(), + pass_started_at: now, + pass_pending: BTreeSet::new(), + topology_loaded_at: None, + topology_servers: 0, + census_at: None, + coverage_period_ms: config.coverage_target_ms, + peer: None, + unknown_since: BTreeMap::new(), + last_housekeep_at: None, + topology_refresh_requested: false, + mutations_pause: PauseBackoff::new(), + } + } + + pub fn forget_room(&mut self, room: &RoomKey) { + self.directory.forget(room); + self.scheduler.forget(room); + self.sequencer.forget(room); + self.suspicion.forget(room); + self.pass_pending.remove(room); + self.unknown_since.remove(room); + self.governor.forget_room(room); + } + + pub fn tracked_bytes(&self) -> u64 { + let room_key_entry = (size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; + self.ledger + .tracked_bytes() + .saturating_add(self.directory.tracked_bytes()) + .saturating_add(self.scheduler.tracked_bytes()) + .saturating_add(self.sequencer.tracked_bytes()) + .saturating_add(self.suspicion.tracked_bytes()) + .saturating_add(self.governor.mutations().tracked_bytes()) + .saturating_add(self.census.tracked_bytes()) + .saturating_add((self.pass_pending.len() as u64).saturating_mul(room_key_entry)) + .saturating_add( + (self.unknown_since.len() as u64) + .saturating_mul(room_key_entry.saturating_add(size_of::() as u64)), + ) + } + + pub const fn take_topology_refresh_request(&mut self) -> bool { + let requested = self.topology_refresh_requested; + self.topology_refresh_requested = false; + requested + } + + pub fn next_turn(&mut self) -> TurnId { + self.turn = self.turn.next(); + self.turn + } + + pub fn topology_age_ms(&self, now: Millis) -> Option { + self.topology_loaded_at + .map(|loaded| now.saturating_since(loaded)) + } + + pub fn census_age_ms(&self, now: Millis) -> Option { + self.census_at.map(|at| now.saturating_since(at)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TickInputs { + pub now: Millis, + pub nats_connected: bool, + pub last_decision_at: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct HousekeepReport { + pub effective_mode: ReconMode, + pub breaker: BreakerEvent, + pub divergent_fraction: f64, + pub pruned_connections: usize, + pub dropped_rooms: usize, + pub evicted_rooms: usize, + pub unknown_rooms: usize, + pub wedged_connections: usize, +} + +pub const ROOM_RETENTION_MS: u64 = 600_000; + +fn last_seen_by_any_source(provenance: &crate::discovery::RoomProvenance) -> Option { + provenance + .sources() + .into_iter() + .filter_map(|source| provenance.last_seen_by(source)) + .max() +} + +fn prune_room_set(state: &mut RuntimeState, now: Millis) { + let ledger = &state.ledger; + let scheduler = &mut state.scheduler; + let sequencer = &mut state.sequencer; + let suspicion = &mut state.suspicion; + let pending = &mut state.pass_pending; + + let mut forgotten: Vec = Vec::new(); + state.directory.retain(|room, provenance| { + if ledger.room(room).is_some() { + return true; + } + let cold = last_seen_by_any_source(provenance) + .is_none_or(|seen| now.saturating_since(seen) > ROOM_RETENTION_MS); + if cold { + forgotten.push(*room); + } + !cold + }); + + for room in &forgotten { + scheduler.forget(room); + sequencer.forget(room); + suspicion.forget(room); + pending.remove(room); + } + + pending.retain(|room| scheduler.is_tracked(room)); + + for room in &forgotten { + state.governor.forget_room(room); + state.unknown_since.remove(room); + } +} + +pub const MAX_EVICTIONS_PER_HOUSEKEEP: usize = 256; + +fn evict_to_budget(state: &mut RuntimeState) -> usize { + let limits = state.ledger.limits(); + let mut evicted = 0; + + while evicted < MAX_EVICTIONS_PER_HOUSEKEEP + && (state.tracked_bytes() > limits.memory_budget_bytes + || state.ledger.tracked_rooms() > limits.max_tracked_rooms + || state.ledger.tracked_connections() > limits.max_tracked_connections) + { + let Some(victim) = state.ledger.coldest_room() else { + break; + }; + if !state.ledger.evict_room(&victim) { + break; + } + state.forget_room(&victim); + evicted += 1; + } + + evicted +} + +fn room_is_unknown(room: &crate::ledger::RoomLedger, config: &ReconConfig, now: Millis) -> bool { + room.no_candidates() + || room.partially_unreadable() + || room.gateway_cliff().is_held(now, config.room_cliff_hold_ms) + || room.media_cliff().is_held(now, config.server_cliff_hold_ms) +} + +pub fn housekeep( + state: &mut RuntimeState, + config: &ReconConfig, + metrics: &ReconMetrics, + inputs: TickInputs, +) -> HousekeepReport { + let now = inputs.now; + + state.governor.clamps_mut().release_expired(now); + state.mutations_pause.decay(now); + + let warmup_pending = !state.governor.warmup().elapsed_complete(now); + let coverage_pending = !state.governor.warmup().coverage_pass_complete(); + let topology_stale = state + .topology_age_ms(now) + .is_none_or(|age| age > config.topology_max_age_ms); + let peer_present = state + .peer + .as_ref() + .is_some_and(|peer| now.saturating_since(peer.last_seen_at) < PEER_CLAMP_RELEASE_AFTER_MS); + + { + let clamps = state.governor.clamps_mut(); + clamps.set(ModeClamp::Warmup, warmup_pending); + clamps.set(ModeClamp::ColdStart, coverage_pending); + clamps.set(ModeClamp::NatsReconnect, !inputs.nats_connected); + clamps.set(ModeClamp::TopologyStale, topology_stale); + clamps.set(ModeClamp::SingletonConflict, peer_present); + } + + let divergent_fraction = state.ledger.divergent_fraction(); + let breaker = state.governor.observe_divergence(divergent_fraction, now); + if matches!(breaker, BreakerEvent::Tripped) { + metrics.record_breaker_trip(); + state.ledger.reset_all_corroborations(); + } + + let pruned_connections = state.ledger.prune_retired(now); + let dropped_rooms = state.ledger.drop_empty_rooms(); + state.governor.prune_mutations(now); + let evicted_rooms = evict_to_budget(state); + + state.suspicion.expire(now); + state.scheduler.expire_hot(now); + prune_room_set(state, now); + + let live_servers = state.topology.lens(now).live_locations(); + state.governor.retain_servers(&live_servers); + + let mut no_candidate_rooms = 0; + let mut wedged_connections = 0; + let mut oldest_staleness_ms = 0; + let mut unknown_keys: Vec = Vec::new(); + + for (key, room) in state.ledger.rooms() { + oldest_staleness_ms = oldest_staleness_ms.max(now.saturating_since(room.last_seen())); + if room.no_candidates() { + no_candidate_rooms += 1; + } + if room_is_unknown(room, config, now) { + unknown_keys.push(*key); + } + wedged_connections += room + .connections() + .iter() + .filter(|entry| matches!(entry.state, ConnectionState::Wedged { .. })) + .count(); + } + + let unknown_rooms = unknown_keys.len(); + state + .unknown_since + .retain(|key, _| unknown_keys.binary_search(key).is_ok()); + for key in unknown_keys { + state.unknown_since.entry(key).or_insert(now); + } + + let oldest_unknown_ms = state + .unknown_since + .values() + .map(|since| now.saturating_since(*since)) + .max() + .unwrap_or(0); + + let tracked_rooms = state.ledger.tracked_rooms(); + let effective_mode = state.governor.effective_mode(); + + metrics.set_mode(effective_mode); + for clamp in ModeClamp::ALL { + metrics.set_mode_clamp(clamp, state.governor.clamps().is_engaged(clamp)); + } + metrics.set_ledger( + tracked_rooms, + state.ledger.tracked_connections(), + state.tracked_bytes(), + state + .ledger + .evicted_rooms_total() + .saturating_add(state.ledger.evicted_connections_total()), + ); + metrics.set_wedged_connections(wedged_connections); + metrics.set_rooms_no_candidates(no_candidate_rooms); + metrics.set_rooms_unknown_ratio(if tracked_rooms == 0 { + 0.0 + } else { + unknown_rooms as f64 / tracked_rooms as f64 + }); + metrics.set_oldest_unknown_room_seconds(oldest_unknown_ms as f64 / 1_000.0); + metrics.set_room_staleness_seconds(oldest_staleness_ms as f64 / 1_000.0); + metrics.set_coverage_period_seconds(state.coverage_period_ms as f64 / 1_000.0); + metrics.set_topology_age_seconds( + state + .topology_age_ms(now) + .map_or(f64::INFINITY, |age| age as f64 / 1_000.0), + ); + metrics.set_census_age_seconds( + state + .census_age_ms(now) + .map_or(f64::INFINITY, |age| age as f64 / 1_000.0), + ); + metrics.set_last_decision_age_seconds(inputs.last_decision_at.map_or(f64::INFINITY, |at| { + now.saturating_since(at) as f64 / 1_000.0 + })); + for (location, tracker) in state.server_health.tracked() { + metrics.set_server_health(location, tracker.state()); + } + + state.last_housekeep_at = Some(now); + + HousekeepReport { + effective_mode, + breaker, + divergent_fraction, + pruned_connections, + dropped_rooms, + evicted_rooms, + unknown_rooms, + wedged_connections, + } +} + +pub struct SharedInner { + config: ReconConfig, + instance: InstanceIdentity, + metrics: Arc, + journal: DecisionJournal, + readiness: Readiness, + boot_at: Millis, + state: Mutex, +} + +#[derive(Clone)] +pub struct Shared(Arc); + +impl Shared { + pub fn new(config: ReconConfig, instance: InstanceIdentity, now: Millis) -> Self { + let state = RuntimeState::new(&config, now); + Self(Arc::new(SharedInner { + config, + instance, + metrics: Arc::new(ReconMetrics::new()), + journal: DecisionJournal::default(), + readiness: Readiness::new(), + boot_at: now, + state: Mutex::new(state), + })) + } + + pub fn config(&self) -> &ReconConfig { + &self.0.config + } + + pub fn instance(&self) -> &InstanceIdentity { + &self.0.instance + } + + pub fn metrics(&self) -> &Arc { + &self.0.metrics + } + + pub fn journal(&self) -> &DecisionJournal { + &self.0.journal + } + + pub fn readiness(&self) -> &Readiness { + &self.0.readiness + } + + pub fn boot_at(&self) -> Millis { + self.0.boot_at + } + + fn lock(&self) -> MutexGuard<'_, RuntimeState> { + self.0.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub fn with_state(&self, action: impl FnOnce(&RuntimeState) -> R) -> R { + action(&self.lock()) + } + + pub fn with_state_mut(&self, action: impl FnOnce(&mut RuntimeState) -> R) -> R { + action(&mut self.lock()) + } + + pub fn effective_mode(&self) -> ReconMode { + self.with_state(|state| state.governor.effective_mode()) + } + + pub fn tick(&self, inputs: TickInputs) -> HousekeepReport { + let config = self.0.config.clone(); + let metrics = Arc::clone(&self.0.metrics); + self.with_state_mut(|state| housekeep(state, &config, &metrics, inputs)) + } + + pub fn tick_if_due(&self, inputs: TickInputs, interval_ms: u64) -> Option { + let due = self.with_state(|state| { + state + .last_housekeep_at + .is_none_or(|last| inputs.now.saturating_since(last) >= interval_ms) + }); + due.then(|| self.tick(inputs)) + } + + pub fn set_mode_override(&self, requested: Option) -> ModeTransition { + self.with_state_mut(|state| { + let configured = state.governor.configured_mode(); + let previous_override = state.governor.runtime_override(); + let effective_before = state.governor.effective_mode(); + state.governor.set_runtime_override(requested); + let effective_after = state.governor.effective_mode(); + ModeTransition { + configured, + previous_override, + requested, + effective_before, + effective_after, + } + }) + } + + pub fn note_peer(&self, announce: &InstanceAnnounce, now: Millis) { + self.0.metrics.record_peer_detected(); + self.with_state_mut(|state| { + let detections = state + .peer + .as_ref() + .filter(|peer| peer.instance_id == announce.instance_id) + .map_or(0, |peer| peer.detections); + let first_seen_at = state + .peer + .as_ref() + .filter(|peer| peer.instance_id == announce.instance_id) + .map_or(now, |peer| peer.first_seen_at); + state.peer = Some(PeerRecord { + instance_id: announce.instance_id.clone(), + started_at: announce.started_at, + first_seen_at, + last_seen_at: now, + detections: detections.saturating_add(1), + }); + state + .governor + .clamps_mut() + .set(ModeClamp::SingletonConflict, true); + }); + } +} + +pub fn assert_single_replica( + service_name: &str, + shard_count: u32, + shard_id: u32, +) -> anyhow::Result<()> { + if service_name != SERVICE_NAME { + anyhow::bail!( + "FLUXER_SVC_NAME must be {SERVICE_NAME} so the control subject and metric namespace agree, got {service_name}" + ); + } + if shard_count != 1 { + anyhow::bail!( + "FLUXER_SVC_SHARD_COUNT must be 1 for a single-replica service, got {shard_count}" + ); + } + if shard_id != 0 { + anyhow::bail!("FLUXER_SVC_SHARD_ID must be 0 for a single-replica service, got {shard_id}"); + } + Ok(()) +} + +pub fn assert_coverage_budget(config: &ReconConfig) -> anyhow::Result<()> { + if config.coverage_target_ms > config.coverage_period_hard_cap_ms { + anyhow::bail!( + "FLUXER_RECON_COVERAGE_TARGET_MS ({}) exceeds FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({})", + config.coverage_target_ms, + config.coverage_period_hard_cap_ms + ); + } + if config.tick_ms == 0 { + anyhow::bail!("FLUXER_RECON_TICK_MS must be greater than zero"); + } + if config.worker_threads == 0 { + anyhow::bail!("FLUXER_RECON_WORKER_THREADS must be greater than zero"); + } + + let auditable = max_auditable_rooms(config); + let derived = + derived_coverage_period_ms(&CoverageInputs::for_rooms(config.expected_rooms, config)); + if derived > config.coverage_period_hard_cap_ms { + anyhow::bail!( + "FLUXER_RECON_EXPECTED_ROOMS ({}) needs a coverage period of {derived} ms at the configured read rates, above FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({}); this process can audit {auditable} rooms within that cap", + config.expected_rooms, + config.coverage_period_hard_cap_ms + ); + } + + tracing::info!( + expected_rooms = config.expected_rooms, + derived_coverage_period_ms = derived, + max_auditable_rooms = auditable, + "the configured read rates admit this many rooms inside the coverage hard cap" + ); + Ok(()) +} + +pub fn boot_assertions(service: &ServiceConfig, config: &ReconConfig) -> anyhow::Result<()> { + assert_single_replica(&service.service_name, service.shard_count, service.shard_id)?; + assert_coverage_budget(config) +} diff --git a/fluxer_recon/src/schedule.rs b/fluxer_recon/src/schedule.rs new file mode 100644 index 000000000..e8e44c6fd --- /dev/null +++ b/fluxer_recon/src/schedule.rs @@ -0,0 +1,562 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; + +use crate::config::ReconConfig; +use crate::ids::{Millis, RoomKey}; +use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; +use crate::turn::Digest; + +pub const SCHEDULE_ENTRY_BYTES: u64 = (size_of::() + + size_of::() + + size_of::() + + (2 * MAP_ENTRY_OVERHEAD_BYTES)) as u64; + +pub const GATEWAY_READS_PER_ROOM_TURN: u32 = 2; +pub const LIVEKIT_READS_PER_ROOM_TURN: u32 = 1; + +const fn room_hash(room: RoomKey) -> u64 { + match room { + RoomKey::Guild { + guild_id, + channel_id, + } => Digest::new() + .text("guild") + .number(guild_id.get()) + .number(channel_id.get()) + .finish(), + RoomKey::Dm { channel_id } => Digest::new().text("dm").number(channel_id.get()).finish(), + } +} + +const fn tiebreak(room: RoomKey, stamp: u64) -> u64 { + Digest::new().number(room_hash(room)).number(stamp).finish() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Tier { + Hot, + Background, +} + +impl Tier { + pub const fn label(self) -> &'static str { + match self { + Self::Hot => "hot", + Self::Background => "background", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Promotion { + Promoted, + Extended, + Refused, +} + +impl Promotion { + pub const fn label(self) -> &'static str { + match self { + Self::Promoted => "promoted", + Self::Extended => "extended", + Self::Refused => "refused", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeferReason { + Budget, + Unreadable, +} + +impl DeferReason { + pub const fn label(self) -> &'static str { + match self { + Self::Budget => "budget", + Self::Unreadable => "unreadable", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SchedulerConfig { + pub hot_period_ms: u64, + pub background_period_ms: u64, + pub max_hot_rooms: usize, + pub suspicion_hold_ms: u64, + pub turns_per_tick: usize, +} + +impl SchedulerConfig { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + hot_period_ms: config.hot_period_ms, + background_period_ms: config.coverage_target_ms, + max_hot_rooms: config.max_hot_rooms, + suspicion_hold_ms: config.suspicion_hold_ms, + turns_per_tick: config.turns_per_tick, + } + } + + pub const fn period_ms(&self, tier: Tier) -> u64 { + match tier { + Tier::Hot => self.hot_period_ms, + Tier::Background => self.background_period_ms, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Due { + at: Millis, + tiebreak: u64, + stamp: u64, + room: RoomKey, +} + +impl Ord for Due { + fn cmp(&self, other: &Self) -> Ordering { + other + .at + .cmp(&self.at) + .then_with(|| other.tiebreak.cmp(&self.tiebreak)) + .then_with(|| other.room.cmp(&self.room)) + .then_with(|| other.stamp.cmp(&self.stamp)) + } +} + +impl PartialOrd for Due { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RoomSchedule { + stamp: u64, + tier: Tier, + hot_until: Option, + due_at: Millis, + in_flight: bool, + served: u64, +} + +#[derive(Clone, Debug)] +pub struct Scheduler { + config: SchedulerConfig, + heap: BinaryHeap, + rooms: BTreeMap, + hot: BTreeSet, + stamps: u64, + in_flight: usize, + served_total: u64, + starved_total: u64, + deferred_total: u64, + hot_refused_total: u64, + demoted_total: u64, +} + +impl Scheduler { + pub fn tracked_bytes(&self) -> u64 { + (self.rooms.len() as u64) + .saturating_mul(SCHEDULE_ENTRY_BYTES) + .saturating_add( + (self.hot.len() as u64) + .saturating_mul((size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64), + ) + } + + pub fn new(config: SchedulerConfig) -> Self { + Self { + config, + heap: BinaryHeap::new(), + rooms: BTreeMap::new(), + hot: BTreeSet::new(), + stamps: 0, + in_flight: 0, + served_total: 0, + starved_total: 0, + deferred_total: 0, + hot_refused_total: 0, + demoted_total: 0, + } + } + + pub const fn config(&self) -> SchedulerConfig { + self.config + } + + pub const fn turns_per_tick(&self) -> usize { + self.config.turns_per_tick + } + + pub fn tracked(&self) -> usize { + self.rooms.len() + } + + pub fn is_empty(&self) -> bool { + self.rooms.is_empty() + } + + pub fn hot_rooms(&self) -> usize { + self.hot.len() + } + + pub const fn in_flight(&self) -> usize { + self.in_flight + } + + pub const fn served_total(&self) -> u64 { + self.served_total + } + + pub const fn starved_total(&self) -> u64 { + self.starved_total + } + + pub const fn deferred_total(&self) -> u64 { + self.deferred_total + } + + pub const fn hot_refused_total(&self) -> u64 { + self.hot_refused_total + } + + pub const fn demoted_total(&self) -> u64 { + self.demoted_total + } + + pub fn tier(&self, room: &RoomKey) -> Option { + self.rooms.get(room).map(|entry| entry.tier) + } + + pub fn due_at(&self, room: &RoomKey) -> Option { + self.rooms.get(room).map(|entry| entry.due_at) + } + + pub fn served(&self, room: &RoomKey) -> Option { + self.rooms.get(room).map(|entry| entry.served) + } + + pub fn is_tracked(&self, room: &RoomKey) -> bool { + self.rooms.contains_key(room) + } + + pub fn track(&mut self, room: RoomKey, now: Millis) -> bool { + if self.rooms.contains_key(&room) { + return false; + } + let entry = RoomSchedule { + stamp: self.next_stamp(), + tier: Tier::Background, + hot_until: None, + due_at: now, + in_flight: false, + served: 0, + }; + self.rooms.insert(room, entry); + self.push(room, entry); + true + } + + pub fn forget(&mut self, room: &RoomKey) -> bool { + self.hot.remove(room); + match self.rooms.remove(room) { + None => false, + Some(entry) => { + if entry.in_flight { + self.in_flight = self.in_flight.saturating_sub(1); + } + true + } + } + } + + pub fn retain(&mut self, mut keep: F) + where + F: FnMut(&RoomKey) -> bool, + { + let dropped: Vec = self + .rooms + .keys() + .copied() + .filter(|room| !keep(room)) + .collect(); + for room in dropped { + self.forget(&room); + } + } + + pub fn take_due(&mut self, now: Millis) -> Vec { + self.take_due_up_to(now, self.config.turns_per_tick) + } + + pub fn take_due_up_to(&mut self, now: Millis, limit: usize) -> Vec { + let mut taken: Vec = Vec::new(); + while taken.len() < limit { + let Some(candidate) = self.heap.peek().copied() else { + break; + }; + if candidate.at > now { + break; + } + self.heap.pop(); + let Some(entry) = self.rooms.get_mut(&candidate.room) else { + continue; + }; + if entry.stamp != candidate.stamp || entry.in_flight { + continue; + } + entry.in_flight = true; + self.in_flight = self.in_flight.saturating_add(1); + taken.push(candidate.room); + } + taken + } + + pub fn completed(&mut self, room: RoomKey, now: Millis) -> bool { + let Some(mut entry) = self.rooms.get(&room).copied() else { + return false; + }; + self.release(&mut entry); + entry.served = entry.served.saturating_add(1); + self.served_total = self.served_total.saturating_add(1); + entry.stamp = self.next_stamp(); + entry.due_at = now.saturating_add_millis(self.config.period_ms(entry.tier)); + self.rooms.insert(room, entry); + self.push(room, entry); + true + } + + pub fn defer(&mut self, room: RoomKey, now: Millis, reason: DeferReason) -> bool { + let Some(mut entry) = self.rooms.get(&room).copied() else { + return false; + }; + self.release(&mut entry); + self.deferred_total = self.deferred_total.saturating_add(1); + if matches!(reason, DeferReason::Budget) { + self.starved_total = self.starved_total.saturating_add(1); + } + entry.stamp = self.next_stamp(); + entry.due_at = now; + self.rooms.insert(room, entry); + self.push(room, entry); + true + } + + pub fn promote(&mut self, room: RoomKey, now: Millis) -> Promotion { + self.track(room, now); + let Some(mut entry) = self.rooms.get(&room).copied() else { + return Promotion::Refused; + }; + let hot_until = now.saturating_add_millis(self.config.suspicion_hold_ms); + + if matches!(entry.tier, Tier::Hot) { + entry.hot_until = Some(hot_until); + self.rooms.insert(room, entry); + self.hot.insert(room); + return Promotion::Extended; + } + if self.hot.len() >= self.config.max_hot_rooms { + self.hot_refused_total = self.hot_refused_total.saturating_add(1); + return Promotion::Refused; + } + + entry.tier = Tier::Hot; + entry.hot_until = Some(hot_until); + entry.stamp = self.next_stamp(); + entry.due_at = now; + self.rooms.insert(room, entry); + self.hot.insert(room); + if !entry.in_flight { + self.push(room, entry); + } + Promotion::Promoted + } + + pub fn expire_hot(&mut self, now: Millis) -> usize { + let expired: Vec = self + .hot + .iter() + .copied() + .filter(|room| { + self.rooms + .get(room) + .is_none_or(|entry| entry.hot_until.is_none_or(|until| until <= now)) + }) + .collect(); + + for room in &expired { + self.hot.remove(room); + let Some(mut entry) = self.rooms.get(room).copied() else { + continue; + }; + entry.tier = Tier::Background; + entry.hot_until = None; + entry.stamp = self.next_stamp(); + entry.due_at = now.saturating_add_millis(self.config.background_period_ms); + self.rooms.insert(*room, entry); + if !entry.in_flight { + self.push(*room, entry); + } + self.demoted_total = self.demoted_total.saturating_add(1); + } + expired.len() + } + + fn release(&mut self, entry: &mut RoomSchedule) { + if entry.in_flight { + self.in_flight = self.in_flight.saturating_sub(1); + } + entry.in_flight = false; + } + + fn next_stamp(&mut self) -> u64 { + self.stamps = self.stamps.saturating_add(1); + self.stamps + } + + fn push(&mut self, room: RoomKey, entry: RoomSchedule) { + self.heap.push(Due { + at: entry.due_at, + tiebreak: tiebreak(room, entry.stamp), + stamp: entry.stamp, + room, + }); + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Cadence { + interval_ms: u64, + next_at: Option, + fired_total: u64, + forced: bool, +} + +impl Cadence { + pub const fn new(interval_ms: u64) -> Self { + Self { + interval_ms: if interval_ms == 0 { 1 } else { interval_ms }, + next_at: None, + fired_total: 0, + forced: false, + } + } + + pub const fn interval_ms(self) -> u64 { + self.interval_ms + } + + pub const fn next_due_at(self) -> Option { + self.next_at + } + + pub const fn fired_total(self) -> u64 { + self.fired_total + } + + pub const fn force(&mut self) { + self.forced = true; + } + + pub fn due(&mut self, now: Millis) -> bool { + let ready = self.forced || self.next_at.is_none_or(|at| now >= at); + if !ready { + return false; + } + self.forced = false; + self.next_at = Some(now.saturating_add_millis(self.interval_ms)); + self.fired_total = self.fired_total.saturating_add(1); + true + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CoverageInputs { + pub rooms: usize, + pub gateway_reads_per_turn: u32, + pub livekit_reads_per_turn: u32, + pub gateway_read_rps: u32, + pub livekit_read_rps: u32, +} + +impl CoverageInputs { + pub const fn for_rooms(rooms: usize, config: &ReconConfig) -> Self { + Self { + rooms, + gateway_reads_per_turn: GATEWAY_READS_PER_ROOM_TURN, + livekit_reads_per_turn: LIVEKIT_READS_PER_ROOM_TURN, + gateway_read_rps: config.gateway_read_rps, + livekit_read_rps: config.livekit_read_rps, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CoverageVerdict { + WithinTarget, + AboveTarget, +} + +impl CoverageVerdict { + pub const fn label(self) -> &'static str { + match self { + Self::WithinTarget => "within_target", + Self::AboveTarget => "above_target", + } + } +} + +fn lane_period_ms(rooms: usize, reads_per_turn: u32, rps: u32) -> u64 { + let calls = u64::try_from(rooms) + .unwrap_or(u64::MAX) + .saturating_mul(u64::from(reads_per_turn)); + let rate = u64::from(rps.max(1)); + calls.saturating_mul(1_000).div_ceil(rate) +} + +pub fn derived_coverage_period_ms(inputs: &CoverageInputs) -> u64 { + let gateway = lane_period_ms( + inputs.rooms, + inputs.gateway_reads_per_turn, + inputs.gateway_read_rps, + ); + let livekit = lane_period_ms( + inputs.rooms, + inputs.livekit_reads_per_turn, + inputs.livekit_read_rps, + ); + gateway.max(livekit) +} + +pub fn max_auditable_rooms(config: &ReconConfig) -> usize { + let gateway = u64::from(config.gateway_read_rps) + .saturating_mul(config.coverage_period_hard_cap_ms) + .checked_div(1_000 * u64::from(GATEWAY_READS_PER_ROOM_TURN.max(1))) + .unwrap_or(0); + let livekit = u64::from(config.livekit_read_rps) + .saturating_mul(config.coverage_period_hard_cap_ms) + .checked_div(1_000 * u64::from(LIVEKIT_READS_PER_ROOM_TURN.max(1))) + .unwrap_or(0); + usize::try_from(gateway.min(livekit)).unwrap_or(usize::MAX) +} + +pub fn check_coverage_period( + period_ms: u64, + target_ms: u64, + hard_cap_ms: u64, +) -> anyhow::Result { + if period_ms > hard_cap_ms { + anyhow::bail!( + "the derived coverage period is {period_ms} ms, above FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({hard_cap_ms} ms); the room count this process can audit at the configured read rates has been exceeded" + ); + } + if period_ms > target_ms { + return Ok(CoverageVerdict::AboveTarget); + } + Ok(CoverageVerdict::WithinTarget) +} diff --git a/fluxer_recon/src/service.rs b/fluxer_recon/src/service.rs new file mode 100644 index 000000000..7206925f0 --- /dev/null +++ b/fluxer_recon/src/service.rs @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::RwLock; + +use fluxer_svc::config::{DatabaseBackend, ServiceConfig}; +use fluxer_svc::transport::Transport; + +use crate::actuate::{ActuationPolicy, Ports, TurnFacts, apply_decision}; +use crate::budget::{BreakerEvent, BudgetClass}; +use crate::census::{CensusOutcome, read_census}; +use crate::clock::{Clock, SharedClock, WallAnchor}; +use crate::config::ReconConfig; +use crate::decide::{Decision, decide}; +use crate::discovery::{ + ExpectationSource, LocationReadout, RoomContext, RoomReadPlan, RoomTurnReads, RoomView, + ServerRoomList, list_fleet_rooms, observe_room, read_room, record_health, +}; +use crate::evidence::{ + CandidateSet, GatewayRead, GatewayVoiceState, MediaSighting, SideKind, TopologyFreshness, + candidate_set, room_cliff_input, room_cliff_trips, +}; +use crate::gateway::GatewayApi; +use crate::gateway::nats::NatsGateway; +use crate::health::ModeClamp; +use crate::ids::{Location, Millis, RoomKey, TurnId, WallMillis}; +use crate::ledger::{LocationIndex, RoomLedger}; +use crate::livekit::twirp::{TwirpLiveKit, TwirpTimeouts}; +use crate::livekit::{ + LiveKitApi, LiveKitFault, ParticipantRecord, ReadResult, RemoveOutcome, ServerCredentials, +}; +use crate::metrics::ReconMetrics; +use crate::names::ParticipantIdentity; +use crate::observe::{DecisionJournal, ObservedSet, journal_decisions}; +use crate::runtime::{HOUSEKEEP_INTERVAL_MS, RuntimeState, Shared, TickInputs}; +use crate::schedule::{ + Cadence, CoverageInputs, CoverageVerdict, DeferReason, GATEWAY_READS_PER_ROOM_TURN, + check_coverage_period, derived_coverage_period_ms, max_auditable_rooms, +}; +use crate::suspicion::{SuspicionHint, SuspicionSource}; +use crate::topology::{InternalEndpoint, RefreshOutcome, TopologyStore, VoiceServer}; +use crate::turn::Fresh; +use crate::turn::{TurnScope, TurnToken}; + +pub const CADENCE_POLL_MS: u64 = 250; +pub const FORCED_TOPOLOGY_REFRESH_FLOOR_MS: u64 = 60_000; +pub const DISCOVERY_LIST_ROOMS_COST: u32 = 1; + +pub trait Uplink: Send + Sync { + fn is_connected(&self) -> bool; +} + +impl Uplink for NatsGateway { + fn is_connected(&self) -> bool { + self.transport().is_connected() + } +} + +pub trait Fleet { + fn install(&mut self, servers: Vec) -> anyhow::Result<()>; +} + +impl Fleet for TwirpLiveKit { + fn install(&mut self, servers: Vec) -> anyhow::Result<()> { + self.replace_servers(servers) + } +} + +#[derive(Debug)] +pub struct SharedFleet { + inner: Arc>, +} + +impl Clone for SharedFleet { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl SharedFleet { + pub fn new(livekit: L) -> Self { + Self { + inner: Arc::new(RwLock::new(livekit)), + } + } +} + +impl SharedFleet { + pub async fn install(&self, servers: Vec) -> anyhow::Result<()> { + self.inner.write().await.install(servers) + } +} + +impl LiveKitApi for SharedFleet { + async fn list_rooms(&self, location: &Location) -> ReadResult>> { + self.inner.read().await.list_rooms(location).await + } + + async fn list_participants( + &self, + location: &Location, + room: RoomKey, + ) -> ReadResult> { + self.inner + .read() + .await + .list_participants(location, room) + .await + } + + async fn remove_participant( + &self, + location: &Location, + room: RoomKey, + identity: &ParticipantIdentity, + ) -> RemoveOutcome { + self.inner + .read() + .await + .remove_participant(location, room, identity) + .await + } +} + +pub struct GuardedClock { + inner: SharedClock, + anchor: WallAnchor, +} + +impl std::fmt::Debug for GuardedClock { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GuardedClock") + .field("anchor", &self.anchor) + .finish() + } +} + +impl GuardedClock { + pub fn anchored(inner: SharedClock) -> Self { + let anchor = WallAnchor::new(inner.now(), inner.wall_now()); + Self { inner, anchor } + } + + pub fn shared(inner: SharedClock) -> SharedClock { + Arc::new(Self::anchored(inner)) + } +} + +impl Clock for GuardedClock { + fn now(&self) -> Millis { + self.inner.now() + } + + fn wall_now(&self) -> WallMillis { + self.anchor.guard(self.inner.now(), self.inner.wall_now()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TurnOutcome { + Completed, + Unreadable, + Starved, + Broken, +} + +impl TurnOutcome { + pub const fn label(self) -> &'static str { + match self { + Self::Completed => "completed", + Self::Unreadable => "unreadable", + Self::Starved => "starved", + Self::Broken => "broken", + } + } +} + +struct RoomTurnPlan { + token: TurnToken, + locations: Vec, + gateway_reads: u32, +} + +struct RoomTurnPermits { + locations: Vec, + livekit_held: bool, +} + +struct TurnReading { + decisions: Vec, + media: Vec, + turn: TurnId, + outcome: TurnOutcome, + divergent: bool, +} + +pub struct ReconEngine { + shared: Shared, + gateway: G, + livekit: L, + clock: SharedClock, + policy: ActuationPolicy, +} + +impl ReconEngine +where + G: GatewayApi + Uplink, + L: LiveKitApi, +{ + pub fn new(shared: Shared, gateway: G, livekit: L, clock: SharedClock) -> Self { + let policy = ActuationPolicy::from_config(shared.config()); + Self { + shared, + gateway, + livekit, + clock, + policy, + } + } + + pub fn shared(&self) -> &Shared { + &self.shared + } + + fn config(&self) -> &ReconConfig { + self.shared.config() + } + + fn metrics(&self) -> Arc { + Arc::clone(self.shared.metrics()) + } + + pub async fn step(&self) { + let now = self.clock.now(); + + if let Some(report) = self.shared.tick_if_due( + TickInputs { + now, + nats_connected: self.gateway.is_connected(), + last_decision_at: self.shared.journal().last_at_ms().map(Millis::new), + }, + HOUSEKEEP_INTERVAL_MS, + ) { + if matches!(report.breaker, BreakerEvent::Tripped) { + tracing::error!( + divergent_fraction = report.divergent_fraction, + "divergent fraction over budget, breaker tripped and every corroboration reset" + ); + } + if self.shared.readiness().mark_ready() { + tracing::info!( + mode = report.effective_mode.as_str(), + tick_ms = self.config().tick_ms, + "recon engine ready" + ); + } + } + + let due = self + .shared + .with_state_mut(|state| admit_and_take_due(state, now)); + + for room in due { + let started = self.clock.now(); + let outcome = self.room_turn(room).await; + let settled = self.clock.now(); + self.metrics() + .observe_turn_duration(settled.saturating_since(started)); + self.shared + .with_state_mut(|state| settle_turn(state, room, outcome, settled)); + if matches!(outcome, TurnOutcome::Starved) { + self.metrics().record_scheduler_starvation(); + } + } + + self.close_pass_if_complete(); + } + + fn close_pass_if_complete(&self) { + let now = self.clock.now(); + let target_ms = self.config().coverage_target_ms; + let hard_cap_ms = self.config().coverage_period_hard_cap_ms; + let config = self.config().clone(); + + let closed = self.shared.with_state_mut(|state| { + state + .pass_pending + .retain(|room| state.scheduler.is_tracked(room)); + if !state.pass_pending.is_empty() { + return None; + } + if now.saturating_since(state.pass_started_at) < target_ms { + return None; + } + if state.scheduler.served_total() == 0 { + return None; + } + + state.counters.note_expectations_accounted(); + let cliffed = state.cliffs.observe_pass(&state.counters, now); + state.counters.clear(); + for location in &cliffed { + let rooms: Vec = state + .directory + .room_keys() + .into_iter() + .filter(|room| state.directory.locations_for(room).contains(location)) + .collect(); + for room in rooms { + if let Some(entry) = state.ledger.room_mut(&room) { + entry.reset_corroborations(); + } + } + } + + let period_ms = now.saturating_since(state.pass_started_at); + state.coverage_period_ms = period_ms; + state.pass_started_at = now; + state.pass_pending = state + .directory + .room_keys() + .into_iter() + .filter(|room| state.scheduler.is_tracked(room)) + .collect(); + state.governor.warmup_mut().note_coverage_pass(); + + let rooms = state.directory.len(); + Some((period_ms, rooms, cliffed)) + }); + + let Some((period_ms, rooms, cliffed)) = closed else { + return; + }; + + let derived = derived_coverage_period_ms(&CoverageInputs::for_rooms(rooms, &config)); + let verdict = check_coverage_period(derived, target_ms, hard_cap_ms); + let overrun = verdict.is_err(); + self.shared.with_state_mut(|state| { + state + .governor + .clamps_mut() + .set(ModeClamp::CoverageOverrun, overrun); + }); + match verdict { + Err(error) => { + tracing::error!( + error = %error, + rooms, + derived_coverage_period_ms = derived, + max_auditable_rooms = max_auditable_rooms(&config), + "the room count is beyond what this process can audit at the configured read \ + rates; the destructive lane is clamped off until the read rates or the room \ + count change" + ); + } + Ok(CoverageVerdict::AboveTarget) if derived > target_ms.saturating_mul(2) => { + tracing::warn!( + rooms, + derived_coverage_period_ms = derived, + coverage_target_ms = target_ms, + "the derived coverage period is more than twice the target" + ); + } + Ok(_) => {} + } + + tracing::debug!( + period_ms, + rooms, + cliffed = cliffed.len(), + "coverage pass complete" + ); + } + + async fn room_turn(&self, room: RoomKey) -> TurnOutcome { + let opened_at = self.clock.now(); + let Some(plan) = self + .shared + .with_state_mut(|state| plan_room_turn(state, room, opened_at)) + else { + return TurnOutcome::Starved; + }; + + let permits = RoomTurnPermits { + locations: plan.locations.clone(), + livekit_held: !plan.locations.is_empty(), + }; + let outcome = self.take_room_turn(room, plan).await; + self.shared + .with_state_mut(|state| release_room_turn(state, &permits)); + outcome + } + + async fn take_room_turn(&self, room: RoomKey, plan: RoomTurnPlan) -> TurnOutcome { + let RoomTurnPlan { + mut token, + locations, + gateway_reads, + } = plan; + + let reads = read_room( + &self.gateway, + &self.livekit, + &RoomReadPlan { + room, + locations: &locations, + }, + self.clock.as_ref(), + |_| true, + &mut token, + ) + .await; + + let fresh = match reads { + Err(error) => { + tracing::error!( + ?error, + channel_id = room.channel_id().get(), + "a room read broke the turn accounting, abandoning the turn" + ); + self.shared.with_state_mut(|state| { + state.sequencer.abandon(token); + }); + return TurnOutcome::Broken; + } + Ok(fresh) => fresh, + }; + + let at = self.clock.now(); + let wall_at = self.clock.wall_now(); + let metrics = self.metrics(); + let reading = self.shared.with_state_mut(|state| { + observe_and_decide( + state, + &ObserveInputs { + config: self.config(), + metrics: &metrics, + journal: self.shared.journal(), + room, + at, + wall_at, + gateway_reads, + }, + token, + fresh, + ) + }); + + let Some(reading) = reading else { + return TurnOutcome::Broken; + }; + + if reading.divergent { + let hint = SuspicionHint::new(room, SuspicionSource::Divergence, at); + self.shared.with_state_mut(|state| { + state.suspicion.note(hint); + }); + } + + let facts = TurnFacts { + room, + authorizing_turn: reading.turn, + media: reading.media, + }; + let ports = Ports { + gateway: &self.gateway, + livekit: &self.livekit, + clock: self.clock.as_ref(), + shared: &self.shared, + policy: self.policy, + }; + + for decision in &reading.decisions { + let actuation = apply_decision(&ports, &facts, decision).await; + if actuation.called_out() { + tracing::info!( + channel_id = room.channel_id().get(), + guild_id = room.guild_id().map(|guild| guild.get()), + connection_id = decision.connection.connection.as_str(), + user_id = decision.user_id.get(), + action = decision.action.label(), + outcome = ?actuation, + "recon issued a reconciliation call" + ); + } + } + + reading.outcome + } +} + +fn admit_and_take_due(state: &mut RuntimeState, now: Millis) -> Vec { + for room in state.directory.room_keys() { + if state.scheduler.track(room, now) { + state.pass_pending.insert(room); + } + } + + for room in state.suspicion.drain_raised() { + state.scheduler.promote(room, now); + } + + let free = state + .governor + .limits() + .max_inflight_room_turns + .min(state.scheduler.turns_per_tick()); + state.scheduler.take_due_up_to(now, free) +} + +fn settle_turn(state: &mut RuntimeState, room: RoomKey, outcome: TurnOutcome, now: Millis) { + match outcome { + TurnOutcome::Completed | TurnOutcome::Unreadable => { + state.scheduler.completed(room, now); + state.pass_pending.remove(&room); + } + TurnOutcome::Starved => { + state.scheduler.defer(room, now, DeferReason::Budget); + } + TurnOutcome::Broken => { + state.scheduler.defer(room, now, DeferReason::Unreadable); + } + } +} + +fn ledger_last_known(state: &RuntimeState, room: RoomKey) -> Vec { + let indexes = state + .ledger + .room(&room) + .map(|entry| entry.last_known_locations().to_vec()) + .unwrap_or_default(); + indexes + .into_iter() + .filter_map(|index| state.ledger.location(index).cloned()) + .collect() +} + +fn believed_homes(state: &RuntimeState, room: RoomKey) -> Vec { + let indexes = state + .ledger + .room(&room) + .map(RoomLedger::believed_homes) + .unwrap_or_default(); + indexes + .into_iter() + .filter_map(|index| state.ledger.location(index).cloned()) + .collect() +} + +fn planned_candidates(state: &RuntimeState, room: RoomKey, now: Millis) -> CandidateSet { + let sources = state + .directory + .sources_for(&room, Vec::new(), ledger_last_known(state, room)); + candidate_set(&sources, &state.topology.lens(now)) +} + +fn record_read_calls(metrics: &ReconMetrics, reads: &RoomTurnReads) { + let gateway_outcome = match &reads.gateway { + GatewayRead::Ok { .. } => "ok", + GatewayRead::Failed(fault) => fault.label(), + }; + if reads.gateway_calls > 1 { + metrics.record_gateway_rpc("voice_states_for_channel", "ok"); + metrics.record_gateway_rpc("pending_joins_for_channel", gateway_outcome); + } else { + metrics.record_gateway_rpc("voice_states_for_channel", gateway_outcome); + } + + for readout in &reads.readouts { + let outcome = readout.fault().map_or("ok", LiveKitFault::label); + metrics.record_livekit_call("list_participants", outcome); + } +} + +fn release_room_turn(state: &mut RuntimeState, permits: &RoomTurnPermits) { + for location in &permits.locations { + state.governor.server_inflight(location).release(); + } + if permits.livekit_held { + state.governor.livekit_inflight().release(); + } + state.governor.gateway_inflight().release(); + state.governor.room_turns().release(); +} + +fn abandon_room_turn(state: &mut RuntimeState, permits: &RoomTurnPermits) { + for location in &permits.locations { + state.governor.refund_server_read(location, 1); + } + release_room_turn(state, permits); +} + +fn admit_locations( + state: &mut RuntimeState, + candidates: Vec, + now: Millis, +) -> Option> { + let mut admitted: Vec = Vec::new(); + for location in candidates { + if !state.governor.server_inflight(&location).try_acquire() { + release_locations(state, &admitted); + return None; + } + if !state.governor.try_acquire_server_read(&location, 1, now) { + state.governor.server_inflight(&location).release(); + release_locations(state, &admitted); + return None; + } + admitted.push(location); + } + Some(admitted) +} + +fn release_locations(state: &mut RuntimeState, locations: &[Location]) { + for location in locations { + state.governor.refund_server_read(location, 1); + state.governor.server_inflight(location).release(); + } +} + +fn plan_room_turn(state: &mut RuntimeState, room: RoomKey, now: Millis) -> Option { + let candidates = planned_candidates(state, room, now); + let offered: Vec = candidates + .locations() + .iter() + .filter(|location| state.server_health.may_probe(location, now)) + .cloned() + .collect(); + + let gateway_reads = GATEWAY_READS_PER_ROOM_TURN; + + if !state.governor.room_turns().try_acquire() { + return None; + } + if !state.governor.gateway_inflight().try_acquire() { + state.governor.room_turns().release(); + return None; + } + + let Some(locations) = admit_locations(state, offered, now) else { + state.governor.gateway_inflight().release(); + state.governor.room_turns().release(); + return None; + }; + let livekit_reads = u32::try_from(locations.len()).unwrap_or(u32::MAX); + let livekit_held = livekit_reads > 0 && state.governor.livekit_inflight().try_acquire(); + let permits = RoomTurnPermits { + locations: locations.clone(), + livekit_held, + }; + + if livekit_reads > 0 && !livekit_held { + abandon_room_turn(state, &permits); + return None; + } + if state.governor.tokens(BudgetClass::GatewayRead, now) < f64::from(gateway_reads) { + abandon_room_turn(state, &permits); + return None; + } + if state.governor.tokens(BudgetClass::LiveKitRead, now) < f64::from(livekit_reads) { + abandon_room_turn(state, &permits); + return None; + } + if !state + .governor + .try_acquire_many(BudgetClass::GatewayRead, gateway_reads, now) + { + abandon_room_turn(state, &permits); + return None; + } + if livekit_reads > 0 + && !state + .governor + .try_acquire_many(BudgetClass::LiveKitRead, livekit_reads, now) + { + state + .governor + .refund(BudgetClass::GatewayRead, gateway_reads); + abandon_room_turn(state, &permits); + return None; + } + + Some(RoomTurnPlan { + token: state.sequencer.open(TurnScope::room_turn(room), now), + locations, + gateway_reads, + }) +} + +fn gateway_hints(states: &[GatewayVoiceState]) -> Vec { + let mut hints: Vec = states + .iter() + .filter_map(|state| state.hint.clone()) + .collect(); + hints.sort(); + hints.dedup(); + hints +} + +fn media_sightings(reads: &RoomTurnReads) -> Vec { + let mut sightings: Vec = Vec::new(); + for readout in &reads.readouts { + let Some(roster) = readout.roster() else { + continue; + }; + for participant in roster.participants() { + sightings.push(MediaSighting { + connection: participant.connection.clone(), + user_id: participant.user_id, + }); + } + } + sightings.sort_by(|left, right| left.connection.cmp(&right.connection)); + sightings.dedup_by(|left, right| left.connection == right.connection); + sightings +} + +fn note_roster_reads( + state: &mut RuntimeState, + room: RoomKey, + readouts: &[LocationReadout], + at: Millis, +) { + let indices: Vec = readouts + .iter() + .filter(|readout| readout.roster().is_some()) + .filter_map(|readout| state.ledger.intern_location(readout.location())) + .collect(); + if indices.is_empty() { + return; + } + if state.ledger.ensure_room(room, at).is_err() { + return; + } + let Some(entry) = state.ledger.room_mut(&room) else { + return; + }; + for index in indices { + entry.note_roster_read(index, at); + } +} + +fn note_expected_from_ledger(state: &mut RuntimeState, room: RoomKey) { + let Some(entry) = state.ledger.room(&room) else { + return; + }; + let counts: Vec<(Location, u32)> = entry + .believed_home_counts() + .into_iter() + .filter_map(|(index, count)| { + state + .ledger + .location(index) + .cloned() + .map(|location| (location, count)) + }) + .collect(); + for (location, count) in counts { + state + .counters + .note_room_expected(&location, room, ExpectationSource::Ledger, count); + } +} + +struct ObserveInputs<'a> { + config: &'a ReconConfig, + metrics: &'a ReconMetrics, + journal: &'a DecisionJournal, + room: RoomKey, + at: Millis, + wall_at: WallMillis, + gateway_reads: u32, +} + +fn observe_and_decide( + state: &mut RuntimeState, + inputs: &ObserveInputs<'_>, + token: TurnToken, + fresh: Fresh, +) -> Option { + let ObserveInputs { + config, + metrics, + journal, + room, + at, + wall_at, + gateway_reads, + } = *inputs; + + let turn = token.turn(); + state.turn = turn; + + record_read_calls(metrics, fresh.peek()); + + let unused_gateway_reads = gateway_reads.saturating_sub(fresh.peek().gateway_calls); + if unused_gateway_reads > 0 { + state + .governor + .refund(BudgetClass::GatewayRead, unused_gateway_reads); + } + + record_health(&fresh.peek().readouts, &mut state.server_health, at); + note_roster_reads(state, room, &fresh.peek().readouts, at); + if fresh + .peek() + .readouts + .iter() + .any(|readout| readout.fault().is_some_and(LiveKitFault::is_auth_failure)) + { + state.topology_refresh_requested = true; + } + + let hints = gateway_hints(fresh.peek().gateway_states()); + for hint in &hints { + state.directory.note_pinned(room, hint.clone(), at); + } + + let gateway_connections = fresh.peek().gateway_connections(); + let media_connections = fresh.peek().media_connections(); + let readable = matches!(fresh.peek().gateway, GatewayRead::Ok { .. }); + + let prior = state + .ledger + .room(&room) + .map(RoomLedger::prior_connections) + .unwrap_or_default(); + let cliff_input = room_cliff_input(&prior, &gateway_connections, &media_connections); + let hold_ms = config.room_cliff_hold_ms; + if state.ledger.ensure_room(room, at).is_ok() + && let Some(entry) = state.ledger.room_mut(&room) + { + if readable && room_cliff_trips(cliff_input) { + entry.gateway_cliff_mut().trip(at); + entry.reset_corroborations(); + } else { + entry.gateway_cliff_mut().expire(at, hold_ms); + } + } + let room_cliffed = state + .ledger + .room(&room) + .is_some_and(|entry| entry.gateway_cliff().is_held(at, hold_ms)); + + let media_hold_ms = config.server_cliff_hold_ms; + let readouts = &fresh.peek().readouts; + let media_fully_readable = + !readouts.is_empty() && readouts.iter().all(LocationReadout::is_readable); + let media_cliff_input = room_cliff_input(&prior, &media_connections, &gateway_connections); + if let Some(entry) = state.ledger.room_mut(&room) { + if media_fully_readable && room_cliff_trips(media_cliff_input) { + entry.media_cliff_mut().trip(at); + entry.reset_corroborations(); + } else { + entry.media_cliff_mut().expire(at, media_hold_ms); + } + } + let media_cliffed = state + .ledger + .room(&room) + .is_some_and(|entry| entry.media_cliff().is_held(at, media_hold_ms)); + + let sources = state + .directory + .sources_for(&room, hints, ledger_last_known(state, room)); + let candidates = candidate_set(&sources, &state.topology.lens(at)); + let census = state.census.cross_check(&room, at); + let topology = TopologyFreshness { + age_ms: state.topology.age_ms(at).unwrap_or(u64::MAX), + max_age_ms: config.topology_max_age_ms, + }; + let census_epoch = state.census.epoch(); + let topology_epoch = state.topology.epoch(); + let media = media_sightings(fresh.peek()); + let homes = believed_homes(state, room); + + let observed = { + let mut holed = state.suspicion.holed_servers(); + if media_cliffed { + holed.extend(candidates.locations().iter().cloned()); + holed.sort(); + holed.dedup(); + } + let context = RoomContext { + room, + turn, + at, + wall_at, + candidates: &candidates, + health: &state.server_health, + cliffs: &state.cliffs, + holed: &holed, + believed_homes: &homes, + room_cliffed, + census, + topology, + census_epoch, + topology_epoch, + max_connections_per_room: config.max_connections_per_room, + }; + let view = fresh.map(|reads| observe_room(reads, &context)); + state.counters.note_view(view.peek()); + state.sequencer.seal(token, at, view) + }; + + let view: RoomView = match observed { + Err(error) => { + tracing::error!( + ?error, + channel_id = room.channel_id().get(), + "sealing a room turn failed, so no decision was taken from it" + ); + return None; + } + Ok(observed) => observed.into_parts().1, + }; + + note_expected_from_ledger(state, room); + + let gateway_ok = view.authority().gateway.is_authoritative(); + let media_ok = view.authority().media.is_authoritative(); + let budget = state.governor.view(at); + let outcome = decide(&mut state.ledger, view.observation(), at, &budget); + let mode = state.governor.effective_mode(); + + for decision in &outcome.decisions { + metrics.record_decision(decision); + } + for (side, reason) in [ + (SideKind::Gateway, view.authority().gateway.unknown_reason()), + (SideKind::Media, view.authority().media.unknown_reason()), + ] { + if let Some(reason) = reason { + metrics.record_unknown(side, reason.label()); + } + } + + let ledger_snapshot = &state.ledger; + journal_decisions( + journal, + &ObservedSet { + set: &outcome, + observation: view.observation(), + mode, + pending_join_skew_ms: config.pending_join_skew_ms, + }, + |decision| ledger_snapshot.connection(&decision.connection).cloned(), + ); + + let divergent = state + .ledger + .room(&room) + .is_some_and(|entry| entry.divergent_connections() > 0); + + if state.ledger.room(&room).is_some() { + state.directory.note_ledger(room, at); + } + + Some(TurnReading { + decisions: outcome.decisions, + media, + turn, + outcome: if gateway_ok && media_ok { + TurnOutcome::Completed + } else { + TurnOutcome::Unreadable + }, + divergent, + }) +} + +pub async fn run_engine(engine: ReconEngine) -> anyhow::Result<()> +where + G: GatewayApi + Uplink, + L: LiveKitApi, +{ + let tick = Duration::from_millis(engine.config().tick_ms.max(1)); + let mut ticker = tokio::time::interval(tick); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + engine.step().await; + } +} + +pub async fn run_census(shared: Shared, gateway: G, clock: SharedClock) -> anyhow::Result<()> +where + G: GatewayApi, +{ + let mut cadence = Cadence::new(shared.config().census_interval_ms.max(1)); + let mut ticker = tokio::time::interval(Duration::from_millis(CADENCE_POLL_MS)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + let now = clock.now(); + if !cadence.due(now) { + continue; + } + + take_census(&shared, &gateway, clock.as_ref()).await; + } +} + +pub async fn take_census(shared: &Shared, gateway: &G, clock: &dyn Clock) +where + G: GatewayApi, +{ + if !shared.with_state_mut(|state| state.governor.gateway_inflight().try_acquire()) { + return; + } + let read = read_census(gateway).await; + shared.with_state_mut(|state| state.governor.gateway_inflight().release()); + let label = read.label(); + shared + .metrics() + .record_gateway_rpc("active_voice_rooms", label); + let at = clock.now(); + let outcome = shared.with_state_mut(|state| { + let outcome = state.census.accept(&read, at); + let baseline = state.census.applied_total() <= 1; + if let Some(snapshot) = state.census.snapshot() { + state.census_at = Some(snapshot.taken_at()); + let seeded = state.directory.note_census(snapshot, at); + if !seeded.is_empty() { + tracing::debug!( + seeded = seeded.len(), + baseline, + "the census seeded rooms into the directory" + ); + } + if !baseline { + for room in seeded { + state.suspicion.note_room(room, SuspicionSource::Census, at); + } + } + } + outcome + }); + + match outcome { + CensusOutcome::Failed => { + tracing::warn!( + read = label, + "the census was unreadable, keeping the last good snapshot and skipping the \ + cross-check rather than stalling the service" + ); + } + CensusOutcome::Applied { rooms, .. } => { + tracing::debug!(rooms, "census applied"); + } + CensusOutcome::Unchanged { .. } => {} + } +} + +pub async fn run_discovery( + shared: Shared, + fleet: SharedFleet, + store: Arc, + clock: SharedClock, +) -> anyhow::Result<()> +where + L: LiveKitApi + Fleet + Send + Sync, +{ + let mut topology_cadence = Cadence::new(shared.config().topology_refresh_ms.max(1)); + let mut rooms_cadence = Cadence::new(shared.config().discovery_interval_ms.max(1)); + let mut forced = Cadence::new(FORCED_TOPOLOGY_REFRESH_FLOOR_MS); + let mut ticker = tokio::time::interval(Duration::from_millis(CADENCE_POLL_MS)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + let now = clock.now(); + + if shared.with_state_mut(RuntimeState::take_topology_refresh_request) && forced.due(now) { + topology_cadence.force(); + } + + if topology_cadence.due(now) { + refresh_topology(&shared, &fleet, store.as_ref(), clock.as_ref()).await; + } + + if rooms_cadence.due(now) { + discover_rooms(&shared, &fleet, clock.as_ref()).await; + } + } +} + +pub fn internal_endpoint(config: &ReconConfig) -> InternalEndpoint { + InternalEndpoint { + url: config.livekit_internal_url.clone(), + default_region_id: config.livekit_default_region_id.clone(), + } +} + +fn credentials_of(servers: &[VoiceServer], internal: &InternalEndpoint) -> Vec { + servers + .iter() + .map(|server| ServerCredentials { + location: server.location().clone(), + endpoint: Box::from(internal.resolve(server)), + api_key: Box::from(server.api_key()), + api_secret: server.api_secret().clone(), + }) + .collect() +} + +pub async fn refresh_topology( + shared: &Shared, + fleet: &SharedFleet, + store: &dyn TopologyStore, + clock: &dyn Clock, +) where + L: LiveKitApi + Fleet + Send + Sync, +{ + let loaded = store.load().await; + let at = clock.now(); + let internal = internal_endpoint(shared.config()); + + let (outcome, credentials) = shared.with_state_mut(|state| match loaded { + Err(error) => { + tracing::warn!( + backend = store.backend(), + error = %error, + "the topology load failed, keeping the last good snapshot and letting it age out" + ); + (state.topology.note_failure(at), Vec::new()) + } + Ok(servers) => { + let outcome = state.topology.accept(servers, at); + let credentials = credentials_of(state.topology.servers(), &internal); + if !outcome.kept_previous_snapshot() { + state.topology_loaded_at = Some(at); + state.topology_servers = state.topology.servers().len(); + } + (outcome, credentials) + } + }); + + match outcome { + RefreshOutcome::Applied { epoch, servers } => { + if let Err(error) = fleet.install(credentials).await { + tracing::error!(error = %error, "the livekit fleet refused a topology snapshot"); + } + tracing::info!( + backend = store.backend(), + epoch = epoch.get(), + servers, + "topology applied" + ); + } + RefreshOutcome::Unchanged { .. } => {} + RefreshOutcome::RejectedEmpty => { + tracing::error!( + backend = store.backend(), + "the topology load returned zero servers, which is refused: an empty fleet would \ + make every room's media side look empty rather than unreadable" + ); + } + RefreshOutcome::Failed => {} + } +} + +pub async fn discover_rooms(shared: &Shared, fleet: &SharedFleet, clock: &dyn Clock) +where + L: LiveKitApi + Fleet + Send + Sync, +{ + let now = clock.now(); + let locations = shared.with_state(|state| { + state + .topology + .servers() + .iter() + .map(|server| server.location().clone()) + .collect::>() + }); + if locations.is_empty() { + return; + } + + let admitted = shared.with_state_mut(|state| { + let mut admitted: Vec = Vec::new(); + if !state.governor.livekit_inflight().try_acquire() { + return admitted; + } + for location in &locations { + if !state.server_health.may_probe(location, now) { + continue; + } + if !state.governor.server_inflight(location).try_acquire() { + continue; + } + if !state + .governor + .try_acquire_server_read(location, DISCOVERY_LIST_ROOMS_COST, now) + { + state.governor.server_inflight(location).release(); + continue; + } + if !state.governor.try_acquire_many( + BudgetClass::LiveKitRead, + DISCOVERY_LIST_ROOMS_COST, + now, + ) { + state + .governor + .refund_server_read(location, DISCOVERY_LIST_ROOMS_COST); + state.governor.server_inflight(location).release(); + break; + } + admitted.push(location.clone()); + } + if admitted.is_empty() { + state.governor.livekit_inflight().release(); + } + admitted + }); + + let results = list_fleet_rooms(fleet, &admitted, |_| true).await; + let at = clock.now(); + + for (_, list) in &results { + let outcome = match list { + ServerRoomList::Listed { .. } => "ok", + ServerRoomList::Unreadable(fault) => fault.label(), + }; + shared.metrics().record_livekit_call("list_rooms", outcome); + } + + shared.with_state_mut(|state| { + if !admitted.is_empty() { + state.governor.livekit_inflight().release(); + } + for location in &admitted { + state.governor.server_inflight(location).release(); + } + }); + + shared.with_state_mut(|state| { + for (location, list) in &results { + state.server_health.record(location, list.outcome(), at); + if list.fault().is_some_and(LiveKitFault::is_auth_failure) { + state.topology_refresh_requested = true; + } + let filled = list.is_complete() + && state + .suspicion + .hole_for(location) + .is_some_and(|hole| hole.since().get() <= now.get()) + && state.suspicion.clear_hole(location); + if filled { + tracing::info!( + region = location.region.as_str(), + server = location.server.as_str(), + "a full room list read after the dropped webhook closed that server's hole" + ); + } + } + let pass = state.directory.apply_pass(&results, at); + for room in state.directory.drain_relocated() { + state + .suspicion + .note_room(room, SuspicionSource::Discovery, at); + } + if pass.unreadable_servers() > 0 { + tracing::debug!( + servers = pass.servers, + unreadable = pass.unreadable_servers(), + "a discovery pass was incomplete, so no location was withdrawn from those servers" + ); + } + }); +} + +pub async fn connect_topology(service: &ServiceConfig) -> anyhow::Result> { + match service.database_backend { + DatabaseBackend::Postgres => { + let config = fluxer_svc::postgres::PostgresConfig::from_service_config(service); + let store = crate::topology::postgres::PostgresTopology::connect(&config).await?; + Ok(Arc::new(store)) + } + DatabaseBackend::Cassandra => connect_scylla_topology(service).await, + } +} + +#[cfg(feature = "scylla")] +async fn connect_scylla_topology( + service: &ServiceConfig, +) -> anyhow::Result> { + let config = fluxer_svc::scylla::ScyllaConfig::from_service_config(service); + let store = crate::topology::scylla::ScyllaTopology::connect(&config).await?; + Ok(Arc::new(store)) +} + +#[cfg(not(feature = "scylla"))] +async fn connect_scylla_topology( + _service: &ServiceConfig, +) -> anyhow::Result> { + anyhow::bail!( + "FLUXER_DATABASE_BACKEND selects cassandra but this binary was built without the scylla feature" + ) +} + +pub fn build_livekit( + servers: &[VoiceServer], + internal: &InternalEndpoint, +) -> anyhow::Result { + TwirpLiveKit::new(credentials_of(servers, internal), TwirpTimeouts::DEFAULT) +} diff --git a/fluxer_recon/src/singleton.rs b/fluxer_recon/src/singleton.rs new file mode 100644 index 000000000..9e37519f1 --- /dev/null +++ b/fluxer_recon/src/singleton.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use fluxer_svc::transport::{Transport, TransportMessage, TransportSubscriber, reply_message}; + +use crate::control::INSTANCE_SUBJECT; +use crate::ids::Millis; +use crate::runtime::{Shared, monotonic_now, wall_now_ms}; + +pub const SINGLETON_PROBE_INTERVAL_MS: u64 = 30_000; +pub const SINGLETON_PROBE_DEADLINE_MS: u64 = 750; +pub const PEER_CLAMP_RELEASE_AFTER_MS: u64 = 3 * SINGLETON_PROBE_INTERVAL_MS; +pub const RESUBSCRIBE_BACKOFF_MS: u64 = 1_000; +pub const RESUBSCRIBE_BACKOFF_CEILING_MS: u64 = 30_000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum SingletonLayer { + Deployment, + RuntimeDetector, + SameTurnAuthorization, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SingletonResponse { + RefuseToBoot, + DegradeToConstructive, + NoActionCorrectnessHolds, +} + +impl SingletonResponse { + pub const fn terminates_a_running_process(self) -> bool { + match self { + Self::RefuseToBoot | Self::DegradeToConstructive | Self::NoActionCorrectnessHolds => { + false + } + } + } +} + +impl SingletonLayer { + pub const ALL: [Self; 3] = [ + Self::Deployment, + Self::RuntimeDetector, + Self::SameTurnAuthorization, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::Deployment => "deployment", + Self::RuntimeDetector => "runtime_detector", + Self::SameTurnAuthorization => "same_turn_authorization", + } + } + + pub const fn response(self) -> SingletonResponse { + match self { + Self::Deployment => SingletonResponse::RefuseToBoot, + Self::RuntimeDetector => SingletonResponse::DegradeToConstructive, + Self::SameTurnAuthorization => SingletonResponse::NoActionCorrectnessHolds, + } + } + + pub const fn acts_before_serving(self) -> bool { + matches!(self, Self::Deployment) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InstanceIdentity { + pub id: String, + pub started_at: u64, +} + +impl InstanceIdentity { + pub fn generate() -> Self { + static SEQUENCE: AtomicU64 = AtomicU64::new(0); + let started_at = wall_now_ms(); + let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed); + let entropy = mix(u64::from(std::process::id())) + ^ mix(started_at) + ^ mix(monotonic_now().get()) + ^ mix(sequence); + Self { + id: format!("{entropy:016x}"), + started_at, + } + } + + pub fn announce(&self) -> InstanceAnnounce { + InstanceAnnounce { + instance_id: self.id.clone(), + started_at: self.started_at, + } + } + + pub fn probe(&self) -> InstanceProbe { + InstanceProbe { + instance_id: self.id.clone(), + } + } +} + +const fn mix(value: u64) -> u64 { + let mut hash = value ^ 0x9e37_79b9_7f4a_7c15; + hash = (hash ^ (hash >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + hash = (hash ^ (hash >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + hash ^ (hash >> 31) +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct InstanceProbe { + pub instance_id: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct InstanceAnnounce { + pub instance_id: String, + pub started_at: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PeerVerdict { + SelfEcho, + Peer, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PeerRecord { + pub instance_id: String, + pub started_at: u64, + pub first_seen_at: Millis, + pub last_seen_at: Millis, + pub detections: u64, +} + +pub fn should_answer_probe(own: &InstanceIdentity, probe: &InstanceProbe) -> bool { + probe.instance_id != own.id +} + +pub fn peer_verdict(own: &InstanceIdentity, announce: &InstanceAnnounce) -> PeerVerdict { + if announce.instance_id == own.id { + PeerVerdict::SelfEcho + } else { + PeerVerdict::Peer + } +} + +pub const fn next_backoff_ms(previous: u64) -> u64 { + let doubled = previous.saturating_mul(2); + if doubled > RESUBSCRIBE_BACKOFF_CEILING_MS { + RESUBSCRIBE_BACKOFF_CEILING_MS + } else { + doubled + } +} + +pub async fn run_singleton(shared: Shared, transport: T) -> anyhow::Result<()> { + let announce = serde_json::to_vec(&shared.instance().announce())?; + let probe = serde_json::to_vec(&shared.instance().probe())?; + let deadline = Duration::from_millis(SINGLETON_PROBE_DEADLINE_MS); + let mut ticker = tokio::time::interval(Duration::from_millis(SINGLETON_PROBE_INTERVAL_MS)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut backoff_ms = RESUBSCRIBE_BACKOFF_MS; + + loop { + let mut subscription = match transport.subscribe(INSTANCE_SUBJECT).await { + Ok(subscription) => { + backoff_ms = RESUBSCRIBE_BACKOFF_MS; + subscription + } + Err(error) => { + tracing::error!( + error = %error, + subject = INSTANCE_SUBJECT, + retry_in_ms = backoff_ms, + "the singleton detector could not subscribe, retrying rather than ending the \ + task, because a restart loop is worse than a detector that is briefly deaf" + ); + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; + backoff_ms = next_backoff_ms(backoff_ms); + continue; + } + }; + + tracing::info!( + subject = INSTANCE_SUBJECT, + instance_id = shared.instance().id, + "singleton detector listening" + ); + + loop { + tokio::select! { + message = subscription.next() => { + let Some(message) = message else { + tracing::warn!("singleton subscription ended, will re-subscribe"); + break; + }; + let requester: InstanceProbe = match serde_json::from_slice(message.payload()) { + Ok(requester) => requester, + Err(error) => { + tracing::debug!(error = %error, "ignoring an undecodable instance probe"); + continue; + } + }; + if !should_answer_probe(shared.instance(), &requester) { + continue; + } + if let Err(error) = reply_message(&message, &transport, &announce).await { + tracing::debug!(error = %error, "failed to answer an instance probe"); + } + } + _ = ticker.tick() => { + let reply = transport.request(INSTANCE_SUBJECT, &probe, deadline).await; + let Ok(bytes) = reply else { + continue; + }; + let Ok(peer) = serde_json::from_slice::(&bytes) else { + continue; + }; + if matches!(peer_verdict(shared.instance(), &peer), PeerVerdict::SelfEcho) { + continue; + } + shared.note_peer(&peer, monotonic_now()); + tracing::error!( + peer_instance_id = peer.instance_id, + peer_started_at = peer.started_at, + own_instance_id = shared.instance().id, + layer = SingletonLayer::RuntimeDetector.label(), + response = ?SingletonLayer::RuntimeDetector.response(), + "a second recon instance answered, degrading below the destructive lane \ + and continuing to serve" + ); + } + _ = transport.wait_for_reconnect() => { + tracing::info!("NATS reconnected, re-subscribing the singleton detector"); + break; + } + } + } + } +} diff --git a/fluxer_recon/src/suspicion.rs b/fluxer_recon/src/suspicion.rs new file mode 100644 index 000000000..23f9fe95b --- /dev/null +++ b/fluxer_recon/src/suspicion.rs @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::BTreeMap; + +use crate::config::ReconConfig; +use crate::ids::{Location, Millis, RoomKey}; +use crate::ledger::{LOCATION_MAP_BYTES, MAP_ENTRY_OVERHEAD_BYTES}; + +pub const SUSPICION_ENTRY_BYTES: u64 = + (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SuspicionSource { + Webhook, + WebhookDrop, + Census, + Discovery, + Divergence, + Control, +} + +impl SuspicionSource { + pub const ALL: [Self; 6] = [ + Self::Webhook, + Self::WebhookDrop, + Self::Census, + Self::Discovery, + Self::Divergence, + Self::Control, + ]; + + pub const fn label(self) -> &'static str { + match self { + Self::Webhook => "webhook", + Self::WebhookDrop => "webhook_drop", + Self::Census => "census", + Self::Discovery => "discovery", + Self::Divergence => "divergence", + Self::Control => "control", + } + } + + pub const fn index(self) -> usize { + match self { + Self::Webhook => 0, + Self::WebhookDrop => 1, + Self::Census => 2, + Self::Discovery => 3, + Self::Divergence => 4, + Self::Control => 5, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SuspicionHint { + room: RoomKey, + source: SuspicionSource, + at: Millis, +} + +impl SuspicionHint { + pub const fn new(room: RoomKey, source: SuspicionSource, at: Millis) -> Self { + Self { room, source, at } + } + + pub const fn room(&self) -> RoomKey { + self.room + } + + pub const fn source(&self) -> SuspicionSource { + self.source + } + + pub const fn at(&self) -> Millis { + self.at + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Raised { + Raised, + Extended, + Refused, +} + +impl Raised { + pub const fn label(self) -> &'static str { + match self { + Self::Raised => "raised", + Self::Extended => "extended", + Self::Refused => "refused", + } + } + + pub const fn is_held(self) -> bool { + matches!(self, Self::Raised | Self::Extended) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SuspicionLimits { + pub hold_ms: u64, + pub max_rooms: usize, +} + +impl SuspicionLimits { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + hold_ms: config.suspicion_hold_ms, + max_rooms: config.max_hot_rooms, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Hold { + source: SuspicionSource, + since: Millis, + until: Millis, + hints: u32, +} + +impl Hold { + pub const fn source(&self) -> SuspicionSource { + self.source + } + + pub const fn since(&self) -> Millis { + self.since + } + + pub const fn until(&self) -> Millis { + self.until + } + + pub const fn hints(&self) -> u32 { + self.hints + } + + pub const fn is_held(&self, now: Millis) -> bool { + now.get() < self.until.get() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ServerHole { + dropped: u32, + since: Millis, + events: u32, +} + +impl ServerHole { + pub const fn dropped(&self) -> u32 { + self.dropped + } + + pub const fn since(&self) -> Millis { + self.since + } + + pub const fn events(&self) -> u32 { + self.events + } +} + +#[derive(Clone, Debug)] +pub struct SuspicionLane { + limits: SuspicionLimits, + holds: BTreeMap, + raised: Vec, + holes: BTreeMap, + by_source: [u64; SuspicionSource::ALL.len()], + raised_total: u64, + extended_total: u64, + refused_total: u64, + expired_total: u64, + holes_total: u64, +} + +impl SuspicionLane { + pub fn tracked_bytes(&self) -> u64 { + (self.holds.len() as u64) + .saturating_mul(SUSPICION_ENTRY_BYTES) + .saturating_add((self.holes.len() as u64).saturating_mul(LOCATION_MAP_BYTES)) + } + + pub const fn new(limits: SuspicionLimits) -> Self { + Self { + limits, + holds: BTreeMap::new(), + raised: Vec::new(), + holes: BTreeMap::new(), + by_source: [0; SuspicionSource::ALL.len()], + raised_total: 0, + extended_total: 0, + refused_total: 0, + expired_total: 0, + holes_total: 0, + } + } + + pub fn from_config(config: &ReconConfig) -> Self { + Self::new(SuspicionLimits::from_config(config)) + } + + pub const fn limits(&self) -> SuspicionLimits { + self.limits + } + + pub const fn raised_total(&self) -> u64 { + self.raised_total + } + + pub const fn extended_total(&self) -> u64 { + self.extended_total + } + + pub const fn refused_total(&self) -> u64 { + self.refused_total + } + + pub const fn expired_total(&self) -> u64 { + self.expired_total + } + + pub const fn holes_total(&self) -> u64 { + self.holes_total + } + + pub const fn hints_from(&self, source: SuspicionSource) -> u64 { + self.by_source[source.index()] + } + + pub fn held(&self) -> usize { + self.holds.len() + } + + pub fn is_empty(&self) -> bool { + self.holds.is_empty() + } + + pub fn hold_for(&self, room: &RoomKey) -> Option<&Hold> { + self.holds.get(room) + } + + pub fn is_suspect(&self, room: &RoomKey, now: Millis) -> bool { + self.holds.get(room).is_some_and(|hold| hold.is_held(now)) + } + + pub fn suspect_rooms(&self, now: Millis) -> Vec { + self.holds + .iter() + .filter(|(_, hold)| hold.is_held(now)) + .map(|(room, _)| *room) + .collect() + } + + pub fn note(&mut self, hint: SuspicionHint) -> Raised { + self.by_source[hint.source.index()] = self.by_source[hint.source.index()].saturating_add(1); + let until = hint.at.saturating_add_millis(self.limits.hold_ms); + + if let Some(hold) = self.holds.get_mut(&hint.room) { + hold.until = until; + hold.source = hint.source; + hold.hints = hold.hints.saturating_add(1); + self.extended_total = self.extended_total.saturating_add(1); + return Raised::Extended; + } + + if self.holds.len() >= self.limits.max_rooms { + self.refused_total = self.refused_total.saturating_add(1); + return Raised::Refused; + } + + self.holds.insert( + hint.room, + Hold { + source: hint.source, + since: hint.at, + until, + hints: 1, + }, + ); + self.raised.push(hint.room); + self.raised_total = self.raised_total.saturating_add(1); + Raised::Raised + } + + pub fn note_room(&mut self, room: RoomKey, source: SuspicionSource, now: Millis) -> Raised { + self.note(SuspicionHint::new(room, source, now)) + } + + pub fn drain_raised(&mut self) -> Vec { + std::mem::take(&mut self.raised) + } + + pub fn expire(&mut self, now: Millis) -> usize { + let expired: Vec = self + .holds + .iter() + .filter(|(_, hold)| !hold.is_held(now)) + .map(|(room, _)| *room) + .collect(); + for room in &expired { + self.holds.remove(room); + } + self.expired_total = self.expired_total.saturating_add(expired.len() as u64); + expired.len() + } + + pub fn forget(&mut self, room: &RoomKey) -> bool { + self.raised.retain(|held| held != room); + self.holds.remove(room).is_some() + } + + pub fn note_dropped(&mut self, location: &Location, dropped: u32, now: Millis) -> bool { + if dropped == 0 { + return false; + } + match self.holes.get_mut(location) { + Some(hole) => { + hole.dropped = hole.dropped.saturating_add(dropped); + hole.events = hole.events.saturating_add(1); + } + None => { + self.holes.insert( + location.clone(), + ServerHole { + dropped, + since: now, + events: 1, + }, + ); + self.holes_total = self.holes_total.saturating_add(1); + } + } + true + } + + pub fn hole_for(&self, location: &Location) -> Option<&ServerHole> { + self.holes.get(location) + } + + pub fn holed_servers(&self) -> Vec { + self.holes.keys().cloned().collect() + } + + pub fn clear_hole(&mut self, location: &Location) -> bool { + self.holes.remove(location).is_some() + } + + pub fn holes(&self) -> usize { + self.holes.len() + } +} diff --git a/fluxer_recon/src/topology/mod.rs b/fluxer_recon/src/topology/mod.rs new file mode 100644 index 000000000..29d891024 --- /dev/null +++ b/fluxer_recon/src/topology/mod.rs @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub mod postgres; + +#[cfg(feature = "scylla")] +pub mod scylla; + +use std::future::Future; +use std::pin::Pin; + +use crate::config::ReconConfig; +use crate::evidence::{ServerRecord, TopologyFreshness, TopologyLens}; +use crate::ids::{ApiSecret, Epoch, IdError, Location, Millis, RegionId, ServerId}; + +pub const VOICE_SERVERS_TABLE: &str = "voice_servers"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum RowError { + #[error("the {0} column is missing")] + MissingColumn(&'static str), + #[error("the {0} column is empty")] + EmptyColumn(&'static str), + #[error("the {column} column is not a valid identifier: {source}")] + InvalidColumn { + column: &'static str, + source: IdError, + }, + #[error("the row cannot be decoded as a voice_servers row")] + Undecodable, +} + +#[derive(Debug, thiserror::Error)] +pub enum TopologyError { + #[error("the topology backend is unavailable: {0}")] + Unavailable(String), + #[error("the topology query failed: {0}")] + Query(String), + #[error("a voice_servers row is malformed: {0}")] + Row(#[from] RowError), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VoiceServerRow { + pub region_id: String, + pub server_id: String, + pub endpoint: String, + pub api_key: String, + pub api_secret: String, + pub is_active: bool, +} + +impl VoiceServerRow { + pub fn into_server(self) -> Result { + let region = RegionId::new(self.region_id.trim()).map_err(|source| match source { + IdError::Empty => RowError::EmptyColumn("region_id"), + IdError::ContainsWhitespace | IdError::ContainsUnderscore => RowError::InvalidColumn { + column: "region_id", + source, + }, + })?; + let server = ServerId::new(self.server_id.trim()).map_err(|source| match source { + IdError::Empty => RowError::EmptyColumn("server_id"), + IdError::ContainsWhitespace | IdError::ContainsUnderscore => RowError::InvalidColumn { + column: "server_id", + source, + }, + })?; + + let endpoint = self.endpoint.trim(); + if endpoint.is_empty() { + return Err(RowError::EmptyColumn("endpoint")); + } + let api_key = self.api_key.trim(); + if api_key.is_empty() { + return Err(RowError::EmptyColumn("api_key")); + } + if self.api_secret.is_empty() { + return Err(RowError::EmptyColumn("api_secret")); + } + + Ok(VoiceServer { + location: Location::new(region, server), + endpoint: Box::from(endpoint), + api_key: Box::from(api_key), + api_secret: ApiSecret::new(&self.api_secret), + is_active: self.is_active, + }) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InternalEndpoint { + pub url: Option, + pub default_region_id: Option, +} + +impl InternalEndpoint { + pub fn resolve<'a>(&'a self, server: &'a VoiceServer) -> &'a str { + let (Some(url), Some(region)) = (self.url.as_deref(), self.default_region_id.as_deref()) + else { + return server.endpoint(); + }; + if server.region().as_str() != region + || server.server().as_str() != format!("{region}-server-1") + { + return server.endpoint(); + } + url + } +} + +pub fn default_region_id(blob: &str) -> Option { + let trimmed = blob.trim(); + if trimmed.is_empty() { + return None; + } + let parsed: serde_json::Value = serde_json::from_str(trimmed).ok()?; + match parsed { + serde_json::Value::String(id) => Some(id), + serde_json::Value::Object(fields) => fields + .get("id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned), + _ => None, + } + .filter(|id| !id.trim().is_empty()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VoiceServer { + location: Location, + endpoint: Box, + api_key: Box, + api_secret: ApiSecret, + is_active: bool, +} + +impl VoiceServer { + pub const fn location(&self) -> &Location { + &self.location + } + + pub const fn region(&self) -> &RegionId { + &self.location.region + } + + pub const fn server(&self) -> &ServerId { + &self.location.server + } + + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + pub fn api_key(&self) -> &str { + &self.api_key + } + + pub const fn api_secret(&self) -> &ApiSecret { + &self.api_secret + } + + pub const fn is_active(&self) -> bool { + self.is_active + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TopologySnapshot { + servers: Vec, + records: Vec, + loaded_at: Millis, + epoch: Epoch, +} + +impl TopologySnapshot { + pub fn servers(&self) -> &[VoiceServer] { + &self.servers + } + + pub fn records(&self) -> &[ServerRecord] { + &self.records + } + + pub const fn loaded_at(&self) -> Millis { + self.loaded_at + } + + pub const fn epoch(&self) -> Epoch { + self.epoch + } + + pub fn len(&self) -> usize { + self.servers.len() + } + + pub fn is_empty(&self) -> bool { + self.servers.is_empty() + } + + pub fn draining(&self) -> Vec { + self.records + .iter() + .filter(|record| record.removed_at.is_some()) + .map(|record| record.location.clone()) + .collect() + } + + pub fn server(&self, location: &Location) -> Option<&VoiceServer> { + self.servers + .iter() + .find(|server| &server.location == location) + } + + pub fn locations(&self) -> Vec { + self.servers + .iter() + .map(|server| server.location.clone()) + .collect() + } + + pub const fn age_ms(&self, now: Millis) -> u64 { + now.saturating_since(self.loaded_at) + } + + pub const fn is_stale(&self, now: Millis, max_age_ms: u64) -> bool { + self.age_ms(now) > max_age_ms + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TopologyLimits { + pub max_age_ms: u64, + pub drain_grace_ms: u64, +} + +impl TopologyLimits { + pub const fn from_config(config: &ReconConfig) -> Self { + Self { + max_age_ms: config.topology_max_age_ms, + drain_grace_ms: config.topology_drain_grace_ms, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefreshOutcome { + Applied { epoch: Epoch, servers: usize }, + Unchanged { epoch: Epoch }, + RejectedEmpty, + Failed, +} + +impl RefreshOutcome { + pub const fn label(self) -> &'static str { + match self { + Self::Applied { .. } => "applied", + Self::Unchanged { .. } => "unchanged", + Self::RejectedEmpty => "rejected_empty", + Self::Failed => "failed", + } + } + + pub const fn kept_previous_snapshot(self) -> bool { + match self { + Self::Applied { .. } | Self::Unchanged { .. } => false, + Self::RejectedEmpty | Self::Failed => true, + } + } +} + +#[derive(Clone, Debug)] +pub struct TopologyCache { + limits: TopologyLimits, + snapshot: Option, + epoch: Epoch, + last_failure_at: Option, + applied_total: u64, + unchanged_total: u64, + rejected_empty_total: u64, + failed_total: u64, +} + +impl TopologyCache { + pub const fn new(limits: TopologyLimits) -> Self { + Self { + limits, + snapshot: None, + epoch: Epoch::FIRST, + last_failure_at: None, + applied_total: 0, + unchanged_total: 0, + rejected_empty_total: 0, + failed_total: 0, + } + } + + pub const fn limits(&self) -> TopologyLimits { + self.limits + } + + pub const fn snapshot(&self) -> Option<&TopologySnapshot> { + self.snapshot.as_ref() + } + + pub const fn epoch(&self) -> Epoch { + self.epoch + } + + pub const fn last_failure_at(&self) -> Option { + self.last_failure_at + } + + pub const fn applied_total(&self) -> u64 { + self.applied_total + } + + pub const fn unchanged_total(&self) -> u64 { + self.unchanged_total + } + + pub const fn rejected_empty_total(&self) -> u64 { + self.rejected_empty_total + } + + pub const fn failed_total(&self) -> u64 { + self.failed_total + } + + pub fn records(&self) -> &[ServerRecord] { + self.snapshot + .as_ref() + .map_or(&[], TopologySnapshot::records) + } + + pub fn servers(&self) -> &[VoiceServer] { + self.snapshot + .as_ref() + .map_or(&[], TopologySnapshot::servers) + } + + pub fn server(&self, location: &Location) -> Option<&VoiceServer> { + self.snapshot + .as_ref() + .and_then(|snapshot| snapshot.server(location)) + } + + pub fn age_ms(&self, now: Millis) -> Option { + self.snapshot.as_ref().map(|snapshot| snapshot.age_ms(now)) + } + + pub fn is_stale(&self, now: Millis) -> bool { + self.snapshot + .as_ref() + .is_none_or(|snapshot| snapshot.is_stale(now, self.limits.max_age_ms)) + } + + pub fn freshness(&self, now: Millis) -> TopologyFreshness { + TopologyFreshness { + age_ms: self.age_ms(now).unwrap_or(u64::MAX), + max_age_ms: self.limits.max_age_ms, + } + } + + pub fn lens(&self, now: Millis) -> TopologyLens<'_> { + TopologyLens { + servers: self.records(), + now, + drain_grace_ms: self.limits.drain_grace_ms, + } + } + + pub fn note_failure(&mut self, now: Millis) -> RefreshOutcome { + self.failed_total = self.failed_total.saturating_add(1); + self.last_failure_at = Some(now); + RefreshOutcome::Failed + } + + pub fn accept(&mut self, servers: Vec, now: Millis) -> RefreshOutcome { + if servers.is_empty() { + self.rejected_empty_total = self.rejected_empty_total.saturating_add(1); + return RefreshOutcome::RejectedEmpty; + } + + let mut servers = servers; + servers.sort_by(|left, right| left.location.cmp(&right.location)); + servers.dedup_by(|left, right| left.location == right.location); + let records = self.records_for(&servers, now); + + let unchanged = self + .snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.servers == servers && snapshot.records == records); + if unchanged { + self.unchanged_total = self.unchanged_total.saturating_add(1); + if let Some(snapshot) = self.snapshot.as_mut() { + snapshot.loaded_at = now; + } + return RefreshOutcome::Unchanged { epoch: self.epoch }; + } + + self.epoch = self.epoch.next(); + self.applied_total = self.applied_total.saturating_add(1); + let count = servers.len(); + self.snapshot = Some(TopologySnapshot { + servers, + records, + loaded_at: now, + epoch: self.epoch, + }); + RefreshOutcome::Applied { + epoch: self.epoch, + servers: count, + } + } + + fn records_for(&self, servers: &[VoiceServer], now: Millis) -> Vec { + let mut records: Vec = servers + .iter() + .map(|server| ServerRecord { + location: server.location.clone(), + removed_at: None, + }) + .collect(); + + if let Some(previous) = self.snapshot.as_ref() { + for record in &previous.records { + if servers + .iter() + .any(|server| server.location == record.location) + { + continue; + } + records.push(ServerRecord { + location: record.location.clone(), + removed_at: Some(record.removed_at.unwrap_or(now)), + }); + } + } + + records.sort_by(|left, right| left.location.cmp(&right.location)); + records + } +} + +pub type LoadFuture<'a> = + Pin, TopologyError>> + Send + 'a>>; + +pub trait TopologyStore: Send + Sync { + fn load(&self) -> LoadFuture<'_>; + + fn backend(&self) -> &'static str; +} diff --git a/fluxer_recon/src/topology/postgres.rs b/fluxer_recon/src/topology/postgres.rs new file mode 100644 index 000000000..4dec563ca --- /dev/null +++ b/fluxer_recon/src/topology/postgres.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use serde::Deserialize; +use serde_json::Value; + +use fluxer_svc::postgres::{KvClient, PostgresConfig, connect as connect_pool, decode_row}; + +use super::{ + LoadFuture, RowError, TopologyError, TopologyStore, VOICE_SERVERS_TABLE, VoiceServer, + VoiceServerRow, +}; + +pub const BACKEND: &str = "postgres"; +const WHOLE_TABLE_PREFIX: &str = ""; + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +struct VoiceServerKvRow { + region_id: Option, + server_id: Option, + endpoint: Option, + api_key: Option, + api_secret: Option, + is_active: Option, +} + +fn column(value: Option, name: &'static str) -> Result { + value.ok_or(RowError::MissingColumn(name)) +} + +pub fn row_from_json(value: Value) -> Result { + let decoded = decode_row(value).map_err(|_| RowError::Undecodable)?; + if !decoded.is_object() { + return Err(RowError::Undecodable); + } + let row: VoiceServerKvRow = + serde_json::from_value(decoded).map_err(|_| RowError::Undecodable)?; + + Ok(VoiceServerRow { + region_id: column(row.region_id, "region_id")?, + server_id: column(row.server_id, "server_id")?, + endpoint: column(row.endpoint, "endpoint")?, + api_key: column(row.api_key, "api_key")?, + api_secret: column(row.api_secret, "api_secret")?, + is_active: row.is_active.unwrap_or(false), + }) +} + +#[derive(Clone)] +pub struct PostgresTopology { + kv: KvClient, +} + +impl PostgresTopology { + pub fn new(kv: KvClient) -> Self { + Self { kv } + } + + pub async fn connect(config: &PostgresConfig) -> anyhow::Result { + let pool = connect_pool(config).await?; + Ok(Self::new(KvClient::new(pool, config)?)) + } + + async fn read(&self) -> Result, TopologyError> { + let rows = self + .kv + .get_row_key_prefix_rows(VOICE_SERVERS_TABLE, WHOLE_TABLE_PREFIX) + .await + .map_err(|error| TopologyError::Query(error.to_string()))?; + + let mut servers = Vec::with_capacity(rows.len()); + for (_, value) in rows { + servers.push(row_from_json(value)?.into_server()?); + } + Ok(servers) + } +} + +impl std::fmt::Debug for PostgresTopology { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PostgresTopology") + .field("table", &VOICE_SERVERS_TABLE) + .finish() + } +} + +impl TopologyStore for PostgresTopology { + fn load(&self) -> LoadFuture<'_> { + Box::pin(async move { self.read().await }) + } + + fn backend(&self) -> &'static str { + BACKEND + } +} diff --git a/fluxer_recon/src/topology/scylla.rs b/fluxer_recon/src/topology/scylla.rs new file mode 100644 index 000000000..cf32d95c9 --- /dev/null +++ b/fluxer_recon/src/topology/scylla.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::sync::Arc; + +use scylla::DeserializeRow; +use scylla::client::session::Session; +use scylla::statement::prepared::PreparedStatement; + +use fluxer_svc::scylla::{ScyllaConfig, connect}; + +use super::{LoadFuture, RowError, TopologyError, TopologyStore, VoiceServer, VoiceServerRow}; + +pub const BACKEND: &str = "scylla"; +pub const VOICE_SERVERS_CQL: &str = + "SELECT region_id, server_id, endpoint, api_key, api_secret, is_active FROM voice_servers"; + +#[derive(Debug, DeserializeRow)] +struct VoiceServerCqlRow { + region_id: Option, + server_id: Option, + endpoint: Option, + api_key: Option, + api_secret: Option, + is_active: Option, +} + +fn column(value: Option, name: &'static str) -> Result { + value.ok_or(RowError::MissingColumn(name)) +} + +fn row_from_cql(row: VoiceServerCqlRow) -> Result { + Ok(VoiceServerRow { + region_id: column(row.region_id, "region_id")?, + server_id: column(row.server_id, "server_id")?, + endpoint: column(row.endpoint, "endpoint")?, + api_key: column(row.api_key, "api_key")?, + api_secret: column(row.api_secret, "api_secret")?, + is_active: row.is_active.unwrap_or(false), + }) +} + +#[derive(Clone)] +pub struct ScyllaTopology { + session: Arc, + statement: PreparedStatement, +} + +impl ScyllaTopology { + pub async fn new(session: Arc) -> anyhow::Result { + let statement = session.prepare(VOICE_SERVERS_CQL).await?; + Ok(Self { session, statement }) + } + + pub async fn connect(config: &ScyllaConfig) -> anyhow::Result { + Self::new(connect(config).await?).await + } + + async fn read(&self) -> Result, TopologyError> { + let result = self + .session + .execute_unpaged(&self.statement, ()) + .await + .map_err(|error| TopologyError::Unavailable(error.to_string()))?; + let rows = result + .into_rows_result() + .map_err(|error| TopologyError::Query(error.to_string()))?; + + let mut servers = Vec::new(); + for row in rows + .rows::() + .map_err(|error| TopologyError::Query(error.to_string()))? + { + let row = row.map_err(|error| TopologyError::Query(error.to_string()))?; + servers.push(row_from_cql(row)?.into_server()?); + } + Ok(servers) + } +} + +impl std::fmt::Debug for ScyllaTopology { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ScyllaTopology") + .field("statement", &VOICE_SERVERS_CQL) + .finish() + } +} + +impl TopologyStore for ScyllaTopology { + fn load(&self) -> LoadFuture<'_> { + Box::pin(async move { self.read().await }) + } + + fn backend(&self) -> &'static str { + BACKEND + } +} diff --git a/fluxer_recon/src/turn.rs b/fluxer_recon/src/turn.rs new file mode 100644 index 000000000..fc657bf6b --- /dev/null +++ b/fluxer_recon/src/turn.rs @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::ids::{Millis, RoomKey, TurnId}; +use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; + +pub const SEQUENCER_ENTRY_BYTES: u64 = + (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; + +const FIRST_TURN: TurnId = TurnId::new(1); +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Digest(u64); + +impl Default for Digest { + fn default() -> Self { + Self::new() + } +} + +impl Digest { + pub const fn new() -> Self { + Self(FNV_OFFSET) + } + + pub const fn bytes(self, bytes: &[u8]) -> Self { + let mut hash = self.0; + let mut index = 0; + while index < bytes.len() { + hash ^= bytes[index] as u64; + hash = hash.wrapping_mul(FNV_PRIME); + index += 1; + } + Self(hash) + } + + pub const fn text(self, text: &str) -> Self { + self.bytes(text.as_bytes()) + } + + pub const fn number(self, value: u64) -> Self { + Self(self.0 ^ mix(value)).bytes(&[0xff]) + } + + pub const fn flag(self, value: bool) -> Self { + self.number(value as u64) + } + + pub const fn finish(self) -> u64 { + self.0 + } +} + +pub fn digest_bytes(bytes: &[u8]) -> u64 { + Digest::new().bytes(bytes).finish() +} + +pub fn digest_str(text: &str) -> u64 { + Digest::new().text(text).finish() +} + +const fn mix(value: u64) -> u64 { + let mut hash = value; + hash ^= hash >> 30; + hash = hash.wrapping_mul(0xbf58_476d_1ce4_e5b9); + hash ^= hash >> 27; + hash = hash.wrapping_mul(0x94d0_49bb_1331_11eb); + hash ^ (hash >> 31) +} + +fn fold_digest(receipts: &[ReadReceipt]) -> u64 { + let mut digest = Digest::new(); + for receipt in receipts { + digest = digest + .number(receipt.digest) + .number(u64::from(receipt.ticket)); + } + digest.finish() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum TurnPurpose { + RoomTurn, + Preflight, +} + +impl TurnPurpose { + pub const fn label(self) -> &'static str { + match self { + Self::RoomTurn => "room_turn", + Self::Preflight => "preflight", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TurnScope { + pub room: RoomKey, + pub purpose: TurnPurpose, +} + +impl TurnScope { + pub const fn room_turn(room: RoomKey) -> Self { + Self { + room, + purpose: TurnPurpose::RoomTurn, + } + } + + pub const fn preflight(room: RoomKey) -> Self { + Self { + room, + purpose: TurnPurpose::Preflight, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ReadSource { + GatewayVoiceStates, + GatewayPendingJoins, + MediaParticipants, + MediaRoomList, + Census, +} + +impl ReadSource { + pub const fn label(self) -> &'static str { + match self { + Self::GatewayVoiceStates => "gateway_voice_states", + Self::GatewayPendingJoins => "gateway_pending_joins", + Self::MediaParticipants => "media_participants", + Self::MediaRoomList => "media_room_list", + Self::Census => "census", + } + } +} + +impl std::fmt::Display for ReadSource { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.label()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReadReceipt { + ticket: u32, + source: ReadSource, + at: Millis, + digest: u64, +} + +impl ReadReceipt { + pub const fn ticket(self) -> u32 { + self.ticket + } + + pub const fn source(self) -> ReadSource { + self.source + } + + pub const fn at(self) -> Millis { + self.at + } + + pub const fn digest(self) -> u64 { + self.digest + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum TurnError { + #[error("a read taken in turn {found} cannot be used in turn {expected}")] + StaleRead { expected: TurnId, found: TurnId }, + #[error("turn {turn} took {issued} reads but the observation accounts for {accounted}")] + UnaccountedReads { + turn: TurnId, + issued: usize, + accounted: usize, + }, + #[error("turn {turn} opened at {opened_at} cannot be sealed at {sealed_at}")] + SealedBeforeOpened { + turn: TurnId, + opened_at: Millis, + sealed_at: Millis, + }, + #[error("turn {turn} was never issued by this sequencer")] + UnknownTurn { turn: TurnId }, + #[error("turn {turn} carries a {read} read taken at {at}, outside the turn")] + ReadOutsideTurn { + turn: TurnId, + read: ReadSource, + at: Millis, + }, +} + +#[derive(Debug)] +#[must_use = "an opened turn must be sealed or abandoned"] +pub struct TurnToken { + turn: TurnId, + scope: TurnScope, + opened_at: Millis, + receipts: Vec, + discarded: BTreeSet, +} + +impl TurnToken { + pub const fn turn(&self) -> TurnId { + self.turn + } + + pub const fn scope(&self) -> TurnScope { + self.scope + } + + pub const fn room(&self) -> RoomKey { + self.scope.room + } + + pub const fn opened_at(&self) -> Millis { + self.opened_at + } + + pub fn reads(&self) -> &[ReadReceipt] { + &self.receipts + } + + pub fn read(&mut self, source: ReadSource, at: Millis, digest: u64, value: T) -> Fresh { + let ticket = u32::try_from(self.receipts.len()).unwrap_or(u32::MAX); + self.receipts.push(ReadReceipt { + ticket, + source, + at, + digest, + }); + Fresh { + turn: self.turn, + tickets: vec![ticket], + value, + } + } + + pub fn discard(&mut self, value: Fresh) -> Result { + if value.turn != self.turn { + return Err(TurnError::StaleRead { + expected: self.turn, + found: value.turn, + }); + } + self.discarded.extend(value.tickets.iter().copied()); + Ok(value.value) + } +} + +#[derive(Debug)] +#[must_use = "a read must be sealed into its turn or discarded"] +pub struct Fresh { + turn: TurnId, + tickets: Vec, + value: T, +} + +impl Fresh { + pub fn turn(&self) -> TurnId { + self.turn + } + + pub fn peek(&self) -> &T { + &self.value + } + + pub fn map(self, transform: F) -> Fresh + where + F: FnOnce(T) -> U, + { + Fresh { + turn: self.turn, + tickets: self.tickets, + value: transform(self.value), + } + } + + pub fn zip(self, other: Fresh) -> Result, TurnError> { + if self.turn != other.turn { + return Err(TurnError::StaleRead { + expected: self.turn, + found: other.turn, + }); + } + let mut tickets = self.tickets; + tickets.extend(other.tickets); + tickets.sort_unstable(); + tickets.dedup(); + Ok(Fresh { + turn: self.turn, + tickets, + value: (self.value, other.value), + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TurnRecord { + turn: TurnId, + scope: TurnScope, + opened_at: Millis, + sealed_at: Millis, + digest: u64, + reads: Vec, +} + +impl TurnRecord { + pub const fn turn(&self) -> TurnId { + self.turn + } + + pub const fn scope(&self) -> TurnScope { + self.scope + } + + pub const fn room(&self) -> RoomKey { + self.scope.room + } + + pub const fn opened_at(&self) -> Millis { + self.opened_at + } + + pub const fn sealed_at(&self) -> Millis { + self.sealed_at + } + + pub const fn digest(&self) -> u64 { + self.digest + } + + pub fn reads(&self) -> &[ReadReceipt] { + &self.reads + } + + pub const fn duration_ms(&self) -> u64 { + self.sealed_at.saturating_since(self.opened_at) + } + + pub const fn age_ms(&self, now: Millis) -> u64 { + now.saturating_since(self.sealed_at) + } + + pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { + self.age_ms(now) <= max_age_ms + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Observed { + record: TurnRecord, + value: T, +} + +impl Observed { + pub const fn record(&self) -> &TurnRecord { + &self.record + } + + pub const fn turn(&self) -> TurnId { + self.record.turn + } + + pub const fn scope(&self) -> TurnScope { + self.record.scope + } + + pub const fn room(&self) -> RoomKey { + self.record.scope.room + } + + pub const fn opened_at(&self) -> Millis { + self.record.opened_at + } + + pub const fn sealed_at(&self) -> Millis { + self.record.sealed_at + } + + pub const fn digest(&self) -> u64 { + self.record.digest + } + + pub fn reads(&self) -> &[ReadReceipt] { + &self.record.reads + } + + pub const fn age_ms(&self, now: Millis) -> u64 { + self.record.age_ms(now) + } + + pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { + self.record.is_fresh(now, max_age_ms) + } + + pub const fn value(&self) -> &T { + &self.value + } + + pub fn into_parts(self) -> (TurnRecord, T) { + (self.record, self.value) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomTurns { + last_opened: TurnId, + last_sealed: Option, +} + +impl RoomTurns { + pub const fn last_opened(&self) -> TurnId { + self.last_opened + } + + pub fn last_sealed(&self) -> Option<&TurnRecord> { + self.last_sealed.as_ref() + } +} + +#[derive(Clone, Debug)] +pub struct TurnSequencer { + next: TurnId, + opened_total: u64, + sealed_total: u64, + abandoned_total: u64, + rooms: BTreeMap, +} + +impl Default for TurnSequencer { + fn default() -> Self { + Self::new() + } +} + +impl TurnSequencer { + pub fn tracked_bytes(&self) -> u64 { + (self.rooms.len() as u64).saturating_mul(SEQUENCER_ENTRY_BYTES) + } + + pub const fn new() -> Self { + Self { + next: FIRST_TURN, + opened_total: 0, + sealed_total: 0, + abandoned_total: 0, + rooms: BTreeMap::new(), + } + } + + pub const fn next_turn(&self) -> TurnId { + self.next + } + + pub const fn opened_total(&self) -> u64 { + self.opened_total + } + + pub const fn sealed_total(&self) -> u64 { + self.sealed_total + } + + pub const fn abandoned_total(&self) -> u64 { + self.abandoned_total + } + + pub fn tracked_rooms(&self) -> usize { + self.rooms.len() + } + + pub fn history(&self, room: &RoomKey) -> Option<&RoomTurns> { + self.rooms.get(room) + } + + pub fn last_sealed(&self, room: &RoomKey) -> Option<&TurnRecord> { + self.rooms.get(room).and_then(RoomTurns::last_sealed) + } + + pub fn is_latest_sealed(&self, room: &RoomKey, turn: TurnId) -> bool { + self.last_sealed(room) + .is_some_and(|record| record.turn == turn) + } + + pub fn forget(&mut self, room: &RoomKey) -> bool { + self.rooms.remove(room).is_some() + } + + pub fn retain(&mut self, mut keep: F) + where + F: FnMut(&RoomKey) -> bool, + { + self.rooms.retain(|room, _| keep(room)); + } + + pub fn open(&mut self, scope: TurnScope, at: Millis) -> TurnToken { + let turn = self.next; + self.next = self.next.next(); + self.opened_total = self.opened_total.saturating_add(1); + self.rooms + .entry(scope.room) + .and_modify(|history| history.last_opened = turn) + .or_insert(RoomTurns { + last_opened: turn, + last_sealed: None, + }); + TurnToken { + turn, + scope, + opened_at: at, + receipts: Vec::new(), + discarded: BTreeSet::new(), + } + } + + pub fn seal( + &mut self, + token: TurnToken, + at: Millis, + value: Fresh, + ) -> Result, TurnError> { + if token.turn >= self.next { + return Err(TurnError::UnknownTurn { turn: token.turn }); + } + if value.turn != token.turn { + return Err(TurnError::StaleRead { + expected: token.turn, + found: value.turn, + }); + } + if at < token.opened_at { + return Err(TurnError::SealedBeforeOpened { + turn: token.turn, + opened_at: token.opened_at, + sealed_at: at, + }); + } + + for receipt in &token.receipts { + if receipt.at < token.opened_at || receipt.at > at { + return Err(TurnError::ReadOutsideTurn { + turn: token.turn, + read: receipt.source, + at: receipt.at, + }); + } + } + + let accounted: BTreeSet = value + .tickets + .iter() + .copied() + .chain(token.discarded.iter().copied()) + .collect(); + if accounted.len() != token.receipts.len() { + return Err(TurnError::UnaccountedReads { + turn: token.turn, + issued: token.receipts.len(), + accounted: accounted.len(), + }); + } + + let record = TurnRecord { + turn: token.turn, + scope: token.scope, + opened_at: token.opened_at, + sealed_at: at, + digest: fold_digest(&token.receipts), + reads: token.receipts, + }; + self.sealed_total = self.sealed_total.saturating_add(1); + self.rooms + .entry(record.scope.room) + .and_modify(|history| history.last_sealed = Some(record.clone())) + .or_insert_with(|| RoomTurns { + last_opened: record.turn, + last_sealed: Some(record.clone()), + }); + + Ok(Observed { + record, + value: value.value, + }) + } + + pub fn abandon(&mut self, token: TurnToken) -> TurnId { + self.abandoned_total = self.abandoned_total.saturating_add(1); + token.turn + } +} diff --git a/fluxer_recon/src/webhook.rs b/fluxer_recon/src/webhook.rs new file mode 100644 index 000000000..16cef8114 --- /dev/null +++ b/fluxer_recon/src/webhook.rs @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::net::SocketAddr; + +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use base64::prelude::*; +use hmac::{Hmac, KeyInit, Mac}; +use serde::Deserialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::clock::SharedClock; +use crate::ids::RoomKey; +use crate::ids::{ApiSecret, Location, Millis}; +use crate::names::{ParticipantIdentity, parse_participant_identity, parse_room_name}; +use crate::runtime::{RuntimeState, Shared}; +use crate::suspicion::{Raised, SuspicionSource}; + +type HmacSha256 = Hmac; + +pub const WEBHOOK_PATH: &str = "/livekit/webhook"; +pub const MAX_WEBHOOK_BODY_BYTES: usize = 64 * 1024; + +const HEADER_ALG: &str = "HS256"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum WebhookError { + #[error("the webhook token is not three dot separated parts")] + Malformed, + #[error("the webhook token is not signed with HS256")] + UnsupportedHeader, + #[error("a webhook token part is not base64url")] + BadBase64, + #[error("the webhook token does not decode as json")] + BadJson, + #[error("the webhook token signature does not verify")] + BadSignature, + #[error("the webhook token names an api key this fleet does not carry")] + UnknownIssuer, + #[error("the webhook token carries no body digest")] + MissingDigest, + #[error("the webhook body does not match the digest the token signed")] + DigestMismatch, + #[error("the webhook token has expired")] + Expired, + #[error("the webhook body does not decode as json")] + BadBody, +} + +impl WebhookError { + pub const fn label(self) -> &'static str { + match self { + Self::Malformed => "malformed", + Self::UnsupportedHeader => "unsupported_header", + Self::BadBase64 => "bad_base64", + Self::BadJson => "bad_json", + Self::BadSignature => "bad_signature", + Self::UnknownIssuer => "unknown_issuer", + Self::MissingDigest => "missing_digest", + Self::DigestMismatch => "digest_mismatch", + Self::Expired => "expired", + Self::BadBody => "bad_body", + } + } + + pub const fn is_authentication_failure(self) -> bool { + match self { + Self::Malformed + | Self::UnsupportedHeader + | Self::BadBase64 + | Self::BadJson + | Self::BadSignature + | Self::UnknownIssuer + | Self::MissingDigest + | Self::DigestMismatch + | Self::Expired => true, + Self::BadBody => false, + } + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +struct TokenHeader { + alg: String, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct WebhookClaims { + #[serde(default)] + pub iss: String, + #[serde(default)] + pub exp: u64, + #[serde(default)] + pub sha256: Option, +} + +fn decode_part(part: &str) -> Result, WebhookError> { + BASE64_URL_SAFE_NO_PAD + .decode(part.as_bytes()) + .or_else(|_| BASE64_STANDARD.decode(part.as_bytes())) + .map_err(|_| WebhookError::BadBase64) +} + +fn decode_digest(value: &str) -> Option> { + BASE64_STANDARD + .decode(value.as_bytes()) + .or_else(|_| BASE64_URL_SAFE_NO_PAD.decode(value.as_bytes())) + .ok() +} + +pub fn body_digest(body: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(body); + BASE64_STANDARD.encode(hasher.finalize()) +} + +pub fn verify_webhook_token( + token: &str, + api_key: &str, + api_secret: &ApiSecret, + body: &[u8], + now_unix_seconds: u64, +) -> Result { + let token = token.trim(); + let token = token.strip_prefix("Bearer ").unwrap_or(token).trim(); + + let mut parts = token.split('.'); + let (Some(header_b64), Some(payload_b64), Some(signature_b64), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(WebhookError::Malformed); + }; + + let header_bytes = decode_part(header_b64)?; + let header: TokenHeader = + serde_json::from_slice(&header_bytes).map_err(|_| WebhookError::BadJson)?; + if header.alg != HEADER_ALG { + return Err(WebhookError::UnsupportedHeader); + } + + let payload_bytes = decode_part(payload_b64)?; + let claims: WebhookClaims = + serde_json::from_slice(&payload_bytes).map_err(|_| WebhookError::BadJson)?; + if claims.iss != api_key { + return Err(WebhookError::UnknownIssuer); + } + + let signature = decode_part(signature_b64)?; + let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) + .expect("hmac accepts any key length"); + mac.update(header_b64.as_bytes()); + mac.update(b"."); + mac.update(payload_b64.as_bytes()); + mac.verify_slice(&signature) + .map_err(|_| WebhookError::BadSignature)?; + + let Some(claimed) = claims.sha256.as_deref() else { + return Err(WebhookError::MissingDigest); + }; + let (Some(claimed), Some(actual)) = (decode_digest(claimed), decode_digest(&body_digest(body))) + else { + return Err(WebhookError::DigestMismatch); + }; + if claimed != actual { + return Err(WebhookError::DigestMismatch); + } + + if claims.exp > 0 && now_unix_seconds >= claims.exp { + return Err(WebhookError::Expired); + } + + Ok(claims) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum WebhookEventKind { + RoomStarted, + RoomFinished, + ParticipantJoined, + ParticipantLeft, + TrackPublished, + TrackUnpublished, + Unrecognised, +} + +impl WebhookEventKind { + pub const ALL: [Self; 7] = [ + Self::RoomStarted, + Self::RoomFinished, + Self::ParticipantJoined, + Self::ParticipantLeft, + Self::TrackPublished, + Self::TrackUnpublished, + Self::Unrecognised, + ]; + + pub const fn wire(self) -> &'static str { + match self { + Self::RoomStarted => "room_started", + Self::RoomFinished => "room_finished", + Self::ParticipantJoined => "participant_joined", + Self::ParticipantLeft => "participant_left", + Self::TrackPublished => "track_published", + Self::TrackUnpublished => "track_unpublished", + Self::Unrecognised => "unrecognised", + } + } + + pub fn from_wire(value: &str) -> Self { + match value { + "room_started" => Self::RoomStarted, + "room_finished" => Self::RoomFinished, + "participant_joined" => Self::ParticipantJoined, + "participant_left" => Self::ParticipantLeft, + "track_published" => Self::TrackPublished, + "track_unpublished" => Self::TrackUnpublished, + _ => Self::Unrecognised, + } + } + + pub const fn changes_the_roster(self) -> bool { + match self { + Self::RoomFinished + | Self::ParticipantJoined + | Self::ParticipantLeft + | Self::TrackPublished + | Self::TrackUnpublished => true, + Self::RoomStarted | Self::Unrecognised => false, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WebhookEvent { + pub kind: WebhookEventKind, + pub room: Option, + pub identity: Option, + pub num_dropped: u32, + pub created_at_unix_seconds: u64, + pub unparseable_room: bool, + pub unparseable_identity: bool, +} + +fn text_of(value: &Value) -> Option<&str> { + value.as_str() +} + +fn number_of(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_u64(), + Value::String(raw) => raw.parse::().ok(), + Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None, + } +} + +pub fn decode_event(body: &[u8]) -> Result { + let value: Value = serde_json::from_slice(body).map_err(|_| WebhookError::BadBody)?; + if !value.is_object() { + return Err(WebhookError::BadBody); + } + + let kind = value + .get("event") + .and_then(text_of) + .map_or(WebhookEventKind::Unrecognised, WebhookEventKind::from_wire); + + let named_room = value.get("room").and_then(|room| room.get("name")); + let room = named_room.and_then(text_of).map(parse_room_name); + let named_identity = value + .get("participant") + .and_then(|participant| participant.get("identity")); + let identity = named_identity + .and_then(text_of) + .map(parse_participant_identity); + + Ok(WebhookEvent { + kind, + room: room + .as_ref() + .and_then(|parsed| parsed.as_ref().ok()) + .copied(), + identity: identity + .as_ref() + .and_then(|parsed| parsed.as_ref().ok()) + .cloned(), + num_dropped: value + .get("numDropped") + .or_else(|| value.get("num_dropped")) + .and_then(number_of) + .and_then(|dropped| u32::try_from(dropped).ok()) + .unwrap_or(0), + created_at_unix_seconds: value + .get("createdAt") + .or_else(|| value.get("created_at")) + .and_then(number_of) + .unwrap_or(0), + unparseable_room: named_room.is_some() && room.is_none_or(|parsed| parsed.is_err()), + unparseable_identity: named_identity.is_some() + && identity.is_none_or(|parsed| parsed.is_err()), + }) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Ingested { + pub raised: Option, + pub holed: bool, +} + +impl Ingested { + pub const fn nothing() -> Self { + Self { + raised: None, + holed: false, + } + } +} + +pub fn ingest( + state: &mut RuntimeState, + event: &WebhookEvent, + location: &Location, + at: Millis, +) -> Ingested { + let holed = state + .suspicion + .note_dropped(location, event.num_dropped, at); + let source = if holed { + SuspicionSource::WebhookDrop + } else { + SuspicionSource::Webhook + }; + + let raised = match event.room { + None => None, + Some(room) => { + if !holed && !event.kind.changes_the_roster() { + None + } else { + state.directory.note_suspicion(room, at); + Some(state.suspicion.note_room(room, source, at)) + } + } + }; + + Ingested { raised, holed } +} + +#[derive(Clone)] +pub struct WebhookLane { + shared: Shared, + clock: SharedClock, +} + +impl WebhookLane { + pub const fn new(shared: Shared, clock: SharedClock) -> Self { + Self { shared, clock } + } +} + +impl std::fmt::Debug for WebhookLane { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("WebhookLane").finish() + } +} + +fn credentials(shared: &Shared) -> Vec<(Location, Box, ApiSecret)> { + shared.with_state(|state| { + state + .topology + .servers() + .iter() + .map(|server| { + ( + server.location().clone(), + Box::from(server.api_key()), + server.api_secret().clone(), + ) + }) + .collect() + }) +} + +pub fn authenticate( + shared: &Shared, + token: &str, + body: &[u8], + now_unix_seconds: u64, +) -> Result { + let mut failure = WebhookError::UnknownIssuer; + for (location, api_key, api_secret) in credentials(shared) { + match verify_webhook_token(token, &api_key, &api_secret, body, now_unix_seconds) { + Ok(_) => return Ok(location), + Err(WebhookError::UnknownIssuer) => {} + Err(error) => failure = error, + } + } + Err(failure) +} + +pub fn accept( + shared: &Shared, + clock: &SharedClock, + token: &str, + body: &[u8], +) -> Result { + let wall_now = clock.wall_now().get() / 1_000; + let location = authenticate(shared, token, body, wall_now)?; + let event = decode_event(body)?; + let at = clock.now(); + + let ingested = shared.with_state_mut(|state| ingest(state, &event, &location, at)); + + let age_seconds = if event.created_at_unix_seconds == 0 { + 0.0 + } else { + wall_now.saturating_sub(event.created_at_unix_seconds) as f64 + }; + shared + .metrics() + .set_webhook_last_event_age(location.server.as_str(), age_seconds); + + Ok(ingested) +} + +async fn receive( + State(lane): State, + headers: HeaderMap, + body: axum::body::Bytes, +) -> StatusCode { + if body.len() > MAX_WEBHOOK_BODY_BYTES { + return StatusCode::PAYLOAD_TOO_LARGE; + } + let token = headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + + match accept(&lane.shared, &lane.clock, token, &body) { + Ok(ingested) => { + tracing::debug!( + raised = ?ingested.raised, + holed = ingested.holed, + "a livekit webhook raised suspicion, which only reorders reads" + ); + StatusCode::NO_CONTENT + } + Err(error) => { + tracing::warn!( + error = error.label(), + "a livekit webhook was refused, so detection stays on the polling path" + ); + if error.is_authentication_failure() { + StatusCode::UNAUTHORIZED + } else { + StatusCode::BAD_REQUEST + } + } + } +} + +pub fn router(shared: Shared, clock: SharedClock) -> axum::Router { + axum::Router::new() + .route(WEBHOOK_PATH, axum::routing::post(receive)) + .with_state(WebhookLane::new(shared, clock)) +} + +pub async fn run_webhook( + shared: Shared, + clock: SharedClock, + address: SocketAddr, +) -> anyhow::Result<()> { + let listener = tokio::net::TcpListener::bind(address).await?; + tracing::info!( + addr = %address, + path = WEBHOOK_PATH, + "webhook listener accepting livekit events, which are suspicion only" + ); + axum::serve(listener, router(shared, clock)).await?; + Ok(()) +} diff --git a/fluxer_svc/src/metrics.rs b/fluxer_svc/src/metrics.rs index b725b4de4..86ac5cdfa 100644 --- a/fluxer_svc/src/metrics.rs +++ b/fluxer_svc/src/metrics.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; const ORDERING: Ordering = Ordering::Relaxed; -pub(crate) type AdditionalMetricsRenderer = Arc; +pub type AdditionalMetricsRenderer = Arc; const HISTOGRAM_BUCKETS_MS: &[u64] = &[ 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, @@ -92,7 +92,7 @@ impl Default for ServiceMetrics { } impl ServiceMetrics { - pub(crate) fn with_additional_renderer(renderer: AdditionalMetricsRenderer) -> Self { + pub fn with_additional_renderer(renderer: AdditionalMetricsRenderer) -> Self { Self { additional_renderer: Some(renderer), ..Self::default() diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index 02eab61ce..0fdae7cc3 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -117,15 +117,6 @@ export interface MasterConfig { lane?: 'realtime' | 'unfurl' | 'lifecycle' | 'batch'; task?: string; enable_cron_scheduler?: boolean; - enable_voice_reconciliation?: boolean; - voice_reconciliation?: { - interval_ms?: number; - stagger_delay_ms?: number; - lock_ttl_seconds?: number; - cadence_ttl_seconds?: number; - gateway_only_grace_ms?: number; - livekit_only_grace_ms?: number; - }; lane_concurrency_overrides?: { realtime?: number; unfurl?: number; diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index c99e8821c..4ef7cdece 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -107,34 +107,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { path: ['services', 'api', 'worker', 'enable_cron_scheduler'], parse: parseEnvValue, }, - FLUXER_API_WORKER_ENABLE_VOICE_RECONCILIATION: { - path: ['services', 'api', 'worker', 'enable_voice_reconciliation'], - parse: parseEnvValue, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_INTERVAL_MS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'interval_ms'], - parse: parseInteger, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_STAGGER_DELAY_MS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'stagger_delay_ms'], - parse: parseInteger, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_LOCK_TTL_SECONDS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'lock_ttl_seconds'], - parse: parseInteger, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_CADENCE_TTL_SECONDS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'cadence_ttl_seconds'], - parse: parseInteger, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_GATEWAY_ONLY_GRACE_MS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'gateway_only_grace_ms'], - parse: parseInteger, - }, - FLUXER_API_WORKER_VOICE_RECONCILIATION_LIVEKIT_ONLY_GRACE_MS: { - path: ['services', 'api', 'worker', 'voice_reconciliation', 'livekit_only_grace_ms'], - parse: parseInteger, - }, FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: { path: ['services', 'api', 'worker', 'lane_concurrency_overrides'], parse: parseEnvValue, diff --git a/tools/ci/src/ci_workflow.rs b/tools/ci/src/ci_workflow.rs index a2c8aca7e..de041e58e 100644 --- a/tools/ci/src/ci_workflow.rs +++ b/tools/ci/src/ci_workflow.rs @@ -378,6 +378,11 @@ mod tests { "fluxer-messages", include_str!("../../../fluxer_messages/Dockerfile"), ), + ( + "fluxer_recon", + "fluxer-recon", + include_str!("../../../fluxer_recon/Dockerfile"), + ), ( "fluxer_snowflakes", "fluxer-snowflakes", diff --git a/tools/ci/src/image_set.rs b/tools/ci/src/image_set.rs index 86e277d45..3e8486d6d 100644 --- a/tools/ci/src/image_set.rs +++ b/tools/ci/src/image_set.rs @@ -68,6 +68,10 @@ const COMPONENTS: &[Component] = &[ image: "fluxer-messages", services: &["messages", "messages-shard"], }, + Component { + image: "fluxer-recon", + services: &[], + }, Component { image: "fluxer-snowflakes", services: &["snowflakes", "snowflakes-shard"],