diff --git a/.github/labeller.yaml b/.github/labeller.yaml index c11d62dcf..abbf293ab 100644 --- a/.github/labeller.yaml +++ b/.github/labeller.yaml @@ -28,9 +28,6 @@ f:media_proxy: f:messages: - changed-files: - any-glob-to-any-file: fluxer_messages/**/* -f:recon: - - changed-files: - - any-glob-to-any-file: fluxer_recon/**/* f:snowflakes: - changed-files: - any-glob-to-any-file: fluxer_snowflakes/**/* diff --git a/.github/workflows/build-recon.yaml b/.github/workflows/build-recon.yaml deleted file mode 100644 index 4ec9b31a2..000000000 --- a/.github/workflows/build-recon.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -name: build recon - -on: - workflow_dispatch: - inputs: - build-version: - description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation" - type: string - required: false - default: "" - -permissions: - actions: read - contents: write - packages: write - -jobs: - approve: - name: approve build release - permissions: {} - runs-on: ubuntu-24.04 - environment: builds - timeout-minutes: 5 - steps: - - name: approved - run: echo "Build release approved." - - image: - needs: approve - uses: ./.github/workflows/_build-image.yaml - secrets: inherit - with: - image: fluxer-recon - dockerfile: fluxer_recon/Dockerfile - build-version: ${{ inputs['build-version'] }} diff --git a/Cargo.lock b/Cargo.lock index 473412206..3f47dfac2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1868,25 +1868,6 @@ dependencies = [ "url", ] -[[package]] -name = "fluxer-recon" -version = "0.0.0" -dependencies = [ - "anyhow", - "axum", - "base64", - "fluxer-svc", - "hmac 0.13.0", - "reqwest", - "scylla", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror", - "tokio", - "tracing", -] - [[package]] name = "fluxer-snowflakes" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 34470edaa..277a6e72a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ members = [ "fluxer_users", "fluxer_unfurl", "packages/markdown_parser/rust", - "fluxer_recon", ] exclude = [ "packages/markdown_parser/rust/fuzz", diff --git a/config/env/development.env b/config/env/development.env index 803e48440..a06df9451 100644 --- a/config/env/development.env +++ b/config/env/development.env @@ -65,11 +65,6 @@ FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686} -FLUXER_RECON_MODE=observing -FLUXER_RECON_EXPECTED_ROOMS=64 -FLUXER_RECON_WARMUP_SECONDS=15 -FLUXER_RECON_MAX_HOT_ROOMS=8 - FLUXER_API_PORT=8080 FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED=true FLUXER_API_WORKER_MODE=all_lanes diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index be9cd9783..93f2b9b0a 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -694,8 +694,6 @@ To rotate the key pair, set both names in `.env`, then run `docker compose up -d The `Caddyfile` is a bind mount, so the edge reads the copy that sits on disk beside `docker-compose.yml`. Editing it takes `docker compose restart edge`, because `docker compose up -d` leaves a container alone when only a mounted file changed. An upgrade does that restart itself, which [What the script does](/operator/upgrading/#what-the-script-does) covers. A change to any LiveKit value in `.env` needs `docker compose up -d`, because `restart` reuses the existing container with its old environment. -Voice reconciliation has moved from `worker` to the separate recon service. The old `FLUXER_API_WORKER_ENABLE_VOICE_RECONCILIATION` and `FLUXER_API_WORKER_VOICE_RECONCILIATION_*` settings no longer control it. The bundled Compose stack does not start the recon service. - ## Email Email is off by default, and the conditions below turn it on. The email switch must be on. It is the value saved in the admin dashboard, or `FLUXER_EMAIL_ENABLED` when the dashboard has no saved value. Also, the provider must be `smtp` with a complete SMTP configuration, meaning `FLUXER_EMAIL_FROM_EMAIL`, `FLUXER_EMAIL_SMTP_HOST`, `FLUXER_EMAIL_SMTP_PORT`, `FLUXER_EMAIL_SMTP_USERNAME`, and `FLUXER_EMAIL_SMTP_PASSWORD` are all non-empty. All are optional. diff --git a/fluxer_recon/Cargo.toml b/fluxer_recon/Cargo.toml deleted file mode 100644 index ce00b1c94..000000000 --- a/fluxer_recon/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "fluxer-recon" -edition.workspace = true -license.workspace = true -publish = false - -[dependencies] -fluxer-svc = { path = "../fluxer_svc" } -anyhow = "1.0.104" -thiserror = "2.0.18" -tokio = { version = "1.52.3", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } -serde = { version = "1.0.228", features = ["derive"] } -serde_json = "1.0.150" -tracing = "0.1.44" -reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] } -hmac = "0.13.0" -sha2 = "0.11.0" -base64 = "0.22.1" -axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] } -scylla = { version = "1.6.0", optional = true } - -[features] -default = [] -scylla = ["dep:scylla", "fluxer-svc/scylla"] diff --git a/fluxer_recon/Dockerfile b/fluxer_recon/Dockerfile deleted file mode 100644 index f6f208eb1..000000000 --- a/fluxer_recon/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later - -FROM rust:1-bookworm AS builder - -WORKDIR /usr/src/app - -COPY . . - -RUN cargo build --release -p fluxer-recon --features scylla - -FROM debian:bookworm-slim - -ARG BUILD_VERSION="" -ARG SOURCE_SHA="" -ARG SOURCE_DATE="" - -LABEL org.opencontainers.image.title="fluxer-recon" -LABEL org.opencontainers.image.description="Fluxer voice reconciliation service" -LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" -LABEL org.opencontainers.image.vendor="Fluxer" -LABEL org.opencontainers.image.url="https://fluxer.app" -LABEL org.opencontainers.image.documentation="https://docs.fluxer.app" -LABEL org.opencontainers.image.source="https://github.com/fluxerapp/fluxer" -LABEL org.opencontainers.image.version="${BUILD_VERSION}" -LABEL org.opencontainers.image.revision="${SOURCE_SHA}" -LABEL org.opencontainers.image.created="${SOURCE_DATE}" -LABEL app.fluxer.build-version="${BUILD_VERSION}" - -WORKDIR /usr/local/bin - -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=builder /usr/src/app/target/release/fluxer-recon /usr/local/bin/fluxer-recon - -ENV BUILD_VERSION="${BUILD_VERSION}" - -USER 65532:65532 - -EXPOSE 8090 - -CMD ["/usr/local/bin/fluxer-recon"] diff --git a/fluxer_recon/src/actuate.rs b/fluxer_recon/src/actuate.rs deleted file mode 100644 index 0482afade..000000000 --- a/fluxer_recon/src/actuate.rs +++ /dev/null @@ -1,1325 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::budget::{ - AuthorizationOutcome, AuthorizationRequest, BudgetClass, MutationAuthorization, -}; -use crate::clock::Clock; -use crate::config::{ConnectionIdGuard, ReconConfig}; -use crate::decide::{Decision, DecisionAction}; -use crate::evidence::{GatewayVoiceState, MediaSighting, PendingJoin}; -use crate::gateway::codes::{GatewayError, GatewayErrorCode}; -use crate::gateway::{DisconnectOutcome, GatewayApi, GatewayFault, Nonce, RepairOutcome}; -use crate::guards::{ - AbortReason, DmPosture, FleetSilence, GatewayPreflight, GatewayRemovalGuard, MediaCoverage, - MediaCustody, MediaRemovalGuard, PreflightVoiceState, PreflightWindow, UnknownMediaCause, - connection_id_is_honoured, fleet_silence_cause, gateway_removal_preflight, - media_coverage_cause, media_coverage_refusal, media_custody_cause, media_removal_preflight, -}; -use crate::health::ModeClamp; -use crate::ids::{ConnectionId, ConnectionKey, Location, Millis, RoomKey, TurnId, UserId}; -use crate::ledger::{ - ActionKind, ConnectionLedger, ConnectionState, Ledger, LocationIndex, RepairVerdict, RoomLedger, -}; -use crate::livekit::{LiveKitApi, LiveKitFault, ReadResult, RemoveOutcome}; -use crate::metrics::{MutationOutcome, ReconMetrics}; -use crate::names::{ParticipantIdentity, parse_participant_identity}; -use crate::runtime::{RuntimeState, Shared}; -use crate::turn::{Digest, Fresh, ReadSource, TurnError, TurnRecord, TurnScope, TurnSequencer}; - -pub const PREFLIGHT_GATEWAY_READS: u32 = 2; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Lane { - Constructive, - Destructive, -} - -impl Lane { - pub const fn label(self) -> &'static str { - match self { - Self::Constructive => "constructive", - Self::Destructive => "destructive", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Actuation { - Held, - Refused { - lane: Lane, - reason: AbortReason, - }, - Counterfactual { - lane: Lane, - reason: AbortReason, - }, - Issued { - lane: Lane, - call: &'static str, - outcome: &'static str, - }, - Failed { - lane: Lane, - call: &'static str, - fault: &'static str, - }, -} - -impl Actuation { - pub const fn called_out(&self) -> bool { - matches!(self, Self::Issued { .. } | Self::Failed { .. }) - } - - pub const fn mutated(&self) -> bool { - matches!(self, Self::Issued { .. }) - } - - pub const fn lane(&self) -> Option { - match self { - Self::Held => None, - Self::Refused { lane, .. } - | Self::Counterfactual { lane, .. } - | Self::Issued { lane, .. } - | Self::Failed { lane, .. } => Some(*lane), - } - } - - pub const fn refusal(&self) -> Option { - match self { - Self::Held | Self::Issued { .. } | Self::Failed { .. } => None, - Self::Refused { reason, .. } | Self::Counterfactual { reason, .. } => Some(*reason), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ActuationPolicy { - pub dm_posture: DmPosture, - pub connection_id_guard: ConnectionIdGuard, - pub preflight_max_age_ms: u64, - pub pending_join_skew_ms: u64, -} - -impl ActuationPolicy { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - dm_posture: DmPosture { - dm_gateway_eviction: config.dm_gateway_eviction, - dm_media_eviction: config.dm_media_eviction, - }, - connection_id_guard: config.gateway_connection_id_guard, - preflight_max_age_ms: config.preflight_max_age_ms, - pending_join_skew_ms: config.pending_join_skew_ms, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TurnFacts { - pub room: RoomKey, - pub authorizing_turn: TurnId, - pub media: Vec, -} - -impl TurnFacts { - pub fn media_connections(&self) -> Vec { - self.media - .iter() - .map(|sighting| sighting.connection.clone()) - .collect() - } - - pub fn media_siblings_of(&self, user_id: UserId, target: &ConnectionId) -> Vec { - self.media - .iter() - .filter(|sighting| sighting.user_id == user_id && &sighting.connection != target) - .map(|sighting| sighting.connection.clone()) - .collect() - } -} - -pub struct Ports<'a, G, L> { - pub gateway: &'a G, - pub livekit: &'a L, - pub clock: &'a dyn Clock, - pub shared: &'a Shared, - pub policy: ActuationPolicy, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PreflightReads { - pub voice_states: Vec, - pub pending_joins: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum PreflightError { - Gateway(GatewayFault), - Turn(TurnError), -} - -impl PreflightError { - pub const fn abort_reason(&self) -> AbortReason { - match self { - Self::Gateway(_) => AbortReason::UnknownGateway, - Self::Turn(_) => AbortReason::PreflightDisagreed, - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Gateway(fault) => fault.label(), - Self::Turn(_) => "turn_violation", - } - } -} - -fn digest_voice_states(states: &[GatewayVoiceState]) -> u64 { - let mut digest = Digest::new().text("preflight_voice_states"); - for state in states { - digest = digest - .text(state.connection.as_ref().map_or("", ConnectionId::as_str)) - .flag(state.connection.is_some()) - .number(state.user_id.get()) - .number(state.channel_id.get()); - } - digest.finish() -} - -fn digest_pending_joins(joins: &[PendingJoin]) -> u64 { - let mut digest = Digest::new().text("preflight_pending_joins"); - for join in joins { - digest = digest - .text(join.connection.as_str()) - .number(join.user_id.get()) - .number(join.expires_at.get()); - } - digest.finish() -} - -pub fn open_preflight( - state: &mut RuntimeState, - room: RoomKey, - now: Millis, -) -> Result { - if !state - .governor - .try_acquire_many(BudgetClass::GatewayRead, PREFLIGHT_GATEWAY_READS, now) - { - return Err(AbortReason::Budget); - } - Ok(state.sequencer.open(TurnScope::preflight(room), now)) -} - -pub async fn read_preflight( - gateway: &G, - room: RoomKey, - clock: &dyn Clock, - token: &mut crate::turn::TurnToken, -) -> Result, PreflightError> -where - G: GatewayApi, -{ - let states = gateway - .voice_states_for_channel(room) - .await - .map_err(PreflightError::Gateway)?; - let fresh_states = token.read( - ReadSource::GatewayVoiceStates, - clock.now(), - digest_voice_states(&states), - states, - ); - - let joins = match gateway.pending_joins_for_channel(room).await { - Ok(joins) => joins, - Err(fault) => { - let _ = token.discard(fresh_states); - return Err(PreflightError::Gateway(fault)); - } - }; - let fresh_joins = token.read( - ReadSource::GatewayPendingJoins, - clock.now(), - digest_pending_joins(&joins), - joins, - ); - - fresh_states - .zip(fresh_joins) - .map(|both| { - both.map(|(voice_states, pending_joins)| PreflightReads { - voice_states, - pending_joins, - }) - }) - .map_err(PreflightError::Turn) -} - -pub fn seal_preflight( - sequencer: &mut TurnSequencer, - token: crate::turn::TurnToken, - at: Millis, - reads: Fresh, - media_present: Vec, -) -> Result<(TurnRecord, GatewayPreflight), TurnError> { - let turn = token.turn(); - let taken_at = token.opened_at(); - let preflight = reads.map(|reads| GatewayPreflight { - turn, - taken_at, - voice_states: reads - .voice_states - .iter() - .map(|state| PreflightVoiceState { - user_id: state.user_id, - connection: state.connection.clone(), - }) - .collect(), - pending_joins: reads.pending_joins, - media_present, - }); - sequencer - .seal(token, at, preflight) - .map(crate::turn::Observed::into_parts) -} - -pub fn preflight_is_current( - sequencer: &TurnSequencer, - room: RoomKey, - record: &TurnRecord, - now: Millis, - max_age_ms: u64, -) -> Result<(), AbortReason> { - if !sequencer.is_latest_sealed(&room, record.turn()) { - return Err(AbortReason::PreflightDisagreed); - } - if !record.is_fresh(now, max_age_ms) { - return Err(AbortReason::PreflightStale); - } - Ok(()) -} - -pub fn media_sibling_refusal( - facts: &TurnFacts, - target: &ConnectionId, - user_id: UserId, - posture: ConnectionIdGuard, -) -> Option { - if connection_id_is_honoured(posture, facts.room) { - return None; - } - if facts.media_siblings_of(user_id, target).is_empty() { - None - } else { - Some(AbortReason::SiblingConnection) - } -} - -fn media_divergence_facts( - ledger: &Ledger, - key: &ConnectionKey, -) -> Option<(Millis, Millis, Option)> { - let entry = ledger.connection(key)?; - match &entry.state { - ConnectionState::MediaOnly { - since, - participant_joined_at, - repair_verdict, - .. - } => Some((*since, *participant_joined_at, *repair_verdict)), - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::GatewayOnly { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::ActionTaken { .. } - | ConnectionState::Wedged { .. } - | ConnectionState::Retired { .. } => None, - } -} - -fn mutations_are_paused(fault: &GatewayFault) -> bool { - matches!( - fault, - GatewayFault::NotOk(GatewayError::Known(GatewayErrorCode::EventMutationsPaused)) - ) -} - -fn note_mutation_fault(state: &mut RuntimeState, fault: &GatewayFault, now: Millis) { - if mutations_are_paused(fault) - && let Some(until) = state.mutations_pause.note_refused(now) - { - state - .governor - .clamps_mut() - .engage_until(ModeClamp::MutationsPaused, until); - tracing::warn!( - hold_ms = state.mutations_pause.hold_ms(), - windows = state.mutations_pause.windows(), - "the gateway is refusing mutations, holding the destructive lane for one backoff \ - window that expires on its own" - ); - } -} - -fn note_mutation_success(state: &mut RuntimeState) { - state.mutations_pause.clear(); - state - .governor - .clamps_mut() - .set(ModeClamp::MutationsPaused, false); -} - -fn refuse(metrics: &ReconMetrics, lane: Lane, reason: AbortReason) -> Actuation { - if matches!(lane, Lane::Destructive) { - metrics.record_eviction_aborted(reason); - } - Actuation::Refused { lane, reason } -} - -fn counterfactual(metrics: &ReconMetrics, lane: Lane, reason: AbortReason) -> Actuation { - if matches!(lane, Lane::Destructive) { - metrics.record_eviction_aborted(reason); - } - Actuation::Counterfactual { lane, reason } -} - -fn hold_gateway_slot(shared: &Shared) -> bool { - shared.with_state_mut(|state| state.governor.gateway_inflight().try_acquire()) -} - -fn release_gateway_slot(shared: &Shared) { - shared.with_state_mut(|state| state.governor.gateway_inflight().release()); -} - -fn hold_media_slot(shared: &Shared, location: &Location) -> bool { - shared.with_state_mut(|state| { - if !state.governor.livekit_inflight().try_acquire() { - return false; - } - if !state.governor.server_inflight(location).try_acquire() { - state.governor.livekit_inflight().release(); - return false; - } - true - }) -} - -fn release_media_slot(shared: &Shared, location: &Location) { - shared.with_state_mut(|state| { - state.governor.server_inflight(location).release(); - state.governor.livekit_inflight().release(); - }); -} - -fn constructive_gate( - state: &mut RuntimeState, - connection: &ConnectionKey, - now: Millis, -) -> Result<(), (bool, AbortReason)> { - match state.governor.authorize_constructive(connection, now) { - Ok(()) => Ok(()), - Err(AbortReason::Mode) => Err((true, AbortReason::Mode)), - Err(reason) => Err((false, reason)), - } -} - -pub async fn apply_decision( - ports: &Ports<'_, G, L>, - facts: &TurnFacts, - decision: &Decision, -) -> Actuation -where - G: GatewayApi, - L: LiveKitApi, -{ - let metrics = ports.shared.metrics().clone(); - - if let Some(reason) = decision.blocked_by { - return Actuation::Refused { - lane: if decision.is_destructive() { - Lane::Destructive - } else { - Lane::Constructive - }, - reason, - }; - } - - match &decision.action { - DecisionAction::Hold => Actuation::Held, - DecisionAction::ConfirmConnection { nonce } => { - issue_confirm(ports, &metrics, decision, nonce).await - } - DecisionAction::RepairState => issue_repair(ports, &metrics, decision).await, - DecisionAction::RemoveGatewayState => { - remove_gateway_state(ports, &metrics, facts, decision).await - } - DecisionAction::RemoveParticipant { location } => { - remove_participant(ports, &metrics, facts, decision, location).await - } - } -} - -async fn issue_confirm( - ports: &Ports<'_, G, L>, - metrics: &ReconMetrics, - decision: &Decision, - nonce: &Nonce, -) -> Actuation -where - G: GatewayApi, - L: LiveKitApi, -{ - let now = ports.clock.now(); - let key = decision.connection.clone(); - let room = key.room; - - if let Err((counterfactual_gate, reason)) = ports - .shared - .with_state_mut(|state| constructive_gate(state, &key, now)) - { - return if counterfactual_gate { - counterfactual(metrics, Lane::Constructive, reason) - } else { - refuse(metrics, Lane::Constructive, reason) - }; - } - - if !hold_gateway_slot(ports.shared) { - return refuse(metrics, Lane::Constructive, AbortReason::Budget); - } - let started = ports.clock.now(); - let result = ports - .gateway - .confirm_connection(room, &key.connection, nonce) - .await; - release_gateway_slot(ports.shared); - metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); - - let settled = ports.clock.now(); - ports.shared.with_state_mut(|state| { - state.ledger.record_confirm_issued(&key, settled); - match &result { - Err(fault) => note_mutation_fault(state, fault, settled), - Ok(_) => note_mutation_success(state), - } - }); - - match result { - Err(fault) => { - metrics.record_confirm("failed"); - metrics.record_gateway_rpc("confirm_connection", fault.label()); - Actuation::Failed { - lane: Lane::Constructive, - call: "confirm_connection", - fault: fault.label(), - } - } - Ok(outcome) => { - metrics.record_confirm(outcome.label()); - metrics.record_gateway_rpc("confirm_connection", outcome.label()); - Actuation::Issued { - lane: Lane::Constructive, - call: "confirm_connection", - outcome: outcome.label(), - } - } - } -} - -async fn issue_repair( - ports: &Ports<'_, G, L>, - metrics: &ReconMetrics, - decision: &Decision, -) -> Actuation -where - G: GatewayApi, - L: LiveKitApi, -{ - let now = ports.clock.now(); - let key = decision.connection.clone(); - let room = key.room; - - if let Err((counterfactual_gate, reason)) = ports - .shared - .with_state_mut(|state| constructive_gate(state, &key, now)) - { - return if counterfactual_gate { - counterfactual(metrics, Lane::Constructive, reason) - } else { - refuse(metrics, Lane::Constructive, reason) - }; - } - - if !hold_gateway_slot(ports.shared) { - return refuse(metrics, Lane::Constructive, AbortReason::Budget); - } - let started = ports.clock.now(); - let result = ports - .gateway - .repair_state_from_cache(room, decision.user_id, &key.connection) - .await; - release_gateway_slot(ports.shared); - metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); - - let settled = ports.clock.now(); - let verdict = result.as_ref().ok().map(RepairOutcome::verdict); - ports.shared.with_state_mut(|state| { - state.ledger.record_repair_issued(&key, settled); - match &result { - Err(fault) => note_mutation_fault(state, fault, settled), - Ok(_) => note_mutation_success(state), - } - if let Some(verdict) = verdict { - state.ledger.record_repair_verdict(&key, verdict); - } - }); - - match result { - Err(fault) => { - metrics.record_repair("failed"); - metrics.record_gateway_rpc("repair_state_from_cache", fault.label()); - Actuation::Failed { - lane: Lane::Constructive, - call: "repair_state_from_cache", - fault: fault.label(), - } - } - Ok(outcome) => { - metrics.record_repair_verdict(outcome.verdict()); - metrics.record_gateway_rpc("repair_state_from_cache", outcome.label()); - Actuation::Issued { - lane: Lane::Constructive, - call: "repair_state_from_cache", - outcome: outcome.label(), - } - } - } -} - -struct AuthorizedRemoval { - authorization: MutationAuthorization, -} - -enum RemovalGate { - Authorized(AuthorizedRemoval), - Denied { - counterfactual: bool, - reason: AbortReason, - cause: Option, - }, -} - -pub async fn sweep_media( - livekit: &L, - room: RoomKey, - fleet: Vec, - required: Vec, - budget: impl FnMut(&Location) -> bool, -) -> (Vec, MediaCoverage) -where - L: LiveKitApi, -{ - sweep_rosters(livekit, room, fleet, required, &[], budget).await -} - -pub async fn sweep_rosters( - livekit: &L, - room: RoomKey, - fleet: Vec, - required: Vec, - absent_ok: &[Location], - mut budget: impl FnMut(&Location) -> bool, -) -> (Vec, MediaCoverage) -where - L: LiveKitApi, -{ - let mut coverage = MediaCoverage::over(fleet, required); - let mut sightings: Vec = Vec::new(); - - for location in coverage.fleet().to_vec() { - if !budget(&location) { - continue; - } - let records = match livekit.list_participants(&location, room).await { - ReadResult::Read(records) => records, - ReadResult::Unreadable(LiveKitFault::NotFound) if absent_ok.contains(&location) => { - coverage.note_absent(&location); - continue; - } - ReadResult::Unreadable(_) => { - coverage.note_unreachable(&location); - continue; - } - }; - let mut seen: Vec = Vec::with_capacity(records.len()); - let mut readable = true; - for record in &records { - match parse_participant_identity(&record.identity) { - Err(_) => readable = false, - Ok(identity) => seen.push(MediaSighting { - connection: identity.connection_id, - user_id: identity.user_id, - }), - } - } - if readable { - coverage.note_read(&location); - sightings.extend(seen); - } - } - - sightings.sort_by(|left, right| left.connection.cmp(&right.connection)); - sightings.dedup_by(|left, right| left.connection == right.connection); - (sightings, coverage) -} - -fn fleet_locations(state: &RuntimeState, now: Millis) -> Vec { - state.topology.lens(now).live_locations() -} - -fn believed_footprint(state: &RuntimeState, room: RoomKey) -> Vec { - let mut locations = state.directory.locations_for(&room); - let homes = state - .ledger - .room(&room) - .map(RoomLedger::believed_homes) - .unwrap_or_default(); - for index in homes { - if let Some(location) = state.ledger.location(index) { - locations.push(location.clone()); - } - } - locations.sort(); - locations.dedup(); - locations -} - -fn absent_ok_locations(state: &RuntimeState, room: RoomKey, fleet: &[Location]) -> Vec { - fleet - .iter() - .filter(|location| state.directory.room_is_absent_from(location, &room)) - .cloned() - .collect() -} - -fn written_off_locations(state: &RuntimeState, fleet: &[Location]) -> Vec { - fleet - .iter() - .filter(|location| state.server_health.health(location).is_written_off()) - .cloned() - .collect() -} - -fn unprobeable_locations(state: &RuntimeState, fleet: &[Location], now: Millis) -> Vec { - fleet - .iter() - .filter(|location| !state.server_health.may_probe(location, now)) - .cloned() - .collect() -} - -fn record_roster_stamps( - state: &mut RuntimeState, - room: RoomKey, - fleet: &[Location], - read: &[Location], - at: Millis, -) { - let live: Vec = fleet - .iter() - .filter_map(|location| state.ledger.intern_location(location)) - .collect(); - let indices: Vec = read - .iter() - .filter_map(|location| state.ledger.location_index(location)) - .collect(); - let Some(entry) = state.ledger.room_mut(&room) else { - return; - }; - entry.retain_roster_stamps(&live); - for index in indices { - entry.note_roster_read(index, at); - } -} - -async fn sweep_fleet_media( - ports: &Ports<'_, G, L>, - room: RoomKey, -) -> (Vec, MediaCoverage) -where - G: GatewayApi, - L: LiveKitApi, -{ - let now = ports.clock.now(); - let (fleet, required, absent_ok, unprobeable) = ports.shared.with_state(|state| { - let fleet = fleet_locations(state, now); - let required = believed_footprint(state, room); - let absent_ok = absent_ok_locations(state, room, &fleet); - let unprobeable = unprobeable_locations(state, &fleet, now); - (fleet, required, absent_ok, unprobeable) - }); - let (sightings, mut coverage) = sweep_rosters( - ports.livekit, - room, - fleet, - required, - &absent_ok, - |location| { - if unprobeable.contains(location) { - return false; - } - ports.shared.with_state_mut(|state| { - let at = ports.clock.now(); - if !state.governor.try_acquire_server_read(location, 1, at) { - return false; - } - if state.governor.try_acquire(BudgetClass::LiveKitRead, at) { - return true; - } - state.governor.refund_server_read(location, 1); - false - }) - }, - ) - .await; - for location in &unprobeable { - coverage.note_unreachable(location); - } - let settled = ports.clock.now(); - let swept_fleet = coverage.fleet().to_vec(); - let read = coverage.read().to_vec(); - ports.shared.with_state_mut(|state| { - record_roster_stamps(state, room, &swept_fleet, &read, settled); - }); - (sightings, coverage) -} - -async fn take_preflight( - ports: &Ports<'_, G, L>, - room: RoomKey, - media_present: Vec, -) -> Result<(TurnRecord, GatewayPreflight), AbortReason> -where - G: GatewayApi, - L: LiveKitApi, -{ - let opened_at = ports.clock.now(); - let mut token = ports - .shared - .with_state_mut(|state| open_preflight(state, room, opened_at))?; - - if !hold_gateway_slot(ports.shared) { - ports - .shared - .with_state_mut(|state| state.sequencer.abandon(token)); - return Err(AbortReason::Budget); - } - let reads = read_preflight(ports.gateway, room, ports.clock, &mut token).await; - release_gateway_slot(ports.shared); - let sealed_at = ports.clock.now(); - - ports.shared.with_state_mut(|state| match reads { - Err(error) => { - state.sequencer.abandon(token); - Err(error.abort_reason()) - } - Ok(reads) => seal_preflight(&mut state.sequencer, token, sealed_at, reads, media_present) - .map_err(|_| AbortReason::PreflightDisagreed), - }) -} - -fn render_locations(locations: &[Location]) -> String { - locations - .iter() - .map(ToString::to_string) - .collect::>() - .join(",") -} - -struct CustodyView { - target_home: Option, - user_homes: Vec, - stamped: Vec, -} - -fn custody_view( - state: &RuntimeState, - key: &ConnectionKey, - user_id: UserId, - fleet: &[Location], -) -> CustodyView { - let entry = state.ledger.connection(key); - let target_home = entry - .and_then(|entry| entry.last_location) - .and_then(|index| state.ledger.location(index).cloned()); - let divergence_since = entry - .and_then(|entry| entry.state.divergence_since()) - .unwrap_or(Millis::new(0)); - let room_entry = state.ledger.room(&key.room); - let user_homes = room_entry - .map(|room_entry| { - room_entry - .connections() - .iter() - .filter(|connection| connection.user_id == user_id) - .filter_map(ConnectionLedger::believed_home) - .filter_map(|index| state.ledger.location(index).cloned()) - .collect::>() - }) - .unwrap_or_default(); - let stamped = fleet - .iter() - .filter(|location| { - state - .ledger - .location_index(location) - .and_then(|index| room_entry.and_then(|entry| entry.roster_read_at(index))) - .is_some_and(|at| at >= divergence_since) - }) - .cloned() - .collect(); - CustodyView { - target_home, - user_homes, - stamped, - } -} - -async fn remove_gateway_state( - ports: &Ports<'_, G, L>, - metrics: &ReconMetrics, - facts: &TurnFacts, - decision: &Decision, -) -> Actuation -where - G: GatewayApi, - L: LiveKitApi, -{ - let key = decision.connection.clone(); - let room = key.room; - let target = key.connection.clone(); - let user_id = decision.user_id; - - let (swept, coverage) = sweep_fleet_media(ports, room).await; - if let Some(reason) = media_coverage_refusal(&coverage, ports.policy.connection_id_guard, room) - { - if let Some(cause) = media_coverage_cause(&coverage, ports.policy.connection_id_guard, room) - { - metrics.record_unknown_media_cause(cause); - } - return refuse(metrics, Lane::Destructive, reason); - } - - let fleet = coverage.fleet().to_vec(); - let answered = coverage.answered(); - let read = coverage.read().to_vec(); - let silent = coverage.silent(); - - let (record, preflight) = match take_preflight(ports, room, swept).await { - Err(reason) => return refuse(metrics, Lane::Destructive, reason), - Ok(taken) => taken, - }; - - let now = ports.clock.now(); - let window = PreflightWindow { - now, - wall_now: ports.clock.wall_now(), - preflight_max_age_ms: ports.policy.preflight_max_age_ms, - pending_join_skew_ms: ports.policy.pending_join_skew_ms, - }; - - let gate = ports.shared.with_state_mut(|state| { - if let Err(reason) = preflight_is_current( - &state.sequencer, - room, - &record, - now, - ports.policy.preflight_max_age_ms, - ) { - return RemovalGate::Denied { - counterfactual: false, - reason, - cause: None, - }; - } - if let Err(reason) = gateway_removal_preflight(&GatewayRemovalGuard { - room, - target: &target, - user_id, - posture: ports.policy.connection_id_guard, - dm_posture: ports.policy.dm_posture, - preflight: &preflight, - window, - }) { - return RemovalGate::Denied { - counterfactual: false, - reason, - cause: None, - }; - } - if let Some(reason) = - media_sibling_refusal(facts, &target, user_id, ports.policy.connection_id_guard) - { - return RemovalGate::Denied { - counterfactual: false, - reason, - cause: None, - }; - } - let ledger_view = custody_view(state, &key, user_id, &fleet); - if let Some(cause) = media_custody_cause(&MediaCustody { - room, - posture: ports.policy.connection_id_guard, - fleet: &fleet, - answered: &answered, - read: &read, - target_home: ledger_view.target_home.as_ref(), - user_homes: &ledger_view.user_homes, - stamped: &ledger_view.stamped, - }) { - return RemovalGate::Denied { - counterfactual: false, - reason: AbortReason::UnknownMedia, - cause: Some(cause), - }; - } - let written_off = written_off_locations(state, &fleet); - if let Some(cause) = fleet_silence_cause(&FleetSilence { - room, - posture: ports.policy.connection_id_guard, - fleet: &fleet, - answered: &answered, - written_off: &written_off, - }) { - return RemovalGate::Denied { - counterfactual: false, - reason: AbortReason::UnknownMedia, - cause: Some(cause), - }; - } - authorize( - state, - &AuthorizationRequest { - connection: key.clone(), - user_id, - kind: ActionKind::RemoveGatewayState, - location: None, - authorizing_turn: facts.authorizing_turn, - preflight_turn: record.turn(), - preflight_at: record.opened_at(), - now, - }, - ) - }); - - let authorization = match gate { - RemovalGate::Denied { - counterfactual: is_counterfactual, - reason, - cause, - } => { - if let Some(cause) = cause { - metrics.record_unknown_media_cause(cause); - tracing::info!( - channel_id = room.channel_id().get(), - guild_id = room.guild_id().map(|guild| guild.get()), - connection = %target, - user_id = user_id.get(), - cause = cause.label(), - silent = %render_locations(&silent), - "a user scoped disconnect was held because the fleet did not answer in full" - ); - } - return if is_counterfactual { - counterfactual(metrics, Lane::Destructive, reason) - } else { - refuse(metrics, Lane::Destructive, reason) - }; - } - RemovalGate::Authorized(authorized) => authorized.authorization, - }; - - if !hold_gateway_slot(ports.shared) { - ports.shared.with_state_mut(|state| { - state.governor.refund_identity(&key); - }); - return refuse(metrics, Lane::Destructive, AbortReason::Budget); - } - metrics.record_mutation( - ActionKind::RemoveGatewayState, - room.scope(), - MutationOutcome::Issued, - ); - let started = ports.clock.now(); - let result = execute_gateway_removal(ports.gateway, authorization).await; - release_gateway_slot(ports.shared); - metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); - - let settled = ports.clock.now(); - ports.shared.with_state_mut(|state| { - state.ledger.record_action_issued( - &key, - ActionKind::RemoveGatewayState, - settled, - facts.authorizing_turn, - ); - match &result { - Err(fault) => { - note_mutation_fault(state, fault, settled); - if mutations_are_paused(fault) { - state.governor.refund_identity(&key); - state.ledger.refund_action_attempt(&key); - } - } - Ok(_) => note_mutation_success(state), - } - }); - - match result { - Err(fault) => { - metrics.record_mutation( - ActionKind::RemoveGatewayState, - room.scope(), - MutationOutcome::Failed, - ); - metrics.record_gateway_rpc("disconnect_user_if_in_channel", fault.label()); - Actuation::Failed { - lane: Lane::Destructive, - call: "disconnect_user_if_in_channel", - fault: fault.label(), - } - } - Ok(outcome) => { - metrics.record_mutation( - ActionKind::RemoveGatewayState, - room.scope(), - disconnect_outcome(&outcome), - ); - metrics.record_gateway_rpc("disconnect_user_if_in_channel", outcome.label()); - Actuation::Issued { - lane: Lane::Destructive, - call: "disconnect_user_if_in_channel", - outcome: outcome.label(), - } - } - } -} - -const fn disconnect_outcome(outcome: &DisconnectOutcome) -> MutationOutcome { - match outcome { - DisconnectOutcome::Removed => MutationOutcome::Succeeded, - DisconnectOutcome::Ignored { .. } | DisconnectOutcome::CallNotFound => { - MutationOutcome::NoChange - } - DisconnectOutcome::Failed(_) => MutationOutcome::Failed, - } -} - -pub async fn execute_gateway_removal( - gateway: &G, - authorization: MutationAuthorization, -) -> Result -where - G: GatewayApi, -{ - let room = authorization.connection.room; - let connection = authorization.connection.connection.clone(); - let user_id = authorization.user_id; - drop(authorization); - - gateway - .disconnect_user_if_in_channel(room, user_id, Some(&connection)) - .await -} - -async fn remove_participant( - ports: &Ports<'_, G, L>, - metrics: &ReconMetrics, - facts: &TurnFacts, - decision: &Decision, - location: &Location, -) -> Actuation -where - G: GatewayApi, - L: LiveKitApi, -{ - let key = decision.connection.clone(); - let room = key.room; - let target = key.connection.clone(); - let user_id = decision.user_id; - - let Some((since, joined_at, verdict)) = ports - .shared - .with_state(|state| media_divergence_facts(&state.ledger, &key)) - else { - return refuse(metrics, Lane::Destructive, AbortReason::PreflightDisagreed); - }; - - let (record, preflight) = match take_preflight(ports, room, facts.media.clone()).await { - Err(reason) => return refuse(metrics, Lane::Destructive, reason), - Ok(taken) => taken, - }; - - let now = ports.clock.now(); - let window = PreflightWindow { - now, - wall_now: ports.clock.wall_now(), - preflight_max_age_ms: ports.policy.preflight_max_age_ms, - pending_join_skew_ms: ports.policy.pending_join_skew_ms, - }; - - let gate = ports.shared.with_state_mut(|state| { - if let Err(reason) = preflight_is_current( - &state.sequencer, - room, - &record, - now, - ports.policy.preflight_max_age_ms, - ) { - return RemovalGate::Denied { - counterfactual: false, - reason, - cause: None, - }; - } - if let Err(reason) = media_removal_preflight(&MediaRemovalGuard { - room, - target: &target, - user_id, - dm_posture: ports.policy.dm_posture, - divergence_since: since, - participant_joined_at: joined_at, - repair_verdict: verdict, - server_health: state.server_health.health(location), - preflight: &preflight, - window, - }) { - return RemovalGate::Denied { - counterfactual: false, - reason, - cause: None, - }; - } - authorize( - state, - &AuthorizationRequest { - connection: key.clone(), - user_id, - kind: ActionKind::RemoveParticipant, - location: Some(location.clone()), - authorizing_turn: facts.authorizing_turn, - preflight_turn: record.turn(), - preflight_at: record.opened_at(), - now, - }, - ) - }); - - let authorization = match gate { - RemovalGate::Denied { - counterfactual: is_counterfactual, - reason, - cause, - } => { - if let Some(cause) = cause { - metrics.record_unknown_media_cause(cause); - } - return if is_counterfactual { - counterfactual(metrics, Lane::Destructive, reason) - } else { - refuse(metrics, Lane::Destructive, reason) - }; - } - RemovalGate::Authorized(authorized) => authorized.authorization, - }; - - if !hold_media_slot(ports.shared, location) { - ports.shared.with_state_mut(|state| { - state.governor.refund_identity(&key); - }); - return refuse(metrics, Lane::Destructive, AbortReason::Budget); - } - metrics.record_mutation( - ActionKind::RemoveParticipant, - room.scope(), - MutationOutcome::Issued, - ); - let started = ports.clock.now(); - let outcome = execute_media_removal(ports.livekit, authorization, location.clone()).await; - release_media_slot(ports.shared, location); - metrics.observe_mutation_duration(ports.clock.elapsed_since(started)); - - let settled = ports.clock.now(); - ports.shared.with_state_mut(|state| { - state.ledger.record_action_issued( - &key, - ActionKind::RemoveParticipant, - settled, - facts.authorizing_turn, - ); - }); - - metrics.record_mutation( - ActionKind::RemoveParticipant, - room.scope(), - remove_outcome(outcome), - ); - metrics.record_livekit_call("remove_participant", outcome.label()); - - if outcome.is_success() { - Actuation::Issued { - lane: Lane::Destructive, - call: "remove_participant", - outcome: outcome.label(), - } - } else { - Actuation::Failed { - lane: Lane::Destructive, - call: "remove_participant", - fault: outcome.label(), - } - } -} - -const fn remove_outcome(outcome: RemoveOutcome) -> MutationOutcome { - if outcome.is_success() { - MutationOutcome::Succeeded - } else { - MutationOutcome::Failed - } -} - -pub async fn execute_media_removal( - livekit: &L, - authorization: MutationAuthorization, - location: Location, -) -> RemoveOutcome -where - L: LiveKitApi, -{ - let room = authorization.connection.room; - let identity = ParticipantIdentity { - user_id: authorization.user_id, - connection_id: authorization.connection.connection.clone(), - }; - drop(authorization); - - livekit.remove_participant(&location, room, &identity).await -} - -fn authorize(state: &mut RuntimeState, request: &AuthorizationRequest) -> RemovalGate { - match state.governor.authorize(request) { - AuthorizationOutcome::Denied(reason) => RemovalGate::Denied { - counterfactual: matches!(reason, AbortReason::Mode), - reason, - cause: None, - }, - AuthorizationOutcome::Granted(authorization) => { - RemovalGate::Authorized(AuthorizedRemoval { authorization }) - } - } -} diff --git a/fluxer_recon/src/budget.rs b/fluxer_recon/src/budget.rs deleted file mode 100644 index dfe6d05a3..000000000 --- a/fluxer_recon/src/budget.rs +++ /dev/null @@ -1,1046 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; - -use crate::guards::AbortReason; -use crate::health::{ModeClamp, ModeClamps, ReconMode, WarmupGate, effective_mode}; -use crate::ids::{ConnectionKey, GuildId, Location, Millis, RoomKey, TurnId, UserId}; -use crate::ledger::ActionKind; - -pub const SLIDING_WINDOW_MS: u64 = 60_000; -pub const BREAKER_HYSTERESIS_MS: u64 = 60_000; - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct TokenBucket { - capacity: f64, - tokens: f64, - refill_per_ms: f64, - last_refill: Millis, -} - -impl TokenBucket { - pub fn new(capacity: f64, refill_per_ms: f64, now: Millis) -> Self { - Self { - capacity, - tokens: capacity, - refill_per_ms, - last_refill: now, - } - } - - pub fn per_second(rate: u32, now: Millis) -> Self { - Self::new(f64::from(rate), f64::from(rate) / 1_000.0, now) - } - - fn refill(&mut self, now: Millis) { - let elapsed = now.saturating_since(self.last_refill); - if elapsed == 0 { - return; - } - self.last_refill = now; - self.tokens = (self.tokens + (elapsed as f64) * self.refill_per_ms).min(self.capacity); - } - - pub fn tokens(&mut self, now: Millis) -> f64 { - self.refill(now); - self.tokens - } - - pub fn has_capacity(&mut self, now: Millis) -> bool { - self.tokens(now) >= 1.0 - } - - pub fn try_acquire(&mut self, now: Millis) -> bool { - self.try_acquire_many(1, now) - } - - pub fn try_acquire_many(&mut self, count: u32, now: Millis) -> bool { - self.refill(now); - let wanted = f64::from(count); - if self.tokens < wanted { - return false; - } - self.tokens -= wanted; - true - } - - pub fn refund(&mut self, count: u32) { - self.tokens = (self.tokens + f64::from(count)).min(self.capacity); - } - - pub const fn capacity(&self) -> f64 { - self.capacity - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct SlidingLimiter { - limit: u32, - window: SlidingMinute, -} - -impl SlidingLimiter { - pub fn new(limit: u32) -> Self { - Self { - limit, - window: SlidingMinute::default(), - } - } - - pub fn count(&self, now: Millis) -> u32 { - self.window.count(now) - } - - pub const fn limit(&self) -> u32 { - self.limit - } - - pub fn remaining(&self, now: Millis) -> u32 { - self.limit.saturating_sub(self.count(now)) - } - - pub fn has_capacity(&self, now: Millis) -> bool { - self.count(now) < self.limit - } - - pub fn try_acquire(&mut self, now: Millis) -> bool { - if !self.has_capacity(now) { - return false; - } - self.window.record(now); - true - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum BudgetClass { - GatewayRead, - GatewayMutate, - LiveKitRead, - LiveKitMutate, -} - -impl BudgetClass { - pub const ALL: [Self; 4] = [ - Self::GatewayRead, - Self::GatewayMutate, - Self::LiveKitRead, - Self::LiveKitMutate, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::GatewayRead => "gateway_read", - Self::GatewayMutate => "gateway_mutate", - Self::LiveKitRead => "livekit_read", - Self::LiveKitMutate => "livekit_mutate", - } - } - - pub const fn for_action(action: ActionKind) -> Self { - match action { - ActionKind::RemoveGatewayState => Self::GatewayMutate, - ActionKind::RemoveParticipant => Self::LiveKitMutate, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct InflightSemaphore { - limit: usize, - held: usize, -} - -impl InflightSemaphore { - pub const fn new(limit: usize) -> Self { - Self { limit, held: 0 } - } - - pub const fn try_acquire(&mut self) -> bool { - if self.held >= self.limit { - return false; - } - self.held += 1; - true - } - - pub const fn release(&mut self) { - self.held = self.held.saturating_sub(1); - } - - pub const fn held(&self) -> usize { - self.held - } - - pub const fn limit(&self) -> usize { - self.limit - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct SlidingMinute { - events: Vec, -} - -impl SlidingMinute { - fn count(&self, now: Millis) -> u32 { - let live = self - .events - .iter() - .filter(|at| now.saturating_since(**at) < SLIDING_WINDOW_MS) - .count(); - u32::try_from(live).unwrap_or(u32::MAX) - } - - fn record(&mut self, now: Millis) { - self.events - .retain(|at| now.saturating_since(*at) < SLIDING_WINDOW_MS); - self.events.push(now); - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum MutationLane { - Destructive, - Constructive, -} - -impl MutationLane { - pub const fn label(self) -> &'static str { - match self { - Self::Destructive => "destructive", - Self::Constructive => "constructive", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct IdentityTally { - destructive: u32, - constructive: u32, - last_at: Millis, -} - -impl Default for IdentityTally { - fn default() -> Self { - Self { - destructive: 0, - constructive: 0, - last_at: Millis::ZERO, - } - } -} - -pub const IDENTITY_TALLY_RETENTION_MS: u64 = 86_400_000; -pub const MAX_TRACKED_IDENTITIES: usize = 65_536; -const IDENTITY_TALLY_BYTES: u64 = 96; -const SLIDING_WINDOW_ENTRY_BYTES: u64 = 64; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct LaneWindows { - destructive: SlidingMinute, - constructive: SlidingMinute, -} - -impl LaneWindows { - const fn window(&self, lane: MutationLane) -> &SlidingMinute { - match lane { - MutationLane::Destructive => &self.destructive, - MutationLane::Constructive => &self.constructive, - } - } - - const fn window_mut(&mut self, lane: MutationLane) -> &mut SlidingMinute { - match lane { - MutationLane::Destructive => &mut self.destructive, - MutationLane::Constructive => &mut self.constructive, - } - } - - fn count(&self, lane: MutationLane, now: Millis) -> u32 { - self.window(lane).count(now) - } - - fn record(&mut self, lane: MutationLane, now: Millis) { - self.window_mut(lane).record(now); - } - - fn is_idle(&self, now: Millis) -> bool { - self.destructive.count(now) == 0 && self.constructive.count(now) == 0 - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct MutationLedger { - per_room: BTreeMap, - per_guild: BTreeMap, - global: LaneWindows, - per_identity: BTreeMap, - issued_total: u64, - pruned_identities_total: u64, -} - -impl MutationLedger { - pub fn new() -> Self { - Self::default() - } - - pub fn room_count(&self, room: RoomKey, now: Millis) -> u32 { - self.room_count_in(room, MutationLane::Destructive, now) - } - - pub fn room_count_in(&self, room: RoomKey, lane: MutationLane, now: Millis) -> u32 { - self.per_room - .get(&room) - .map_or(0, |windows| windows.count(lane, now)) - } - - pub fn guild_count(&self, guild: GuildId, now: Millis) -> u32 { - self.guild_count_in(guild, MutationLane::Destructive, now) - } - - pub fn guild_count_in(&self, guild: GuildId, lane: MutationLane, now: Millis) -> u32 { - self.per_guild - .get(&guild) - .map_or(0, |windows| windows.count(lane, now)) - } - - pub fn global_count(&self, now: Millis) -> u32 { - self.global_count_in(MutationLane::Destructive, now) - } - - pub fn global_count_in(&self, lane: MutationLane, now: Millis) -> u32 { - self.global.count(lane, now) - } - - pub fn identity_count(&self, key: &ConnectionKey) -> u32 { - self.per_identity - .get(key) - .map_or(0, |tally| tally.destructive) - } - - pub fn constructive_identity_count(&self, key: &ConnectionKey) -> u32 { - self.per_identity - .get(key) - .map_or(0, |tally| tally.constructive) - } - - pub fn tracked_identities(&self) -> usize { - self.per_identity.len() - } - - pub const fn pruned_identities_total(&self) -> u64 { - self.pruned_identities_total - } - - pub fn tracked_bytes(&self) -> u64 { - let identities = (self.per_identity.len() as u64).saturating_mul(IDENTITY_TALLY_BYTES); - let windows = (self.per_room.len() as u64) - .saturating_add(self.per_guild.len() as u64) - .saturating_add(1) - .saturating_mul(SLIDING_WINDOW_ENTRY_BYTES); - identities.saturating_add(windows) - } - - pub fn record(&mut self, key: &ConnectionKey, lane: MutationLane, now: Millis) { - self.per_room.entry(key.room).or_default().record(lane, now); - if let Some(guild) = key.room.guild_id() { - self.per_guild.entry(guild).or_default().record(lane, now); - } - self.global.record(lane, now); - let tally = self.per_identity.entry(key.clone()).or_default(); - match lane { - MutationLane::Destructive => { - tally.destructive = tally.destructive.saturating_add(1); - } - MutationLane::Constructive => { - tally.constructive = tally.constructive.saturating_add(1); - } - } - tally.last_at = now; - self.issued_total = self.issued_total.saturating_add(1); - } - - pub const fn issued_total(&self) -> u64 { - self.issued_total - } - - pub fn refund_identity(&mut self, key: &ConnectionKey) -> bool { - match self.per_identity.get_mut(key) { - None => false, - Some(tally) => { - tally.destructive = tally.destructive.saturating_sub(1); - if tally.destructive == 0 && tally.constructive == 0 { - self.per_identity.remove(key); - } - true - } - } - } - - pub fn forget_room(&mut self, room: &RoomKey) { - self.per_room.remove(room); - } - - pub fn prune(&mut self, now: Millis) -> usize { - self.per_room.retain(|_, windows| !windows.is_idle(now)); - self.per_guild.retain(|_, windows| !windows.is_idle(now)); - - let before = self.per_identity.len(); - self.per_identity - .retain(|_, tally| now.saturating_since(tally.last_at) < IDENTITY_TALLY_RETENTION_MS); - - while self.per_identity.len() > MAX_TRACKED_IDENTITIES { - let Some(oldest) = self - .per_identity - .iter() - .min_by_key(|(key, tally)| (tally.last_at, (*key).clone())) - .map(|(key, _)| key.clone()) - else { - break; - }; - self.per_identity.remove(&oldest); - } - - let pruned = before.saturating_sub(self.per_identity.len()); - self.pruned_identities_total = self.pruned_identities_total.saturating_add(pruned as u64); - pruned - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct BreakerLimits { - pub max_divergent_fraction: f64, - pub auto_reset_ms: u64, - pub max_auto_resets: u32, -} - -impl Default for BreakerLimits { - fn default() -> Self { - Self { - max_divergent_fraction: 0.10, - auto_reset_ms: 900_000, - max_auto_resets: 2, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BreakerEvent { - Steady, - Tripped, - AutoReset, - HeldForManualReset, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct CircuitBreaker { - tripped_since: Option, - below_half_since: Option, - auto_resets_used: u32, - trips_total: u64, -} - -impl CircuitBreaker { - pub const fn new() -> Self { - Self { - tripped_since: None, - below_half_since: None, - auto_resets_used: 0, - trips_total: 0, - } - } - - pub const fn is_tripped(&self) -> bool { - self.tripped_since.is_some() - } - - pub const fn trips_total(&self) -> u64 { - self.trips_total - } - - pub const fn auto_resets_used(&self) -> u32 { - self.auto_resets_used - } - - pub const fn manual_reset(&mut self) { - self.tripped_since = None; - self.below_half_since = None; - self.auto_resets_used = 0; - } - - pub const fn held_for_manual_reset(&self, limits: BreakerLimits) -> bool { - self.tripped_since.is_some() && self.auto_resets_used >= limits.max_auto_resets - } - - pub fn observe( - &mut self, - divergent_fraction: f64, - now: Millis, - limits: BreakerLimits, - ) -> BreakerEvent { - let over = divergent_fraction > limits.max_divergent_fraction; - let calm = divergent_fraction < limits.max_divergent_fraction / 2.0; - - if calm { - if self.below_half_since.is_none() { - self.below_half_since = Some(now); - } - } else { - self.below_half_since = None; - } - - let Some(tripped_since) = self.tripped_since else { - if over { - self.tripped_since = Some(now); - self.below_half_since = None; - self.trips_total = self.trips_total.saturating_add(1); - return BreakerEvent::Tripped; - } - return BreakerEvent::Steady; - }; - - if now.saturating_since(tripped_since) < limits.auto_reset_ms { - return BreakerEvent::Steady; - } - - let hysteresis_met = self - .below_half_since - .is_some_and(|since| now.saturating_since(since) >= BREAKER_HYSTERESIS_MS); - - if !hysteresis_met { - return BreakerEvent::Steady; - } - - if self.auto_resets_used >= limits.max_auto_resets { - return BreakerEvent::HeldForManualReset; - } - - self.auto_resets_used = self.auto_resets_used.saturating_add(1); - self.tripped_since = None; - self.below_half_since = None; - BreakerEvent::AutoReset - } -} - -pub struct EnforceToken(()); - -impl std::fmt::Debug for EnforceToken { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("EnforceToken") - } -} - -impl PartialEq for EnforceToken { - fn eq(&self, _other: &Self) -> bool { - true - } -} - -impl Eq for EnforceToken {} - -#[derive(Debug, PartialEq, Eq)] -pub struct MutationAuthorization { - pub connection: ConnectionKey, - pub user_id: UserId, - pub kind: ActionKind, - pub location: Option, - pub authorizing_turn: TurnId, - pub preflight_turn: TurnId, - pub preflight_at: Millis, - token: EnforceToken, -} - -impl MutationAuthorization { - pub const fn token(&self) -> &EnforceToken { - &self.token - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AuthorizationRequest { - pub connection: ConnectionKey, - pub user_id: UserId, - pub kind: ActionKind, - pub location: Option, - pub authorizing_turn: TurnId, - pub preflight_turn: TurnId, - pub preflight_at: Millis, - pub now: Millis, -} - -#[derive(Debug, PartialEq, Eq)] -pub enum AuthorizationOutcome { - Granted(MutationAuthorization), - Denied(AbortReason), -} - -impl AuthorizationOutcome { - pub fn granted(self) -> Option { - match self { - Self::Granted(authorization) => Some(authorization), - Self::Denied(_) => None, - } - } - - pub const fn denial(&self) -> Option { - match self { - Self::Granted(_) => None, - Self::Denied(reason) => Some(*reason), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct GovernorLimits { - pub gateway_read_rps: u32, - pub gateway_mutate_rpm: u32, - pub livekit_read_rps: u32, - pub livekit_read_rps_per_server: u32, - pub livekit_mutate_rpm: u32, - pub max_inflight_room_turns: usize, - pub max_inflight_gateway: usize, - pub max_inflight_livekit: usize, - pub max_inflight_per_server: usize, - pub max_mutations_per_room_per_min: u32, - pub max_mutations_per_guild_per_min: u32, - pub max_mutations_per_min: u32, - pub max_action_attempts: u32, - pub preflight_max_age_ms: u64, - pub breaker: BreakerLimits, -} - -impl Default for GovernorLimits { - fn default() -> Self { - Self { - gateway_read_rps: 40, - gateway_mutate_rpm: 30, - livekit_read_rps: 40, - livekit_read_rps_per_server: 40, - livekit_mutate_rpm: 30, - max_inflight_room_turns: 4, - max_inflight_gateway: 8, - max_inflight_livekit: 8, - max_inflight_per_server: 2, - max_mutations_per_room_per_min: 4, - max_mutations_per_guild_per_min: 8, - max_mutations_per_min: 30, - max_action_attempts: 3, - preflight_max_age_ms: 2_000, - breaker: BreakerLimits::default(), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct BudgetView { - pub warmup_complete: bool, - pub coverage_pass_complete: bool, - pub breaker_tripped: bool, - pub destructive_capacity: bool, - pub constructive_capacity: bool, -} - -impl BudgetView { - pub const fn unconstrained() -> Self { - Self { - warmup_complete: true, - coverage_pass_complete: true, - breaker_tripped: false, - destructive_capacity: true, - constructive_capacity: true, - } - } - - pub const fn destructive_block(&self) -> Option { - if !self.warmup_complete || !self.coverage_pass_complete { - return Some(AbortReason::Warmup); - } - if self.breaker_tripped { - return Some(AbortReason::Breaker); - } - if !self.destructive_capacity { - return Some(AbortReason::Budget); - } - None - } - - pub const fn constructive_block(&self) -> Option { - if self.constructive_capacity { - None - } else { - Some(AbortReason::Budget) - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Governor { - configured_mode: ReconMode, - runtime_override: Option, - clamps: ModeClamps, - warmup: WarmupGate, - limits: GovernorLimits, - buckets: BTreeMap, - mutate_windows: BTreeMap, - room_turns: InflightSemaphore, - gateway_inflight: InflightSemaphore, - livekit_inflight: InflightSemaphore, - per_server_inflight: BTreeMap, - per_server_read: BTreeMap, - mutations: MutationLedger, - breaker: CircuitBreaker, - denials: BTreeMap, -} - -impl Governor { - pub fn new( - configured_mode: ReconMode, - limits: GovernorLimits, - warmup: WarmupGate, - now: Millis, - ) -> Self { - let mut buckets = BTreeMap::new(); - buckets.insert( - BudgetClass::GatewayRead, - TokenBucket::per_second(limits.gateway_read_rps, now), - ); - buckets.insert( - BudgetClass::LiveKitRead, - TokenBucket::per_second(limits.livekit_read_rps, now), - ); - - let mut mutate_windows = BTreeMap::new(); - mutate_windows.insert( - BudgetClass::GatewayMutate, - SlidingLimiter::new(limits.gateway_mutate_rpm), - ); - mutate_windows.insert( - BudgetClass::LiveKitMutate, - SlidingLimiter::new(limits.livekit_mutate_rpm), - ); - - Self { - configured_mode, - runtime_override: None, - clamps: ModeClamps::new(), - warmup, - limits, - buckets, - mutate_windows, - room_turns: InflightSemaphore::new(limits.max_inflight_room_turns), - gateway_inflight: InflightSemaphore::new(limits.max_inflight_gateway), - livekit_inflight: InflightSemaphore::new(limits.max_inflight_livekit), - per_server_inflight: BTreeMap::new(), - per_server_read: BTreeMap::new(), - mutations: MutationLedger::new(), - breaker: CircuitBreaker::new(), - denials: BTreeMap::new(), - } - } - - pub const fn configured_mode(&self) -> ReconMode { - self.configured_mode - } - - pub const fn runtime_override(&self) -> Option { - self.runtime_override - } - - pub const fn set_runtime_override(&mut self, mode: Option) { - self.runtime_override = mode; - } - - pub const fn clamps(&self) -> &ModeClamps { - &self.clamps - } - - pub const fn clamps_mut(&mut self) -> &mut ModeClamps { - &mut self.clamps - } - - pub fn effective_mode(&self) -> ReconMode { - effective_mode(self.configured_mode, self.runtime_override, &self.clamps) - } - - pub const fn warmup(&self) -> &WarmupGate { - &self.warmup - } - - pub const fn warmup_mut(&mut self) -> &mut WarmupGate { - &mut self.warmup - } - - pub const fn breaker(&self) -> &CircuitBreaker { - &self.breaker - } - - pub fn reset_breaker(&mut self) -> bool { - let was_tripped = self.breaker.is_tripped(); - self.breaker.manual_reset(); - self.clamps - .set(ModeClamp::BreakerTripped, self.breaker.is_tripped()); - was_tripped - } - - pub const fn mutations(&self) -> &MutationLedger { - &self.mutations - } - - pub const fn mutations_mut(&mut self) -> &mut MutationLedger { - &mut self.mutations - } - - pub fn refund_identity(&mut self, key: &ConnectionKey) -> bool { - self.mutations.refund_identity(key) - } - - pub const fn limits(&self) -> GovernorLimits { - self.limits - } - - pub fn denials(&self) -> impl Iterator { - self.denials.iter() - } - - pub fn denial_count(&self, reason: AbortReason) -> u64 { - self.denials.get(&reason).copied().unwrap_or(0) - } - - pub const fn pending_actions(&self) -> usize { - 0 - } - - pub fn observe_divergence(&mut self, divergent_fraction: f64, now: Millis) -> BreakerEvent { - let event = self - .breaker - .observe(divergent_fraction, now, self.limits.breaker); - self.clamps - .set(ModeClamp::BreakerTripped, self.breaker.is_tripped()); - event - } - - pub fn try_acquire(&mut self, class: BudgetClass, now: Millis) -> bool { - if let Some(window) = self.mutate_windows.get_mut(&class) { - return window.try_acquire(now); - } - self.buckets - .get_mut(&class) - .is_some_and(|bucket| bucket.try_acquire(now)) - } - - pub fn try_acquire_many(&mut self, class: BudgetClass, count: u32, now: Millis) -> bool { - if self.mutate_windows.contains_key(&class) { - return (0..count).all(|_| self.try_acquire(class, now)); - } - self.buckets - .get_mut(&class) - .is_some_and(|bucket| bucket.try_acquire_many(count, now)) - } - - pub fn refund(&mut self, class: BudgetClass, count: u32) { - if let Some(bucket) = self.buckets.get_mut(&class) { - bucket.refund(count); - } - } - - pub fn has_capacity(&mut self, class: BudgetClass, now: Millis) -> bool { - if let Some(window) = self.mutate_windows.get(&class) { - return window.has_capacity(now); - } - self.buckets - .get_mut(&class) - .is_some_and(|bucket| bucket.has_capacity(now)) - } - - pub fn tokens(&mut self, class: BudgetClass, now: Millis) -> f64 { - if let Some(window) = self.mutate_windows.get(&class) { - return f64::from(window.remaining(now)); - } - self.buckets - .get_mut(&class) - .map_or(0.0, |bucket| bucket.tokens(now)) - } - - pub fn try_acquire_server_read( - &mut self, - location: &Location, - count: u32, - now: Millis, - ) -> bool { - let rate = self.limits.livekit_read_rps_per_server; - self.per_server_read - .entry(location.clone()) - .or_insert_with(|| TokenBucket::per_second(rate, now)) - .try_acquire_many(count, now) - } - - pub fn server_read_tokens(&mut self, location: &Location, now: Millis) -> f64 { - let rate = self.limits.livekit_read_rps_per_server; - self.per_server_read - .entry(location.clone()) - .or_insert_with(|| TokenBucket::per_second(rate, now)) - .tokens(now) - } - - pub fn refund_server_read(&mut self, location: &Location, count: u32) { - if let Some(bucket) = self.per_server_read.get_mut(location) { - bucket.refund(count); - } - } - - pub fn retain_servers(&mut self, keep: &[Location]) { - self.per_server_read - .retain(|location, bucket| keep.contains(location) || bucket.tokens < bucket.capacity); - self.per_server_inflight - .retain(|location, held| keep.contains(location) || held.held() > 0); - } - - pub fn prune_mutations(&mut self, now: Millis) -> usize { - self.mutations.prune(now) - } - - pub fn forget_room(&mut self, room: &RoomKey) { - self.mutations.forget_room(room); - } - - pub const fn room_turns(&mut self) -> &mut InflightSemaphore { - &mut self.room_turns - } - - pub const fn gateway_inflight(&mut self) -> &mut InflightSemaphore { - &mut self.gateway_inflight - } - - pub const fn livekit_inflight(&mut self) -> &mut InflightSemaphore { - &mut self.livekit_inflight - } - - pub fn server_inflight(&mut self, location: &Location) -> &mut InflightSemaphore { - self.per_server_inflight - .entry(location.clone()) - .or_insert_with(|| InflightSemaphore::new(self.limits.max_inflight_per_server)) - } - - pub fn view(&mut self, now: Millis) -> BudgetView { - let destructive_capacity = self.has_capacity(BudgetClass::GatewayMutate, now) - || self.has_capacity(BudgetClass::LiveKitMutate, now); - let constructive_capacity = self.has_capacity(BudgetClass::GatewayMutate, now); - - BudgetView { - warmup_complete: self.warmup.elapsed_complete(now), - coverage_pass_complete: self.warmup.coverage_pass_complete(), - breaker_tripped: self.breaker.is_tripped(), - destructive_capacity, - constructive_capacity, - } - } - - fn deny(&mut self, reason: AbortReason) -> AuthorizationOutcome { - *self.denials.entry(reason).or_insert(0) += 1; - AuthorizationOutcome::Denied(reason) - } - - pub fn authorize(&mut self, request: &AuthorizationRequest) -> AuthorizationOutcome { - let now = request.now; - - if !self.warmup.is_complete(now) { - return self.deny(AbortReason::Warmup); - } - - if self.breaker.is_tripped() { - return self.deny(AbortReason::Breaker); - } - - if !self.effective_mode().allows_destructive() { - return self.deny(AbortReason::Mode); - } - - if request.preflight_turn < request.authorizing_turn { - return self.deny(AbortReason::PreflightDisagreed); - } - - if now.saturating_since(request.preflight_at) > self.limits.preflight_max_age_ms { - return self.deny(AbortReason::PreflightStale); - } - - if self.mutations.identity_count(&request.connection) >= self.limits.max_action_attempts { - return self.deny(AbortReason::Budget); - } - - if self.mutations.room_count(request.connection.room, now) - >= self.limits.max_mutations_per_room_per_min - { - return self.deny(AbortReason::Budget); - } - - if let Some(guild) = request.connection.room.guild_id() - && self.mutations.guild_count(guild, now) >= self.limits.max_mutations_per_guild_per_min - { - return self.deny(AbortReason::Budget); - } - - if self.mutations.global_count(now) >= self.limits.max_mutations_per_min { - return self.deny(AbortReason::Budget); - } - - if !self.try_acquire(BudgetClass::for_action(request.kind), now) { - return self.deny(AbortReason::Budget); - } - - self.mutations - .record(&request.connection, MutationLane::Destructive, now); - - AuthorizationOutcome::Granted(MutationAuthorization { - connection: request.connection.clone(), - user_id: request.user_id, - kind: request.kind, - location: request.location.clone(), - authorizing_turn: request.authorizing_turn, - preflight_turn: request.preflight_turn, - preflight_at: request.preflight_at, - token: EnforceToken(()), - }) - } - - pub fn authorize_constructive( - &mut self, - connection: &ConnectionKey, - now: Millis, - ) -> Result<(), AbortReason> { - if !self.effective_mode().allows_constructive() { - self.deny(AbortReason::Mode); - return Err(AbortReason::Mode); - } - - let lane = MutationLane::Constructive; - - if self.mutations.room_count_in(connection.room, lane, now) - >= self.limits.max_mutations_per_room_per_min - { - self.deny(AbortReason::Budget); - return Err(AbortReason::Budget); - } - - if let Some(guild) = connection.room.guild_id() - && self.mutations.guild_count_in(guild, lane, now) - >= self.limits.max_mutations_per_guild_per_min - { - self.deny(AbortReason::Budget); - return Err(AbortReason::Budget); - } - - if self.mutations.global_count_in(lane, now) >= self.limits.max_mutations_per_min { - self.deny(AbortReason::Budget); - return Err(AbortReason::Budget); - } - - if !self.try_acquire(BudgetClass::GatewayMutate, now) { - self.deny(AbortReason::Budget); - return Err(AbortReason::Budget); - } - - self.mutations.record(connection, lane, now); - Ok(()) - } -} diff --git a/fluxer_recon/src/census.rs b/fluxer_recon/src/census.rs deleted file mode 100644 index 453630851..000000000 --- a/fluxer_recon/src/census.rs +++ /dev/null @@ -1,308 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; - -use crate::config::ReconConfig; -use crate::evidence::CensusCrossCheck; -use crate::gateway::{ActiveVoiceRooms, GatewayApi, GatewayFault, GatewayMethod}; -use crate::ids::{Epoch, Millis, RoomKey}; -use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; - -pub const CENSUS_METHOD: GatewayMethod = GatewayMethod::ActiveVoiceRooms; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum CensusRead { - Counted(ActiveVoiceRooms), - Unreadable(GatewayFault), -} - -impl CensusRead { - pub const fn is_readable(&self) -> bool { - matches!(self, Self::Counted(_)) - } - - pub const fn counted(&self) -> Option<&ActiveVoiceRooms> { - match self { - Self::Counted(rooms) => Some(rooms), - Self::Unreadable(_) => None, - } - } - - pub const fn fault(&self) -> Option<&GatewayFault> { - match self { - Self::Counted(_) => None, - Self::Unreadable(fault) => Some(fault), - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Counted(_) => "counted", - Self::Unreadable(_) => "unreadable", - } - } -} - -pub async fn read_census(gateway: &G) -> CensusRead -where - G: GatewayApi, -{ - match gateway.active_voice_rooms().await { - Ok(rooms) => CensusRead::Counted(rooms), - Err(fault) => CensusRead::Unreadable(fault), - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CensusSnapshot { - counts: BTreeMap, - node_count: u32, - unparseable_rooms: u32, - taken_at: Millis, - epoch: Epoch, -} - -impl CensusSnapshot { - pub fn rooms(&self) -> impl Iterator + '_ { - self.counts.iter().map(|(room, count)| (*room, *count)) - } - - pub fn room_keys(&self) -> Vec { - self.counts.keys().copied().collect() - } - - pub fn voice_state_count(&self, room: &RoomKey) -> Option { - self.counts.get(room).copied() - } - - pub const fn node_count(&self) -> u32 { - self.node_count - } - - pub const fn unparseable_rooms(&self) -> u32 { - self.unparseable_rooms - } - - pub const fn taken_at(&self) -> Millis { - self.taken_at - } - - pub const fn epoch(&self) -> Epoch { - self.epoch - } - - pub fn len(&self) -> usize { - self.counts.len() - } - - pub fn is_empty(&self) -> bool { - self.counts.is_empty() - } - - pub const fn age_ms(&self, now: Millis) -> u64 { - now.saturating_since(self.taken_at) - } - - pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { - self.age_ms(now) <= max_age_ms - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct CensusLimits { - pub max_age_ms: u64, - pub interval_ms: u64, -} - -impl CensusLimits { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - max_age_ms: config.census_max_age_ms, - interval_ms: config.census_interval_ms, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum CensusOutcome { - Applied { epoch: Epoch, rooms: usize }, - Unchanged { epoch: Epoch }, - Failed, -} - -impl CensusOutcome { - pub const fn label(self) -> &'static str { - match self { - Self::Applied { .. } => "applied", - Self::Unchanged { .. } => "unchanged", - Self::Failed => "failed", - } - } - - pub const fn kept_previous_snapshot(self) -> bool { - match self { - Self::Applied { .. } | Self::Unchanged { .. } => false, - Self::Failed => true, - } - } - - pub const fn epoch(self) -> Option { - match self { - Self::Applied { epoch, .. } | Self::Unchanged { epoch } => Some(epoch), - Self::Failed => None, - } - } -} - -#[derive(Clone, Debug)] -pub struct CensusCache { - limits: CensusLimits, - snapshot: Option, - epoch: Epoch, - last_failure_at: Option, - consecutive_failures: u32, - applied_total: u64, - unchanged_total: u64, - failed_total: u64, -} - -pub const CENSUS_ENTRY_BYTES: u64 = - (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; - -impl CensusCache { - pub fn tracked_bytes(&self) -> u64 { - self.snapshot.as_ref().map_or(0, |snapshot| { - (snapshot.len() as u64).saturating_mul(CENSUS_ENTRY_BYTES) - }) - } - - pub const fn new(limits: CensusLimits) -> Self { - Self { - limits, - snapshot: None, - epoch: Epoch::FIRST, - last_failure_at: None, - consecutive_failures: 0, - applied_total: 0, - unchanged_total: 0, - failed_total: 0, - } - } - - pub const fn limits(&self) -> CensusLimits { - self.limits - } - - pub const fn snapshot(&self) -> Option<&CensusSnapshot> { - self.snapshot.as_ref() - } - - pub const fn epoch(&self) -> Epoch { - self.epoch - } - - pub const fn last_failure_at(&self) -> Option { - self.last_failure_at - } - - pub const fn consecutive_failures(&self) -> u32 { - self.consecutive_failures - } - - pub const fn applied_total(&self) -> u64 { - self.applied_total - } - - pub const fn unchanged_total(&self) -> u64 { - self.unchanged_total - } - - pub const fn failed_total(&self) -> u64 { - self.failed_total - } - - pub fn age_ms(&self, now: Millis) -> Option { - self.snapshot.as_ref().map(|snapshot| snapshot.age_ms(now)) - } - - pub fn is_fresh(&self, now: Millis) -> bool { - self.snapshot - .as_ref() - .is_some_and(|snapshot| snapshot.is_fresh(now, self.limits.max_age_ms)) - } - - pub fn rooms(&self) -> Vec { - self.snapshot - .as_ref() - .map(CensusSnapshot::room_keys) - .unwrap_or_default() - } - - pub fn node_count(&self) -> Option { - self.snapshot.as_ref().map(CensusSnapshot::node_count) - } - - pub fn accept(&mut self, read: &CensusRead, now: Millis) -> CensusOutcome { - let counted = match read { - CensusRead::Unreadable(_) => { - self.failed_total = self.failed_total.saturating_add(1); - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - self.last_failure_at = Some(now); - return CensusOutcome::Failed; - } - CensusRead::Counted(counted) => counted, - }; - - self.consecutive_failures = 0; - let counts: BTreeMap = counted - .rooms - .iter() - .map(|entry| (entry.room, entry.voice_state_count)) - .collect(); - - let unchanged = self - .snapshot - .as_ref() - .is_some_and(|snapshot| snapshot.counts == counts); - if unchanged { - self.unchanged_total = self.unchanged_total.saturating_add(1); - if let Some(snapshot) = self.snapshot.as_mut() { - snapshot.taken_at = now; - snapshot.node_count = counted.node_count; - snapshot.unparseable_rooms = counted.unparseable_rooms; - } - return CensusOutcome::Unchanged { epoch: self.epoch }; - } - - self.epoch = self.epoch.next(); - self.applied_total = self.applied_total.saturating_add(1); - let rooms = counts.len(); - self.snapshot = Some(CensusSnapshot { - counts, - node_count: counted.node_count, - unparseable_rooms: counted.unparseable_rooms, - taken_at: now, - epoch: self.epoch, - }); - CensusOutcome::Applied { - epoch: self.epoch, - rooms, - } - } - - pub fn cross_check(&self, room: &RoomKey, now: Millis) -> CensusCrossCheck { - let Some(snapshot) = self.snapshot.as_ref() else { - return CensusCrossCheck::Missing; - }; - if !snapshot.is_fresh(now, self.limits.max_age_ms) { - return CensusCrossCheck::Stale; - } - CensusCrossCheck::Fresh { - voice_state_count: snapshot.voice_state_count(room).unwrap_or(0), - } - } - - pub fn forget(&mut self) { - self.snapshot = None; - } -} diff --git a/fluxer_recon/src/clock.rs b/fluxer_recon/src/clock.rs deleted file mode 100644 index 16fc9e1e4..000000000 --- a/fluxer_recon/src/clock.rs +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::ids::{Millis, WallMillis}; - -pub trait Clock: Send + Sync { - fn now(&self) -> Millis; - - fn wall_now(&self) -> WallMillis; - - fn elapsed_since(&self, earlier: Millis) -> u64 { - self.now().saturating_since(earlier) - } -} - -pub type SharedClock = Arc; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct WallAnchor { - monotonic: Millis, - wall: WallMillis, -} - -impl WallAnchor { - pub const fn new(monotonic: Millis, wall: WallMillis) -> Self { - Self { monotonic, wall } - } - - pub const fn projected(&self, monotonic: Millis) -> WallMillis { - self.wall - .saturating_add_millis(monotonic.saturating_since(self.monotonic)) - } - - pub const fn guard(&self, monotonic: Millis, observed: WallMillis) -> WallMillis { - let projected = self.projected(monotonic); - if observed.get() < projected.get() { - observed - } else { - projected - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct SystemClock; - -impl SystemClock { - pub const fn new() -> Self { - Self - } - - pub fn shared() -> SharedClock { - Arc::new(Self) - } -} - -impl Clock for SystemClock { - fn now(&self) -> Millis { - Millis::new(u64::try_from(fluxer_svc::metrics::now_ms()).unwrap_or(0)) - } - - fn wall_now(&self) -> WallMillis { - let since_epoch = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default(); - WallMillis::new(u64::try_from(since_epoch.as_millis()).unwrap_or(u64::MAX)) - } -} - -#[derive(Debug, Default)] -struct TestClockState { - monotonic: AtomicU64, - wall: AtomicU64, -} - -#[derive(Clone, Debug, Default)] -pub struct TestClock { - state: Arc, -} - -impl TestClock { - pub fn new() -> Self { - Self::default() - } - - pub fn starting_at(monotonic: Millis, wall: WallMillis) -> Self { - Self { - state: Arc::new(TestClockState { - monotonic: AtomicU64::new(monotonic.get()), - wall: AtomicU64::new(wall.get()), - }), - } - } - - pub fn shared(&self) -> SharedClock { - Arc::new(self.clone()) - } - - pub fn advance(&self, delta_ms: u64) -> Millis { - let previous = self.state.monotonic.fetch_add(delta_ms, Ordering::SeqCst); - Millis::new(previous.saturating_add(delta_ms)) - } - - pub fn advance_wall(&self, delta_ms: u64) -> WallMillis { - let previous = self.state.wall.fetch_add(delta_ms, Ordering::SeqCst); - WallMillis::new(previous.saturating_add(delta_ms)) - } - - pub fn set_wall(&self, wall: WallMillis) { - self.state.wall.store(wall.get(), Ordering::SeqCst); - } -} - -impl Clock for TestClock { - fn now(&self) -> Millis { - Millis::new(self.state.monotonic.load(Ordering::SeqCst)) - } - - fn wall_now(&self) -> WallMillis { - WallMillis::new(self.state.wall.load(Ordering::SeqCst)) - } -} diff --git a/fluxer_recon/src/config.rs b/fluxer_recon/src/config.rs deleted file mode 100644 index 1e56ce378..000000000 --- a/fluxer_recon/src/config.rs +++ /dev/null @@ -1,402 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::env; -use std::fmt::Display; -use std::str::FromStr; - -use crate::health::ReconMode; -use crate::topology::default_region_id; - -const MIN_COVERAGE_TARGET_MS: u64 = 1_000; -const MIN_REQUIRED_CORROBORATIONS: u32 = 2; -const READ_RPS_RANGE: (u32, u32) = (1, 200); -const MUTATE_RPM_RANGE: (u32, u32) = (0, 600); - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ConnectionIdGuard { - AssumeAbsent, - Honoured, -} - -impl ConnectionIdGuard { - pub fn parse(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "assume_absent" => Some(Self::AssumeAbsent), - "honored" | "honoured" => Some(Self::Honoured), - _ => None, - } - } - - pub const fn as_str(self) -> &'static str { - match self { - Self::AssumeAbsent => "assume_absent", - Self::Honoured => "honored", - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ReconConfig { - pub mode: ReconMode, - pub worker_threads: usize, - pub tick_ms: u64, - pub turns_per_tick: usize, - pub coverage_target_ms: u64, - pub coverage_period_hard_cap_ms: u64, - pub expected_rooms: usize, - pub hot_period_ms: u64, - pub max_hot_rooms: usize, - pub suspicion_hold_ms: u64, - pub required_corroborations: u32, - pub min_corroboration_gap_ms: u64, - pub max_corroboration_window_ms: u64, - pub min_divergence_ms: u64, - pub preflight_max_age_ms: u64, - pub warmup_seconds: u64, - pub pending_join_skew_ms: u64, - pub census_interval_ms: u64, - pub census_max_age_ms: u64, - pub discovery_interval_ms: u64, - pub topology_refresh_ms: u64, - pub topology_max_age_ms: u64, - pub topology_drain_grace_ms: u64, - pub room_cliff_hold_ms: u64, - pub server_cliff_hold_ms: u64, - pub server_cliff_fraction: f64, - pub gateway_read_rps: u32, - pub gateway_mutate_rpm: u32, - pub livekit_read_rps: u32, - pub livekit_read_rps_per_server: u32, - pub livekit_mutate_rpm: u32, - pub max_inflight_room_turns: usize, - pub max_inflight_gateway: usize, - pub max_inflight_livekit: usize, - pub max_inflight_per_server: usize, - pub max_mutations_per_room_per_min: u32, - pub max_mutations_per_guild_per_min: u32, - pub max_mutations_per_min: u32, - pub max_action_attempts: u32, - pub max_divergent_fraction: f64, - pub breaker_auto_reset_ms: u64, - pub max_breaker_auto_resets: u32, - pub max_tracked_rooms: usize, - pub max_tracked_connections: usize, - pub max_connections_per_room: usize, - pub memory_budget_bytes: u64, - pub dm_media_eviction: bool, - pub dm_gateway_eviction: bool, - pub gateway_connection_id_guard: ConnectionIdGuard, - pub webhook_enabled: bool, - pub webhook_port: u16, - pub livekit_internal_url: Option, - pub livekit_default_region_id: Option, -} - -impl ReconConfig { - pub fn from_env() -> anyhow::Result { - Self::from_env_reader(|name| env::var(name).ok()) - } - - pub fn from_env_reader(get: F) -> anyhow::Result - where - F: Fn(&str) -> Option, - { - let shard_count = number(&get, "FLUXER_SVC_SHARD_COUNT", 1u32)?; - let shard_id = number(&get, "FLUXER_SVC_SHARD_ID", 0u32)?; - if shard_count != 1 || shard_id != 0 { - anyhow::bail!( - "fluxer-recon runs as a single replica: FLUXER_SVC_SHARD_COUNT must be 1 and FLUXER_SVC_SHARD_ID must be 0, got {shard_count} and {shard_id}" - ); - } - - let config = Self { - mode: mode(&get, "FLUXER_RECON_MODE", ReconMode::Observing)?, - worker_threads: number(&get, "FLUXER_RECON_WORKER_THREADS", 2)?, - tick_ms: number(&get, "FLUXER_RECON_TICK_MS", 50)?, - turns_per_tick: number(&get, "FLUXER_RECON_TURNS_PER_TICK", 4)?, - coverage_target_ms: number(&get, "FLUXER_RECON_COVERAGE_TARGET_MS", 30_000)?, - coverage_period_hard_cap_ms: number( - &get, - "FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS", - 120_000, - )?, - expected_rooms: number(&get, "FLUXER_RECON_EXPECTED_ROOMS", 800)?, - hot_period_ms: number(&get, "FLUXER_RECON_HOT_PERIOD_MS", 1_000)?, - max_hot_rooms: number(&get, "FLUXER_RECON_MAX_HOT_ROOMS", 64)?, - suspicion_hold_ms: number(&get, "FLUXER_RECON_SUSPICION_HOLD_MS", 30_000)?, - required_corroborations: number(&get, "FLUXER_RECON_REQUIRED_CORROBORATIONS", 3)?, - min_corroboration_gap_ms: number(&get, "FLUXER_RECON_MIN_CORROBORATION_GAP_MS", 1_000)?, - max_corroboration_window_ms: number( - &get, - "FLUXER_RECON_MAX_CORROBORATION_WINDOW_MS", - 60_000, - )?, - min_divergence_ms: number(&get, "FLUXER_RECON_MIN_DIVERGENCE_MS", 15_000)?, - preflight_max_age_ms: number(&get, "FLUXER_RECON_PREFLIGHT_MAX_AGE_MS", 1_000)?, - warmup_seconds: number(&get, "FLUXER_RECON_WARMUP_SECONDS", 120)?, - pending_join_skew_ms: number(&get, "FLUXER_RECON_PENDING_JOIN_SKEW_MS", 5_000)?, - census_interval_ms: number(&get, "FLUXER_RECON_CENSUS_INTERVAL_MS", 60_000)?, - census_max_age_ms: number(&get, "FLUXER_RECON_CENSUS_MAX_AGE_MS", 300_000)?, - discovery_interval_ms: number(&get, "FLUXER_RECON_DISCOVERY_INTERVAL_MS", 15_000)?, - topology_refresh_ms: number(&get, "FLUXER_RECON_TOPOLOGY_REFRESH_MS", 60_000)?, - topology_max_age_ms: number(&get, "FLUXER_RECON_TOPOLOGY_MAX_AGE_MS", 600_000)?, - topology_drain_grace_ms: number(&get, "FLUXER_RECON_TOPOLOGY_DRAIN_GRACE_MS", 600_000)?, - room_cliff_hold_ms: number(&get, "FLUXER_RECON_ROOM_CLIFF_HOLD_MS", 60_000)?, - server_cliff_hold_ms: number(&get, "FLUXER_RECON_SERVER_CLIFF_HOLD_MS", 300_000)?, - server_cliff_fraction: number(&get, "FLUXER_RECON_SERVER_CLIFF_FRACTION", 0.5)?, - gateway_read_rps: clamped( - number(&get, "FLUXER_RECON_GATEWAY_READ_RPS", 40)?, - READ_RPS_RANGE, - ), - gateway_mutate_rpm: clamped( - number(&get, "FLUXER_RECON_GATEWAY_MUTATE_RPM", 30)?, - MUTATE_RPM_RANGE, - ), - livekit_read_rps: clamped( - number(&get, "FLUXER_RECON_LIVEKIT_READ_RPS", 40)?, - READ_RPS_RANGE, - ), - livekit_read_rps_per_server: clamped( - number(&get, "FLUXER_RECON_LIVEKIT_READ_RPS_PER_SERVER", 40)?, - READ_RPS_RANGE, - ), - livekit_mutate_rpm: clamped( - number(&get, "FLUXER_RECON_LIVEKIT_MUTATE_RPM", 30)?, - MUTATE_RPM_RANGE, - ), - max_inflight_room_turns: number(&get, "FLUXER_RECON_MAX_INFLIGHT_ROOM_TURNS", 4)?, - max_inflight_gateway: number(&get, "FLUXER_RECON_MAX_INFLIGHT_GATEWAY", 8)?, - max_inflight_livekit: number(&get, "FLUXER_RECON_MAX_INFLIGHT_LIVEKIT", 8)?, - max_inflight_per_server: number(&get, "FLUXER_RECON_MAX_INFLIGHT_PER_SERVER", 2)?, - max_mutations_per_room_per_min: number( - &get, - "FLUXER_RECON_MAX_MUTATIONS_PER_ROOM_PER_MIN", - 4, - )?, - max_mutations_per_guild_per_min: number( - &get, - "FLUXER_RECON_MAX_MUTATIONS_PER_GUILD_PER_MIN", - 8, - )?, - max_mutations_per_min: number(&get, "FLUXER_RECON_MAX_MUTATIONS_PER_MIN", 30)?, - max_action_attempts: number(&get, "FLUXER_RECON_MAX_ACTION_ATTEMPTS", 3)?, - max_divergent_fraction: number(&get, "FLUXER_RECON_MAX_DIVERGENT_FRACTION", 0.10)?, - breaker_auto_reset_ms: number(&get, "FLUXER_RECON_BREAKER_AUTO_RESET_MS", 900_000)?, - max_breaker_auto_resets: number(&get, "FLUXER_RECON_MAX_BREAKER_AUTO_RESETS", 2)?, - max_tracked_rooms: number(&get, "FLUXER_RECON_MAX_TRACKED_ROOMS", 8_192)?, - max_tracked_connections: number(&get, "FLUXER_RECON_MAX_TRACKED_CONNECTIONS", 65_536)?, - max_connections_per_room: number(&get, "FLUXER_RECON_MAX_CONNECTIONS_PER_ROOM", 512)?, - memory_budget_bytes: number(&get, "FLUXER_RECON_MEMORY_BUDGET_BYTES", 16_777_216)?, - dm_media_eviction: boolean(&get, "FLUXER_RECON_DM_MEDIA_EVICTION", false)?, - dm_gateway_eviction: boolean(&get, "FLUXER_RECON_DM_GATEWAY_EVICTION", true)?, - gateway_connection_id_guard: connection_id_guard( - &get, - "FLUXER_RECON_GATEWAY_CONNECTION_ID_GUARD", - ConnectionIdGuard::AssumeAbsent, - )?, - webhook_enabled: boolean(&get, "FLUXER_RECON_WEBHOOK_ENABLED", false)?, - webhook_port: number(&get, "FLUXER_RECON_WEBHOOK_PORT", 8_092)?, - livekit_internal_url: text(&get, "FLUXER_LIVEKIT_INTERNAL_URL"), - livekit_default_region_id: text(&get, "FLUXER_LIVEKIT_DEFAULT_REGION") - .as_deref() - .and_then(default_region_id), - }; - - validate(&config)?; - Ok(config) - } -} - -fn validate(config: &ReconConfig) -> anyhow::Result<()> { - positive_usize(config.worker_threads, "FLUXER_RECON_WORKER_THREADS")?; - positive_usize(config.turns_per_tick, "FLUXER_RECON_TURNS_PER_TICK")?; - positive_usize( - config.max_inflight_room_turns, - "FLUXER_RECON_MAX_INFLIGHT_ROOM_TURNS", - )?; - positive_usize( - config.max_inflight_gateway, - "FLUXER_RECON_MAX_INFLIGHT_GATEWAY", - )?; - positive_usize( - config.max_inflight_livekit, - "FLUXER_RECON_MAX_INFLIGHT_LIVEKIT", - )?; - positive_usize( - config.max_inflight_per_server, - "FLUXER_RECON_MAX_INFLIGHT_PER_SERVER", - )?; - positive_usize(config.expected_rooms, "FLUXER_RECON_EXPECTED_ROOMS")?; - positive_usize(config.max_tracked_rooms, "FLUXER_RECON_MAX_TRACKED_ROOMS")?; - positive_usize( - config.max_tracked_connections, - "FLUXER_RECON_MAX_TRACKED_CONNECTIONS", - )?; - positive_usize( - config.max_connections_per_room, - "FLUXER_RECON_MAX_CONNECTIONS_PER_ROOM", - )?; - positive_millis(config.tick_ms, "FLUXER_RECON_TICK_MS")?; - positive_millis(config.hot_period_ms, "FLUXER_RECON_HOT_PERIOD_MS")?; - positive_millis(config.census_interval_ms, "FLUXER_RECON_CENSUS_INTERVAL_MS")?; - positive_millis( - config.discovery_interval_ms, - "FLUXER_RECON_DISCOVERY_INTERVAL_MS", - )?; - positive_millis( - config.topology_refresh_ms, - "FLUXER_RECON_TOPOLOGY_REFRESH_MS", - )?; - positive_millis( - config.preflight_max_age_ms, - "FLUXER_RECON_PREFLIGHT_MAX_AGE_MS", - )?; - positive_millis( - config.min_corroboration_gap_ms, - "FLUXER_RECON_MIN_CORROBORATION_GAP_MS", - )?; - fraction( - config.server_cliff_fraction, - "FLUXER_RECON_SERVER_CLIFF_FRACTION", - )?; - fraction( - config.max_divergent_fraction, - "FLUXER_RECON_MAX_DIVERGENT_FRACTION", - )?; - - if config.required_corroborations < MIN_REQUIRED_CORROBORATIONS { - anyhow::bail!( - "FLUXER_RECON_REQUIRED_CORROBORATIONS must be at least {MIN_REQUIRED_CORROBORATIONS}, got {}", - config.required_corroborations - ); - } - - let spread = config - .min_corroboration_gap_ms - .saturating_mul(u64::from(config.required_corroborations.saturating_sub(1))); - if config.max_corroboration_window_ms <= spread { - anyhow::bail!( - "FLUXER_RECON_MAX_CORROBORATION_WINDOW_MS ({}) must exceed the gap times one fewer than the required corroborations ({spread})", - config.max_corroboration_window_ms - ); - } - - if config.coverage_target_ms < MIN_COVERAGE_TARGET_MS { - anyhow::bail!( - "FLUXER_RECON_COVERAGE_TARGET_MS must be at least {MIN_COVERAGE_TARGET_MS}, got {}", - config.coverage_target_ms - ); - } - - if config.coverage_target_ms > config.coverage_period_hard_cap_ms { - anyhow::bail!( - "FLUXER_RECON_COVERAGE_TARGET_MS ({}) must not exceed FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({})", - config.coverage_target_ms, - config.coverage_period_hard_cap_ms - ); - } - - Ok(()) -} - -fn text(get: &F, name: &str) -> Option -where - F: Fn(&str) -> Option, -{ - get(name) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) -} - -fn positive_usize(value: usize, name: &str) -> anyhow::Result<()> { - if value == 0 { - anyhow::bail!("{name} must be greater than zero"); - } - Ok(()) -} - -fn positive_millis(value: u64, name: &str) -> anyhow::Result<()> { - if value == 0 { - anyhow::bail!("{name} must be greater than zero"); - } - Ok(()) -} - -fn fraction(value: f64, name: &str) -> anyhow::Result<()> { - if !(value > 0.0 && value <= 1.0) { - anyhow::bail!("{name} must lie in (0, 1], got {value}"); - } - Ok(()) -} - -fn clamped(value: u32, range: (u32, u32)) -> u32 { - let (low, high) = range; - value.clamp(low, high) -} - -fn read(get: &F, name: &str) -> Option -where - F: Fn(&str) -> Option, -{ - get(name).filter(|value| !value.is_empty()) -} - -fn number(get: &F, name: &str, default: T) -> anyhow::Result -where - F: Fn(&str) -> Option, - T: FromStr, - T::Err: Display, -{ - match read(get, name) { - None => Ok(default), - Some(raw) => raw - .trim() - .parse::() - .map_err(|error| anyhow::anyhow!("invalid {name}: {raw} ({error})")), - } -} - -fn boolean(get: &F, name: &str, default: bool) -> anyhow::Result -where - F: Fn(&str) -> Option, -{ - match read(get, name) { - None => Ok(default), - Some(raw) => parse_bool(&raw) - .ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw} is not a boolean")), - } -} - -fn mode(get: &F, name: &str, default: ReconMode) -> anyhow::Result -where - F: Fn(&str) -> Option, -{ - match read(get, name) { - None => Ok(default), - Some(raw) => ReconMode::parse(&raw).ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw}")), - } -} - -fn connection_id_guard( - get: &F, - name: &str, - default: ConnectionIdGuard, -) -> anyhow::Result -where - F: Fn(&str) -> Option, -{ - match read(get, name) { - None => Ok(default), - Some(raw) => { - ConnectionIdGuard::parse(&raw).ok_or_else(|| anyhow::anyhow!("invalid {name}: {raw}")) - } - } -} - -fn parse_bool(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "1" | "true" | "yes" | "y" | "on" => Some(true), - "0" | "false" | "no" | "n" | "off" => Some(false), - _ => None, - } -} diff --git a/fluxer_recon/src/control.rs b/fluxer_recon/src/control.rs deleted file mode 100644 index b35aa182c..000000000 --- a/fluxer_recon/src/control.rs +++ /dev/null @@ -1,784 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use serde::{Deserialize, Serialize}; - -use fluxer_svc::transport::{Transport, TransportMessage, TransportSubscriber, reply_message}; - -use crate::budget::BudgetClass; -use crate::health::{ModeClamp, ReconMode}; -use crate::ids::{ChannelId, GuildId, Millis, RoomKey}; -use crate::runtime::{Shared, monotonic_now}; -use crate::singleton::{RESUBSCRIBE_BACKOFF_MS, next_backoff_ms}; -use crate::suspicion::SuspicionSource; - -pub const CONTROL_SUBJECT: &str = "svc.recon"; -pub const INSTANCE_SUBJECT: &str = "svc.recon.instance"; -pub const EXPLAIN_DECISION_LIMIT: usize = 16; - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum RawId { - Number(u64), - Text(String), -} - -impl RawId { - pub fn value(&self) -> Option { - match self { - Self::Number(value) => Some(*value), - Self::Text(text) => parse_u64_strict(text), - } - } -} - -fn parse_u64_strict(value: &str) -> Option { - if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { - return None; - } - value.parse::().ok() -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(tag = "method", rename_all = "snake_case")] -pub enum ControlRequest { - Status, - Explain { - #[serde(default)] - guild_id: Option, - channel_id: RawId, - }, - Mode { - #[serde(default)] - mode: Option, - }, - Budget, - Topology { - #[serde(default)] - refresh: Option, - }, - Clamps { - #[serde(default)] - release: Option, - }, - Suspect { - #[serde(default)] - guild_id: Option, - channel_id: RawId, - }, - Cliffs { - #[serde(default)] - release: Option, - }, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ControlError { - DecodeFailed, - UnknownMode, - UnknownClamp, - InvalidChannelId, - InvalidGuildId, -} - -impl ControlError { - pub const fn label(self) -> &'static str { - match self { - Self::DecodeFailed => "decode_failed", - Self::UnknownMode => "unknown_mode", - Self::UnknownClamp => "unknown_clamp", - Self::InvalidChannelId => "invalid_channel_id", - Self::InvalidGuildId => "invalid_guild_id", - } - } -} - -#[derive(Clone, Debug, Serialize)] -pub struct ErrorResponse { - pub ok: bool, - pub error: &'static str, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ModeResponse { - pub ok: bool, - pub configured: &'static str, - pub runtime_override: Option<&'static str>, - pub effective: &'static str, - pub clamps: Vec<&'static str>, - pub changed: bool, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ServerHealthEntry { - pub region: String, - pub server: String, - pub health: &'static str, - pub consecutive_failures: u32, -} - -#[derive(Clone, Debug, Serialize)] -pub struct PeerEntry { - pub instance_id: String, - pub started_at: u64, - pub first_seen_ms_ago: u64, - pub last_seen_ms_ago: u64, - pub detections: u64, -} - -#[derive(Clone, Debug, Serialize)] -pub struct StatusResponse { - pub ok: bool, - pub instance_id: String, - pub started_at: u64, - pub uptime_ms: u64, - pub ready: bool, - pub draining: bool, - pub configured_mode: &'static str, - pub runtime_override: Option<&'static str>, - pub effective_mode: &'static str, - pub clamps: Vec<&'static str>, - pub warmup_elapsed: bool, - pub coverage_pass_complete: bool, - pub tracked_rooms: usize, - pub tracked_connections: usize, - pub tracked_bytes: u64, - pub divergent_connections: usize, - pub divergent_fraction: f64, - pub coverage_period_ms: u64, - pub breaker_tripped: bool, - pub breaker_trips_total: u64, - pub breaker_auto_resets_used: u32, - pub breaker_held_for_manual_reset: bool, - pub mutations_paused_ms_remaining: u64, - pub topology_age_ms: Option, - pub topology_servers: usize, - pub census_age_ms: Option, - pub last_turn: u64, - pub servers: Vec, - pub peer: Option, -} - -#[derive(Clone, Debug, Serialize)] -pub struct BucketEntry { - pub class: &'static str, - pub tokens: f64, -} - -#[derive(Clone, Debug, Serialize)] -pub struct DenialEntry { - pub reason: &'static str, - pub count: u64, -} - -#[derive(Clone, Debug, Serialize)] -pub struct BudgetResponse { - pub ok: bool, - pub effective_mode: &'static str, - pub buckets: Vec, - pub mutations_issued_total: u64, - pub mutations_last_minute: u32, - pub max_mutations_per_min: u32, - pub max_action_attempts: u32, - pub pending_actions: usize, - pub scheduler_starved_total: u64, - pub ledger_evicted_total: u64, - pub denials: Vec, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ConnectionDump { - pub connection_id: String, - pub user_id: String, - pub state: &'static str, - pub corroborations: u32, - pub divergence_since_ms: Option, - pub fingerprint: Option, - pub last_seen_ms_ago: u64, - pub action_attempts: u8, - pub repair_attempts: u8, - pub last_repair_verdict: Option<&'static str>, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ExplainResponse { - pub ok: bool, - pub scope: &'static str, - pub guild_id: Option, - pub channel_id: String, - pub tracked: bool, - pub last_turn: Option, - pub last_seen_ms_ago: Option, - pub partially_unreadable: bool, - pub no_candidates: bool, - pub gateway_cliff_since_ms_ago: Option, - pub media_cliff_since_ms_ago: Option, - pub last_known_locations: Vec, - pub connections: Vec, - pub decisions: Vec, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ClampEntry { - pub clamp: &'static str, - pub engaged: bool, - pub self_clearing: bool, - pub expires_in_ms: Option, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ClampsResponse { - pub ok: bool, - pub configured_mode: &'static str, - pub effective_mode: &'static str, - pub ceiling: &'static str, - pub clamps: Vec, - pub released: Vec<&'static str>, - pub breaker_tripped: bool, - pub breaker_auto_resets_used: u32, - pub breaker_held_for_manual_reset: bool, - pub mutations_paused_windows: u32, - pub mutations_paused_hold_ms: u64, -} - -#[derive(Clone, Debug, Serialize)] -pub struct CliffsResponse { - pub ok: bool, - pub cliffed: Vec, - pub disputed: Vec, - pub holed: Vec, - pub released: Vec, - pub hold_ms: u64, - pub tripped_total: u64, - pub disputed_total: u64, - pub holes_total: u64, -} - -#[derive(Clone, Debug, Serialize)] -pub struct SuspectResponse { - pub ok: bool, - pub scope: &'static str, - pub guild_id: Option, - pub channel_id: String, - pub admitted: &'static str, - pub raised: &'static str, - pub hold_ms: u64, - pub suspect_rooms: usize, -} - -#[derive(Clone, Debug, Serialize)] -pub struct TopologyResponse { - pub ok: bool, - pub servers: usize, - pub age_ms: Option, - pub max_age_ms: u64, - pub stale: bool, - pub refresh_requested: bool, -} - -fn clamp_labels(shared: &Shared) -> Vec<&'static str> { - shared.with_state(|state| { - state - .governor - .clamps() - .engaged() - .map(ModeClamp::label) - .collect() - }) -} - -fn error(code: ControlError) -> serde_json::Value { - serde_json::json!(ErrorResponse { - ok: false, - error: code.label(), - }) -} - -fn room_key(guild_id: Option<&RawId>, channel_id: &RawId) -> Result { - let channel = channel_id.value().ok_or(ControlError::InvalidChannelId)?; - match guild_id { - None => Ok(RoomKey::Dm { - channel_id: ChannelId::new(channel), - }), - Some(raw) => { - let guild = raw.value().ok_or(ControlError::InvalidGuildId)?; - Ok(RoomKey::Guild { - guild_id: GuildId::new(guild), - channel_id: ChannelId::new(channel), - }) - } - } -} - -pub fn status(shared: &Shared, now: Millis) -> StatusResponse { - shared.with_state(|state| StatusResponse { - ok: true, - instance_id: shared.instance().id.clone(), - started_at: shared.instance().started_at, - uptime_ms: now.saturating_since(shared.boot_at()), - ready: shared.readiness().is_ready(), - draining: shared.readiness().is_draining(), - configured_mode: state.governor.configured_mode().as_str(), - runtime_override: state.governor.runtime_override().map(ReconMode::as_str), - effective_mode: state.governor.effective_mode().as_str(), - clamps: state - .governor - .clamps() - .engaged() - .map(ModeClamp::label) - .collect(), - warmup_elapsed: state.governor.warmup().elapsed_complete(now), - coverage_pass_complete: state.governor.warmup().coverage_pass_complete(), - tracked_rooms: state.ledger.tracked_rooms(), - tracked_connections: state.ledger.tracked_connections(), - tracked_bytes: state.ledger.tracked_bytes(), - divergent_connections: state.ledger.divergent_connections(), - divergent_fraction: state.ledger.divergent_fraction(), - coverage_period_ms: state.coverage_period_ms, - breaker_tripped: state.governor.breaker().is_tripped(), - breaker_trips_total: state.governor.breaker().trips_total(), - breaker_auto_resets_used: state.governor.breaker().auto_resets_used(), - breaker_held_for_manual_reset: state - .governor - .breaker() - .held_for_manual_reset(state.governor.limits().breaker), - mutations_paused_ms_remaining: state.mutations_pause.remaining_ms(now), - topology_age_ms: state.topology_age_ms(now), - topology_servers: state.topology_servers, - census_age_ms: state.census_age_ms(now), - last_turn: state.turn.get(), - servers: state - .server_health - .tracked() - .map(|(location, tracker)| ServerHealthEntry { - region: location.region.to_string(), - server: location.server.to_string(), - health: tracker.state().label(), - consecutive_failures: tracker.consecutive_failures(), - }) - .collect(), - peer: state.peer.as_ref().map(|peer| PeerEntry { - instance_id: peer.instance_id.clone(), - started_at: peer.started_at, - first_seen_ms_ago: now.saturating_since(peer.first_seen_at), - last_seen_ms_ago: now.saturating_since(peer.last_seen_at), - detections: peer.detections, - }), - }) -} - -pub fn budget(shared: &Shared, now: Millis) -> BudgetResponse { - let metrics = shared.metrics().clone(); - shared.with_state_mut(|state| { - let buckets = [ - BudgetClass::GatewayRead, - BudgetClass::GatewayMutate, - BudgetClass::LiveKitRead, - BudgetClass::LiveKitMutate, - ] - .into_iter() - .map(|class| BucketEntry { - class: class.label(), - tokens: state.governor.tokens(class, now), - }) - .collect(); - - BudgetResponse { - ok: true, - effective_mode: state.governor.effective_mode().as_str(), - buckets, - mutations_issued_total: state.governor.mutations().issued_total(), - mutations_last_minute: state.governor.mutations().global_count(now), - max_mutations_per_min: state.governor.limits().max_mutations_per_min, - max_action_attempts: state.governor.limits().max_action_attempts, - pending_actions: state.governor.pending_actions(), - scheduler_starved_total: metrics.scheduler_starved_total(), - ledger_evicted_total: metrics.ledger_evicted_total(), - denials: state - .governor - .denials() - .map(|(reason, count)| DenialEntry { - reason: reason.label(), - count: *count, - }) - .collect(), - } - }) -} - -pub fn explain(shared: &Shared, room: RoomKey, now: Millis) -> ExplainResponse { - let decisions = shared.journal().for_room(room, EXPLAIN_DECISION_LIMIT); - shared.with_state(|state| { - let entry = state.ledger.room(&room); - ExplainResponse { - ok: true, - scope: crate::metrics::scope_label(room.scope()), - guild_id: room.guild_id().map(|guild| guild.to_string()), - channel_id: room.channel_id().to_string(), - tracked: entry.is_some(), - last_turn: entry.map(|room| room.last_turn().get()), - last_seen_ms_ago: entry.map(|room| now.saturating_since(room.last_seen())), - partially_unreadable: entry.is_some_and(|room| room.partially_unreadable()), - no_candidates: entry.is_some_and(|room| room.no_candidates()), - gateway_cliff_since_ms_ago: entry - .and_then(|room| room.gateway_cliff().since()) - .map(|since| now.saturating_since(since)), - media_cliff_since_ms_ago: entry - .and_then(|room| room.media_cliff().since()) - .map(|since| now.saturating_since(since)), - last_known_locations: entry - .map(|room| { - room.last_known_locations() - .iter() - .filter_map(|index| state.ledger.location(*index)) - .map(ToString::to_string) - .collect() - }) - .unwrap_or_default(), - connections: entry - .map(|room| { - room.connections() - .iter() - .map(|connection| ConnectionDump { - connection_id: connection.connection.to_string(), - user_id: connection.user_id.to_string(), - state: connection.state.label(), - corroborations: connection.state.corroborations(), - divergence_since_ms: connection - .state - .divergence_since() - .map(|since| now.saturating_since(since)), - fingerprint: connection - .state - .stored_fingerprint() - .map(|fingerprint| format!("{:016x}", fingerprint.get())), - last_seen_ms_ago: now.saturating_since(connection.last_seen), - action_attempts: connection.action_attempts, - repair_attempts: connection.repair_attempts, - last_repair_verdict: connection - .last_repair_verdict - .map(|verdict| verdict.label()), - }) - .collect() - }) - .unwrap_or_default(), - decisions, - } - }) -} - -pub fn set_mode(shared: &Shared, requested: Option<&str>) -> Result { - let parsed = match requested { - None => None, - Some(raw) => Some(ReconMode::parse(raw).ok_or(ControlError::UnknownMode)?), - }; - - let transition = shared.set_mode_override(parsed); - - tracing::warn!( - configured = transition.configured.as_str(), - previous_override = transition.previous_override.map(ReconMode::as_str), - requested = transition.requested.map(ReconMode::as_str), - effective_before = transition.effective_before.as_str(), - effective_after = transition.effective_after.as_str(), - "recon mode override applied over the control plane" - ); - - Ok(ModeResponse { - ok: true, - configured: transition.configured.as_str(), - runtime_override: transition.requested.map(ReconMode::as_str), - effective: transition.effective_after.as_str(), - clamps: clamp_labels(shared), - changed: transition.changed(), - }) -} - -pub fn read_mode(shared: &Shared) -> ModeResponse { - shared.with_state(|state| ModeResponse { - ok: true, - configured: state.governor.configured_mode().as_str(), - runtime_override: state.governor.runtime_override().map(ReconMode::as_str), - effective: state.governor.effective_mode().as_str(), - clamps: state - .governor - .clamps() - .engaged() - .map(ModeClamp::label) - .collect(), - changed: false, - }) -} - -pub fn topology(shared: &Shared, refresh: bool, now: Millis) -> TopologyResponse { - let max_age_ms = shared.config().topology_max_age_ms; - shared.with_state_mut(|state| { - if refresh { - state.topology_refresh_requested = true; - } - let age_ms = state.topology_age_ms(now); - TopologyResponse { - ok: true, - servers: state.topology_servers, - age_ms, - max_age_ms, - stale: age_ms.is_none_or(|age| age > max_age_ms), - refresh_requested: state.topology_refresh_requested, - } - }) -} - -fn release_clamp(shared: &Shared, clamp: ModeClamp) -> bool { - shared.with_state_mut(|state| { - let was_engaged = state.governor.clamps().is_engaged(clamp); - match clamp { - ModeClamp::BreakerTripped => { - state.governor.reset_breaker(); - } - ModeClamp::MutationsPaused => state.mutations_pause.clear(), - ModeClamp::Warmup => state.governor.warmup_mut().force_complete(), - ModeClamp::ColdStart => state.governor.warmup_mut().note_coverage_pass(), - ModeClamp::SingletonConflict => state.peer = None, - ModeClamp::TopologyStale | ModeClamp::NatsReconnect | ModeClamp::CoverageOverrun => {} - } - state.governor.clamps_mut().release(clamp); - was_engaged - }) -} - -pub fn clamps( - shared: &Shared, - release: Option<&str>, - now: Millis, -) -> Result { - let wanted: Vec = match release { - None => Vec::new(), - Some(name) if name.trim().eq_ignore_ascii_case("all") => ModeClamp::ALL.to_vec(), - Some(name) => vec![ModeClamp::parse(name).ok_or(ControlError::UnknownClamp)?], - }; - - let mut released: Vec<&'static str> = Vec::new(); - for clamp in wanted { - if release_clamp(shared, clamp) { - released.push(clamp.label()); - } - } - - if !released.is_empty() { - tracing::warn!( - released = ?released, - "an operator released mode clamps over the control plane, so the effective mode may \ - rise before the condition that engaged them is gone" - ); - } - - Ok(shared.with_state_mut(|state| { - let limits = state.governor.limits().breaker; - ClampsResponse { - ok: true, - configured_mode: state.governor.configured_mode().as_str(), - effective_mode: state.governor.effective_mode().as_str(), - ceiling: state.governor.clamps().ceiling().as_str(), - clamps: ModeClamp::ALL - .into_iter() - .map(|clamp| ClampEntry { - clamp: clamp.label(), - engaged: state.governor.clamps().is_engaged(clamp), - self_clearing: clamp.is_self_clearing(), - expires_in_ms: state - .governor - .clamps() - .deadline(clamp) - .map(|until| until.saturating_since(now)), - }) - .collect(), - released, - breaker_tripped: state.governor.breaker().is_tripped(), - breaker_auto_resets_used: state.governor.breaker().auto_resets_used(), - breaker_held_for_manual_reset: state.governor.breaker().held_for_manual_reset(limits), - mutations_paused_windows: state.mutations_pause.windows(), - mutations_paused_hold_ms: state.mutations_pause.hold_ms(), - } - })) -} - -pub fn cliffs(shared: &Shared, release: bool, now: Millis) -> CliffsResponse { - let hold_ms = shared.config().server_cliff_hold_ms; - let response = shared.with_state_mut(|state| { - let cliffed = state.cliffs.cliffed(now); - let disputed = state.cliffs.disputed(); - let holed = state.suspicion.holed_servers(); - let mut released: Vec = Vec::new(); - - if release { - for location in cliffed.iter().chain(holed.iter()) { - let forgotten = state.cliffs.forget(location); - let filled = state.suspicion.clear_hole(location); - if forgotten || filled { - released.push(location.to_string()); - } - } - released.sort(); - released.dedup(); - } - - CliffsResponse { - ok: true, - cliffed: cliffed.iter().map(ToString::to_string).collect(), - disputed: disputed.iter().map(ToString::to_string).collect(), - holed: holed.iter().map(ToString::to_string).collect(), - released, - hold_ms, - tripped_total: state.cliffs.tripped_total(), - disputed_total: state.cliffs.disputed_total(), - holes_total: state.suspicion.holes_total(), - } - }); - - if !response.released.is_empty() { - tracing::warn!( - released = ?response.released, - "an operator dropped server holds over the control plane, so those servers read as \ - authoritative again before their hold window ended" - ); - } - - response -} - -pub fn suspect(shared: &Shared, room: RoomKey, now: Millis) -> SuspectResponse { - let (admitted, raised, suspect_rooms, hold_ms) = shared.with_state_mut(|state| { - let admitted = state.directory.note_suspicion(room, now); - let raised = state - .suspicion - .note_room(room, SuspicionSource::Control, now); - ( - admitted.label(), - raised.label(), - state.suspicion.suspect_rooms(now).len(), - state.suspicion.limits().hold_ms, - ) - }); - - tracing::info!( - channel_id = room.channel_id().get(), - guild_id = room.guild_id().map(|guild| guild.get()), - admitted, - raised, - "an operator raised suspicion over the control plane, which only reorders reads" - ); - - SuspectResponse { - ok: true, - scope: crate::metrics::scope_label(room.scope()), - guild_id: room.guild_id().map(|guild| guild.to_string()), - channel_id: room.channel_id().to_string(), - admitted, - raised, - hold_ms, - suspect_rooms, - } -} - -pub fn dispatch(shared: &Shared, request: &ControlRequest, now: Millis) -> serde_json::Value { - match request { - ControlRequest::Status => serde_json::json!(status(shared, now)), - ControlRequest::Budget => serde_json::json!(budget(shared, now)), - ControlRequest::Mode { mode } => match mode { - None => serde_json::json!(read_mode(shared)), - Some(raw) => match set_mode(shared, Some(raw)) { - Ok(response) => serde_json::json!(response), - Err(code) => error(code), - }, - }, - ControlRequest::Explain { - guild_id, - channel_id, - } => match room_key(guild_id.as_ref(), channel_id) { - Ok(room) => serde_json::json!(explain(shared, room, now)), - Err(code) => error(code), - }, - ControlRequest::Topology { refresh } => { - serde_json::json!(topology(shared, refresh.unwrap_or(false), now)) - } - ControlRequest::Clamps { release } => match clamps(shared, release.as_deref(), now) { - Ok(response) => serde_json::json!(response), - Err(code) => error(code), - }, - ControlRequest::Suspect { - guild_id, - channel_id, - } => match room_key(guild_id.as_ref(), channel_id) { - Ok(room) => serde_json::json!(suspect(shared, room, now)), - Err(code) => error(code), - }, - ControlRequest::Cliffs { release } => { - serde_json::json!(cliffs(shared, release.unwrap_or(false), now)) - } - } -} - -pub fn handle(shared: &Shared, payload: &[u8], now: Millis) -> Vec { - let response = match serde_json::from_slice::(payload) { - Ok(request) => dispatch(shared, &request, now), - Err(parse_error) => { - tracing::debug!(error = %parse_error, "rejecting an undecodable control request"); - error(ControlError::DecodeFailed) - } - }; - serde_json::to_vec(&response) - .unwrap_or_else(|_| b"{\"ok\":false,\"error\":\"encode_failed\"}".to_vec()) -} - -pub async fn run_control(shared: Shared, transport: T) -> anyhow::Result<()> { - let mut backoff_ms = RESUBSCRIBE_BACKOFF_MS; - - loop { - let mut subscription = match transport.subscribe(CONTROL_SUBJECT).await { - Ok(subscription) => { - backoff_ms = RESUBSCRIBE_BACKOFF_MS; - subscription - } - Err(error) => { - tracing::error!( - error = %error, - subject = CONTROL_SUBJECT, - retry_in_ms = backoff_ms, - "the control plane could not subscribe, retrying rather than ending the task, \ - because the escape hatch for a clamped service must outlive a broken \ - subscription" - ); - tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; - backoff_ms = next_backoff_ms(backoff_ms); - continue; - } - }; - - tracing::info!( - subject = CONTROL_SUBJECT, - instance_id = shared.instance().id, - "recon control plane listening" - ); - - loop { - tokio::select! { - message = subscription.next() => { - let Some(message) = message else { - tracing::warn!("control subscription ended, will re-subscribe"); - break; - }; - if !message.has_reply() { - continue; - } - let response = handle(&shared, message.payload(), monotonic_now()); - if let Err(error) = reply_message(&message, &transport, &response).await { - tracing::debug!(error = %error, "failed to answer a control request"); - } - } - _ = transport.wait_for_reconnect() => { - tracing::info!("NATS reconnected, re-subscribing the control plane"); - break; - } - } - } - } -} diff --git a/fluxer_recon/src/decide.rs b/fluxer_recon/src/decide.rs deleted file mode 100644 index f1a241845..000000000 --- a/fluxer_recon/src/decide.rs +++ /dev/null @@ -1,873 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::budget::BudgetView; -use crate::config::ReconConfig; -use crate::evidence::{PendingJoin, Presence, RoomObservation, Side, UnknownReason}; -use crate::gateway::Nonce; -use crate::guards::{AbortReason, DmPosture}; -use crate::ids::{ConnectionId, ConnectionKey, Location, Millis, RoomKey, Scope, TurnId, UserId}; -use crate::ledger::{ - ActionKind, ConnectionState, Fingerprint, FingerprintInput, Ledger, LocationIndex, - RepairVerdict, WedgedReason, fingerprint, -}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct DecideParams { - pub required_corroborations: u32, - pub min_corroboration_gap_ms: u64, - pub max_corroboration_window_ms: u64, - pub min_divergence_ms: u64, - pub max_action_attempts: u32, - pub pending_join_skew_ms: u64, - pub dm_posture: DmPosture, -} - -impl Default for DecideParams { - fn default() -> Self { - Self { - required_corroborations: 3, - min_corroboration_gap_ms: 1_000, - max_corroboration_window_ms: 60_000, - min_divergence_ms: 15_000, - max_action_attempts: 3, - pending_join_skew_ms: 5_000, - dm_posture: DmPosture::default(), - } - } -} - -impl DecideParams { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - required_corroborations: config.required_corroborations, - min_corroboration_gap_ms: config.min_corroboration_gap_ms, - max_corroboration_window_ms: config.max_corroboration_window_ms, - min_divergence_ms: config.min_divergence_ms, - max_action_attempts: config.max_action_attempts, - pending_join_skew_ms: config.pending_join_skew_ms, - dm_posture: DmPosture { - dm_gateway_eviction: config.dm_gateway_eviction, - dm_media_eviction: config.dm_media_eviction, - }, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ActionClass { - None, - Constructive, - Destructive, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum DecisionAction { - Hold, - ConfirmConnection { nonce: Nonce }, - RepairState, - RemoveGatewayState, - RemoveParticipant { location: Location }, -} - -impl DecisionAction { - pub const fn class(&self) -> ActionClass { - match self { - Self::Hold => ActionClass::None, - Self::ConfirmConnection { .. } | Self::RepairState => ActionClass::Constructive, - Self::RemoveGatewayState | Self::RemoveParticipant { .. } => ActionClass::Destructive, - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Hold => "hold", - Self::ConfirmConnection { .. } => "confirm_connection", - Self::RepairState => "repair_state", - Self::RemoveGatewayState => "remove_gateway_state", - Self::RemoveParticipant { .. } => "remove_participant", - } - } - - pub const fn kind(&self) -> Option { - match self { - Self::Hold | Self::ConfirmConnection { .. } | Self::RepairState => None, - Self::RemoveGatewayState => Some(ActionKind::RemoveGatewayState), - Self::RemoveParticipant { .. } => Some(ActionKind::RemoveParticipant), - } - } - - pub const fn is_destructive(&self) -> bool { - matches!(self.class(), ActionClass::Destructive) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum DecisionReason { - SideUnknown(UnknownReason), - FingerprintChanged, - BothSidesPresent, - BothSidesAbsent, - DivergenceOpened, - CorroborationRecorded, - CorroborationSpacedTooTightly, - CorroborationWindowExpired, - DivergenceTooYoung, - EvidenceComplete, - PendingJoinOutstanding, - RepairAttempted, - RepairSucceeded, - RepairMadeNoChange, - RepairNotPossible, - RepairNotDefinitive, - RepairExhausted, - ActionUnresolved, - ActionsExhausted, - ParticipantJoinedAfterDivergence, - LocationUnavailable, - WedgedHolding, -} - -impl DecisionReason { - pub const fn label(&self) -> &'static str { - match self { - Self::SideUnknown(_) => "side_unknown", - Self::FingerprintChanged => "fingerprint_changed", - Self::BothSidesPresent => "both_sides_present", - Self::BothSidesAbsent => "both_sides_absent", - Self::DivergenceOpened => "divergence_opened", - Self::CorroborationRecorded => "corroboration_recorded", - Self::CorroborationSpacedTooTightly => "corroboration_spaced_too_tightly", - Self::CorroborationWindowExpired => "corroboration_window_expired", - Self::DivergenceTooYoung => "divergence_too_young", - Self::EvidenceComplete => "evidence_complete", - Self::PendingJoinOutstanding => "pending_join_outstanding", - Self::RepairAttempted => "repair_attempted", - Self::RepairSucceeded => "repair_succeeded", - Self::RepairMadeNoChange => "repair_made_no_change", - Self::RepairNotPossible => "repair_not_possible", - Self::RepairNotDefinitive => "repair_not_definitive", - Self::RepairExhausted => "repair_exhausted", - Self::ActionUnresolved => "action_unresolved", - Self::ActionsExhausted => "actions_exhausted", - Self::ParticipantJoinedAfterDivergence => "participant_joined_after_divergence", - Self::LocationUnavailable => "location_unavailable", - Self::WedgedHolding => "wedged_holding", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Decision { - pub connection: ConnectionKey, - pub user_id: UserId, - pub action: DecisionAction, - pub reason: DecisionReason, - pub from: &'static str, - pub to: &'static str, - pub blocked_by: Option, -} - -impl Decision { - pub const fn is_actionable(&self) -> bool { - self.blocked_by.is_none() && !matches!(self.action, DecisionAction::Hold) - } - - pub const fn is_destructive(&self) -> bool { - self.action.is_destructive() - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DecisionSet { - pub room: RoomKey, - pub turn: TurnId, - pub at: Millis, - pub decisions: Vec, -} - -impl DecisionSet { - pub fn empty(room: RoomKey, turn: TurnId, at: Millis) -> Self { - Self { - room, - turn, - at, - decisions: Vec::new(), - } - } - - pub fn actionable(&self) -> impl Iterator { - self.decisions.iter().filter(|entry| entry.is_actionable()) - } - - pub fn destructive(&self) -> impl Iterator { - self.decisions.iter().filter(|entry| entry.is_destructive()) - } - - pub fn actionable_destructive(&self) -> impl Iterator { - self.decisions - .iter() - .filter(|entry| entry.is_actionable() && entry.is_destructive()) - } - - pub fn blocked(&self) -> impl Iterator { - self.decisions - .iter() - .filter_map(|entry| entry.blocked_by.map(|reason| (entry, reason))) - } - - pub fn for_connection(&self, connection: &ConnectionId) -> Option<&Decision> { - self.decisions - .iter() - .find(|entry| &entry.connection.connection == connection) - } -} - -struct Context<'a> { - params: &'a DecideParams, - room: RoomKey, - now: Millis, - fingerprint: Fingerprint, - location: Option, - location_value: Option<&'a Location>, - joined_at: Option, - pending_join: Option<&'a PendingJoin>, - budget: &'a BudgetView, - action_attempts: u32, - repair_attempts: u32, - repair_verdict: Option, -} - -struct Outcome { - state: ConnectionState, - action: DecisionAction, - reason: DecisionReason, - blocked_by: Option, -} - -impl Outcome { - const fn hold(state: ConnectionState, reason: DecisionReason) -> Self { - Self { - state, - action: DecisionAction::Hold, - reason, - blocked_by: None, - } - } -} - -pub fn decide( - ledger: &mut Ledger, - observation: &RoomObservation, - now: Millis, - budget: &BudgetView, -) -> DecisionSet { - let params = *ledger.params(); - let room = observation.room; - - if ledger.ensure_room(room, now).is_err() { - return DecisionSet::empty(room, observation.turn, now); - } - - let no_candidates = observation.candidates.is_empty(); - let partially_unreadable = matches!( - observation.authority.media.unknown_reason(), - Some(UnknownReason::RoomPartiallyUnreadable) - ); - if let Some(entry) = ledger.room_mut(&room) { - entry.note_turn(observation.turn, now); - entry.set_no_candidates(no_candidates); - entry.set_partially_unreadable(partially_unreadable); - } - - let mut work: Vec<(ConnectionId, UserId)> = observation - .connections - .iter() - .map(|entry| (entry.connection.clone(), entry.user_id)) - .collect(); - if let Some(entry) = ledger.room(&room) { - for tracked in entry.connections() { - if !work.iter().any(|(id, _)| id == &tracked.connection) { - work.push((tracked.connection.clone(), tracked.user_id)); - } - } - } - work.sort(); - work.dedup(); - - let unsighted = observation.unsighted_presence(); - let mut decisions = Vec::with_capacity(work.len()); - - for (connection, user_id) in work { - let observed = observation.observation_for(&connection); - let presence: Side = - observed.map_or_else(|| unsighted.clone(), |entry| entry.presence.clone()); - let siblings = observed - .map(|entry| entry.gateway_siblings.clone()) - .unwrap_or_default(); - let locations = observed - .map(|entry| entry.media_locations.clone()) - .unwrap_or_default(); - let joined_at = observed.and_then(|entry| entry.participant_joined_at); - - let print = fingerprint(&FingerprintInput { - room, - connection: &connection, - user_id, - gateway_siblings: &siblings, - media_locations: &locations, - participant_joined_at: joined_at, - census_epoch: observation.census_epoch, - topology_epoch: observation.topology_epoch, - }); - - let location_index = locations - .first() - .and_then(|location| ledger.intern_location(location)); - if let (Some(index), Some(entry)) = (location_index, ledger.room_mut(&room)) { - entry.note_location(index); - } - - let pending_join = observation.pending_join_for( - &connection, - observation.wall_at, - params.pending_join_skew_ms, - ); - - let Ok(entry) = ledger.upsert_connection(room, &connection, user_id, now) else { - continue; - }; - if let Some(index) = location_index { - entry.last_location = Some(index); - } - - let context = Context { - params: ¶ms, - room, - now, - fingerprint: print, - location: location_index, - location_value: locations.first(), - joined_at, - pending_join, - budget, - action_attempts: u32::from(entry.action_attempts), - repair_attempts: u32::from(entry.repair_attempts), - repair_verdict: entry.last_repair_verdict, - }; - - let from = entry.state.label(); - let outcome = transition(&entry.state, &presence, &context); - - if matches!( - outcome.state, - ConnectionState::Nascent | ConnectionState::Consistent - ) { - entry.last_repair_verdict = None; - } - entry.state = outcome.state; - - decisions.push(Decision { - connection: ConnectionKey::new(room, connection), - user_id, - action: outcome.action, - reason: outcome.reason, - from, - to: entry.state.label(), - blocked_by: outcome.blocked_by, - }); - } - - DecisionSet { - room, - turn: observation.turn, - at: now, - decisions, - } -} - -fn transition(prior: &ConnectionState, presence: &Side, ctx: &Context<'_>) -> Outcome { - match (&presence.gateway, &presence.media) { - (Presence::Unknown(reason), Presence::Unknown(_)) - | (Presence::Unknown(reason), Presence::Present) - | (Presence::Unknown(reason), Presence::Absent) - | (Presence::Present, Presence::Unknown(reason)) - | (Presence::Absent, Presence::Unknown(reason)) => Outcome::hold( - ConnectionState::Nascent, - DecisionReason::SideUnknown(reason.clone()), - ), - (Presence::Present, Presence::Present) => Outcome::hold( - ConnectionState::Consistent, - DecisionReason::BothSidesPresent, - ), - (Presence::Absent, Presence::Absent) => Outcome::hold( - ConnectionState::Retired { - at: retired_at(prior, ctx.now), - }, - DecisionReason::BothSidesAbsent, - ), - (Presence::Present, Presence::Absent) => gateway_only_lane(prior, ctx), - (Presence::Absent, Presence::Present) => media_only_lane(prior, ctx), - } -} - -const fn retired_at(prior: &ConnectionState, now: Millis) -> Millis { - match prior { - ConnectionState::Retired { at } => *at, - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::GatewayOnly { .. } - | ConnectionState::MediaOnly { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::ActionTaken { .. } - | ConnectionState::Wedged { .. } => now, - } -} - -fn fingerprint_broke(prior: &ConnectionState, ctx: &Context<'_>) -> bool { - prior - .stored_fingerprint() - .is_some_and(|stored| stored != ctx.fingerprint) -} - -fn nascent_on_fingerprint_change() -> Outcome { - Outcome::hold(ConnectionState::Nascent, DecisionReason::FingerprintChanged) -} - -fn gateway_only_lane(prior: &ConnectionState, ctx: &Context<'_>) -> Outcome { - if fingerprint_broke(prior, ctx) { - return nascent_on_fingerprint_change(); - } - - match prior { - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::MediaOnly { .. } - | ConnectionState::Retired { .. } => open_gateway_divergence(ctx), - ConnectionState::GatewayOnly { - since, - corroborations, - last_corroboration, - fingerprint: _, - } => corroborate_gateway(*since, *corroborations, *last_corroboration, ctx), - ConnectionState::ActionTaken { kind, since, .. } => match kind { - ActionKind::RemoveGatewayState => { - if ctx.action_attempts >= ctx.params.max_action_attempts { - return wedged( - WedgedReason::ActionIneffective, - ctx, - DecisionReason::ActionsExhausted, - ); - } - evaluate_gateway( - *since, - ctx.params.required_corroborations, - ctx.now, - ctx, - DecisionReason::ActionUnresolved, - ) - } - ActionKind::RemoveParticipant => open_gateway_divergence(ctx), - }, - ConnectionState::Wedged { - reason, - at, - fingerprint: print, - } => Outcome::hold( - ConnectionState::Wedged { - reason: *reason, - at: *at, - fingerprint: *print, - }, - DecisionReason::WedgedHolding, - ), - } -} - -fn open_gateway_divergence(ctx: &Context<'_>) -> Outcome { - Outcome::hold( - ConnectionState::GatewayOnly { - since: ctx.now, - corroborations: 1, - fingerprint: ctx.fingerprint, - last_corroboration: ctx.now, - }, - DecisionReason::DivergenceOpened, - ) -} - -fn corroborate_gateway( - since: Millis, - corroborations: u32, - last_corroboration: Millis, - ctx: &Context<'_>, -) -> Outcome { - if ctx.now.saturating_since(since) > ctx.params.max_corroboration_window_ms { - return Outcome::hold( - ConnectionState::GatewayOnly { - since: ctx.now, - corroborations: 1, - fingerprint: ctx.fingerprint, - last_corroboration: ctx.now, - }, - DecisionReason::CorroborationWindowExpired, - ); - } - - if ctx.now.saturating_since(last_corroboration) < ctx.params.min_corroboration_gap_ms { - return evaluate_gateway( - since, - corroborations, - last_corroboration, - ctx, - DecisionReason::CorroborationSpacedTooTightly, - ); - } - - evaluate_gateway( - since, - corroborations.saturating_add(1), - ctx.now, - ctx, - DecisionReason::CorroborationRecorded, - ) -} - -fn evaluate_gateway( - since: Millis, - corroborations: u32, - last_corroboration: Millis, - ctx: &Context<'_>, - progress: DecisionReason, -) -> Outcome { - let state = ConnectionState::GatewayOnly { - since, - corroborations, - fingerprint: ctx.fingerprint, - last_corroboration, - }; - - if corroborations < ctx.params.required_corroborations { - return Outcome::hold(state, progress); - } - - if ctx.now.saturating_since(since) < ctx.params.min_divergence_ms { - return Outcome::hold(state, DecisionReason::DivergenceTooYoung); - } - - if ctx.action_attempts >= ctx.params.max_action_attempts { - return wedged( - WedgedReason::ActionIneffective, - ctx, - DecisionReason::ActionsExhausted, - ); - } - - let blocked_by = if ctx.params.dm_posture.allows_gateway_removal(ctx.room) { - ctx.budget.destructive_block() - } else { - Some(AbortReason::DmPosture) - }; - - Outcome { - state, - action: DecisionAction::RemoveGatewayState, - reason: DecisionReason::EvidenceComplete, - blocked_by, - } -} - -fn media_only_lane(prior: &ConnectionState, ctx: &Context<'_>) -> Outcome { - if fingerprint_broke(prior, ctx) { - return nascent_on_fingerprint_change(); - } - - match prior { - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::GatewayOnly { .. } - | ConnectionState::Retired { .. } => open_media_divergence(ctx), - ConnectionState::PendingJoin { last_confirm, .. } => match ctx.pending_join { - None => open_media_divergence(ctx), - Some(join) => { - let state = ConnectionState::PendingJoin { - expires_at: join.expires_at, - nonce: join.nonce.clone(), - last_confirm: *last_confirm, - }; - if ctx.now.saturating_since(*last_confirm) < ctx.params.min_corroboration_gap_ms { - return Outcome::hold(state, DecisionReason::PendingJoinOutstanding); - } - Outcome { - state, - action: DecisionAction::ConfirmConnection { - nonce: join.nonce.clone(), - }, - reason: DecisionReason::PendingJoinOutstanding, - blocked_by: ctx.budget.constructive_block(), - } - } - }, - ConnectionState::Repairing { - since, - last_attempt, - } => repairing_lane(*since, *last_attempt, ctx), - ConnectionState::MediaOnly { - since, - corroborations, - last_corroboration, - location, - participant_joined_at, - repair_verdict, - fingerprint: _, - } => corroborate_media( - *since, - *corroborations, - *last_corroboration, - *location, - *participant_joined_at, - *repair_verdict, - ctx, - ), - ConnectionState::ActionTaken { kind, since, .. } => match kind { - ActionKind::RemoveParticipant => { - if ctx.action_attempts >= ctx.params.max_action_attempts { - return wedged( - WedgedReason::ActionIneffective, - ctx, - DecisionReason::ActionsExhausted, - ); - } - reopen_media_divergence(*since, ctx, DecisionReason::ActionUnresolved) - } - ActionKind::RemoveGatewayState => open_media_divergence(ctx), - }, - ConnectionState::Wedged { - reason, - at, - fingerprint: print, - } => Outcome::hold( - ConnectionState::Wedged { - reason: *reason, - at: *at, - fingerprint: *print, - }, - DecisionReason::WedgedHolding, - ), - } -} - -fn open_media_divergence(ctx: &Context<'_>) -> Outcome { - if let Some(join) = ctx.pending_join { - return Outcome { - state: ConnectionState::PendingJoin { - expires_at: join.expires_at, - nonce: join.nonce.clone(), - last_confirm: Millis::ZERO, - }, - action: DecisionAction::ConfirmConnection { - nonce: join.nonce.clone(), - }, - reason: DecisionReason::PendingJoinOutstanding, - blocked_by: ctx.budget.constructive_block(), - }; - } - - match ctx.room.scope() { - Scope::Guild => { - if ctx.repair_attempts > ctx.params.max_action_attempts { - return wedged( - WedgedReason::RepairExhausted, - ctx, - DecisionReason::RepairExhausted, - ); - } - Outcome { - state: ConnectionState::Repairing { - since: ctx.now, - last_attempt: Millis::ZERO, - }, - action: DecisionAction::RepairState, - reason: DecisionReason::RepairAttempted, - blocked_by: ctx.budget.constructive_block(), - } - } - Scope::Dm => reopen_media_divergence(ctx.now, ctx, DecisionReason::DivergenceOpened), - } -} - -fn repairing_lane(since: Millis, last_attempt: Millis, ctx: &Context<'_>) -> Outcome { - match ctx.repair_verdict { - Some(RepairVerdict::Repaired) => { - Outcome::hold(ConnectionState::Consistent, DecisionReason::RepairSucceeded) - } - Some(RepairVerdict::NotRepairable) => { - reopen_media_divergence(since, ctx, DecisionReason::RepairNotPossible) - } - Some(RepairVerdict::NoChange) | None => { - if ctx.repair_attempts > ctx.params.max_action_attempts { - return wedged( - WedgedReason::RepairExhausted, - ctx, - DecisionReason::RepairExhausted, - ); - } - let state = ConnectionState::Repairing { - since, - last_attempt, - }; - if ctx.now.saturating_since(last_attempt) < ctx.params.min_corroboration_gap_ms { - return Outcome::hold(state, DecisionReason::RepairMadeNoChange); - } - Outcome { - state, - action: DecisionAction::RepairState, - reason: DecisionReason::RepairAttempted, - blocked_by: ctx.budget.constructive_block(), - } - } - } -} - -fn reopen_media_divergence(since: Millis, ctx: &Context<'_>, reason: DecisionReason) -> Outcome { - let Some(location) = ctx.location else { - return Outcome::hold( - ConnectionState::Nascent, - DecisionReason::LocationUnavailable, - ); - }; - - Outcome::hold( - ConnectionState::MediaOnly { - since, - corroborations: 1, - fingerprint: ctx.fingerprint, - last_corroboration: ctx.now, - location, - participant_joined_at: ctx.joined_at.unwrap_or(ctx.now), - repair_verdict: ctx.repair_verdict, - }, - reason, - ) -} - -#[allow(clippy::too_many_arguments)] -fn corroborate_media( - since: Millis, - corroborations: u32, - last_corroboration: Millis, - location: LocationIndex, - participant_joined_at: Millis, - repair_verdict: Option, - ctx: &Context<'_>, -) -> Outcome { - let joined_at = ctx.joined_at.unwrap_or(participant_joined_at); - - if joined_at > since { - return Outcome::hold( - ConnectionState::MediaOnly { - since: ctx.now, - corroborations: 1, - fingerprint: ctx.fingerprint, - last_corroboration: ctx.now, - location, - participant_joined_at: joined_at, - repair_verdict, - }, - DecisionReason::ParticipantJoinedAfterDivergence, - ); - } - - if ctx.now.saturating_since(since) > ctx.params.max_corroboration_window_ms { - return Outcome::hold( - ConnectionState::MediaOnly { - since: ctx.now, - corroborations: 1, - fingerprint: ctx.fingerprint, - last_corroboration: ctx.now, - location, - participant_joined_at: joined_at, - repair_verdict, - }, - DecisionReason::CorroborationWindowExpired, - ); - } - - let spaced = - ctx.now.saturating_since(last_corroboration) >= ctx.params.min_corroboration_gap_ms; - let corroborations = if spaced { - corroborations.saturating_add(1) - } else { - corroborations - }; - let last_corroboration = if spaced { ctx.now } else { last_corroboration }; - let progress = if spaced { - DecisionReason::CorroborationRecorded - } else { - DecisionReason::CorroborationSpacedTooTightly - }; - - let state = ConnectionState::MediaOnly { - since, - corroborations, - fingerprint: ctx.fingerprint, - last_corroboration, - location, - participant_joined_at: joined_at, - repair_verdict, - }; - - if corroborations < ctx.params.required_corroborations { - return Outcome::hold(state, progress); - } - - if ctx.now.saturating_since(since) < ctx.params.min_divergence_ms { - return Outcome::hold(state, DecisionReason::DivergenceTooYoung); - } - - if matches!(ctx.room, RoomKey::Guild { .. }) - && repair_verdict != Some(RepairVerdict::NotRepairable) - { - return Outcome::hold(state, DecisionReason::RepairNotDefinitive); - } - - if ctx.action_attempts >= ctx.params.max_action_attempts { - return wedged( - WedgedReason::ActionIneffective, - ctx, - DecisionReason::ActionsExhausted, - ); - } - - let Some(target) = ctx.location_value else { - return Outcome::hold(state, DecisionReason::LocationUnavailable); - }; - - let blocked_by = if ctx.params.dm_posture.allows_media_removal(ctx.room) { - ctx.budget.destructive_block() - } else { - Some(AbortReason::DmPosture) - }; - - Outcome { - state, - action: DecisionAction::RemoveParticipant { - location: target.clone(), - }, - reason: DecisionReason::EvidenceComplete, - blocked_by, - } -} - -fn wedged(reason: WedgedReason, ctx: &Context<'_>, decision: DecisionReason) -> Outcome { - Outcome::hold( - ConnectionState::Wedged { - reason, - at: ctx.now, - fingerprint: ctx.fingerprint, - }, - decision, - ) -} diff --git a/fluxer_recon/src/discovery.rs b/fluxer_recon/src/discovery.rs deleted file mode 100644 index 2b3484373..000000000 --- a/fluxer_recon/src/discovery.rs +++ /dev/null @@ -1,1656 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::{BTreeMap, BTreeSet}; - -use crate::census::CensusSnapshot; -use crate::clock::Clock; -use crate::config::ReconConfig; -use crate::evidence::{ - CandidateSet, CandidateSources, CensusCrossCheck, GatewayAuthorityInput, GatewayRead, - GatewayVoiceState, LocationProbe, MediaAuthorityInput, MediaParticipant, PendingJoin, - ProbeResult, RoomObservation, RoomReads, ServerCliffConfig, ServerCliffDetector, Side, - SideAuthority, TopologyFreshness, UnknownReason, gateway_authority, media_authority, - unread_believed_home, -}; -use crate::gateway::GatewayApi; -use crate::health::{ProbeOutcome, ServerHealth, ServerHealthMap}; -use crate::ids::{ConnectionId, Epoch, Location, Millis, RoomKey, TurnId, WallMillis}; -use crate::ledger::{LOCATION_ENTRY_BYTES, LOCATION_MAP_BYTES, MAP_ENTRY_OVERHEAD_BYTES}; -use crate::livekit::{LiveKitApi, LiveKitFault, ParticipantRecord, ReadResult}; -use crate::names::{parse_participant_identity, parse_room_name}; -use crate::turn::{Digest, Fresh, ReadSource, TurnError, TurnToken}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum RoomSource { - GatewayCensus, - MediaRoomList, - PinnedServer, - Ledger, - Suspicion, -} - -impl RoomSource { - pub const ALL: [Self; 5] = [ - Self::GatewayCensus, - Self::MediaRoomList, - Self::PinnedServer, - Self::Ledger, - Self::Suspicion, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::GatewayCensus => "gateway_census", - Self::MediaRoomList => "media_room_list", - Self::PinnedServer => "pinned_server", - Self::Ledger => "ledger", - Self::Suspicion => "suspicion", - } - } - - pub const fn index(self) -> usize { - match self { - Self::GatewayCensus => 0, - Self::MediaRoomList => 1, - Self::PinnedServer => 2, - Self::Ledger => 3, - Self::Suspicion => 4, - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct RoomProvenance { - seen: [Option; RoomSource::ALL.len()], -} - -impl RoomProvenance { - pub const fn new() -> Self { - Self { - seen: [None; RoomSource::ALL.len()], - } - } - - const fn note(&mut self, source: RoomSource, now: Millis) { - self.seen[source.index()] = Some(now); - } - - pub const fn has(&self, source: RoomSource) -> bool { - self.seen[source.index()].is_some() - } - - pub const fn last_seen_by(&self, source: RoomSource) -> Option { - self.seen[source.index()] - } - - pub fn count(&self) -> usize { - self.seen.iter().filter(|entry| entry.is_some()).count() - } - - pub fn sources(&self) -> Vec { - RoomSource::ALL - .into_iter() - .filter(|source| self.has(*source)) - .collect() - } - - pub fn only(&self, source: RoomSource) -> bool { - self.has(source) && self.count() == 1 - } - - pub const fn is_gateway_only(&self) -> bool { - self.has(RoomSource::GatewayCensus) && !self.has(RoomSource::MediaRoomList) - } - - pub const fn is_media_only(&self) -> bool { - self.has(RoomSource::MediaRoomList) && !self.has(RoomSource::GatewayCensus) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Admission { - Admitted, - Refreshed, - Refused, -} - -impl Admission { - pub const fn label(self) -> &'static str { - match self { - Self::Admitted => "admitted", - Self::Refreshed => "refreshed", - Self::Refused => "refused", - } - } - - pub const fn is_tracked(self) -> bool { - matches!(self, Self::Admitted | Self::Refreshed) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct DirectoryLimits { - pub max_rooms: usize, -} - -impl DirectoryLimits { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - max_rooms: config.max_tracked_rooms, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RoomEntry { - provenance: RoomProvenance, - locations: BTreeMap, - pinned: Option, - first_at: Millis, - last_at: Millis, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ServerRoomList { - Listed { - rooms: Vec, - unparseable: usize, - }, - Unreadable(LiveKitFault), -} - -impl ServerRoomList { - pub const fn is_readable(&self) -> bool { - matches!(self, Self::Listed { .. }) - } - - pub const fn is_complete(&self) -> bool { - matches!(self, Self::Listed { unparseable: 0, .. }) - } - - pub fn rooms(&self) -> &[RoomKey] { - match self { - Self::Listed { rooms, .. } => rooms, - Self::Unreadable(_) => &[], - } - } - - pub const fn unparseable(&self) -> usize { - match self { - Self::Listed { unparseable, .. } => *unparseable, - Self::Unreadable(_) => 0, - } - } - - pub const fn fault(&self) -> Option { - match self { - Self::Listed { .. } => None, - Self::Unreadable(fault) => Some(*fault), - } - } - - pub const fn outcome(&self) -> ProbeOutcome { - match self { - Self::Listed { .. } => ProbeOutcome::Success, - Self::Unreadable(fault) => { - if fault.is_auth_failure() { - ProbeOutcome::AuthFailure - } else { - ProbeOutcome::Failure - } - } - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Listed { .. } => "listed", - Self::Unreadable(_) => "unreadable", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RoomListEffect { - Applied { - listed: usize, - added: usize, - withdrawn: usize, - }, - AdditiveOnly { - listed: usize, - added: usize, - unparseable: usize, - }, - Unreadable(LiveKitFault), -} - -impl RoomListEffect { - pub const fn label(self) -> &'static str { - match self { - Self::Applied { .. } => "applied", - Self::AdditiveOnly { .. } => "additive_only", - Self::Unreadable(_) => "unreadable", - } - } - - pub const fn withdrew_locations(self) -> bool { - matches!(self, Self::Applied { .. }) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DiscoveryPass { - pub at: Millis, - pub servers: usize, - pub readable: usize, - pub complete: usize, - pub rooms_added: usize, - pub locations_withdrawn: usize, - pub unreadable: Vec<(Location, LiveKitFault)>, -} - -impl DiscoveryPass { - pub const fn empty(at: Millis) -> Self { - Self { - at, - servers: 0, - readable: 0, - complete: 0, - rooms_added: 0, - locations_withdrawn: 0, - unreadable: Vec::new(), - } - } - - pub fn is_complete(&self) -> bool { - self.servers > 0 && self.complete == self.servers - } - - pub fn unreadable_servers(&self) -> usize { - self.unreadable.len() - } -} - -#[derive(Clone, Debug)] -pub struct RoomDirectory { - limits: DirectoryLimits, - rooms: BTreeMap, - listings: BTreeMap, - relocated: Vec, - admitted_total: u64, - refused_total: u64, - forgotten_total: u64, - unreadable_lists_total: u64, - incomplete_lists_total: u64, - withdrawn_total: u64, -} - -pub const DIRECTORY_ENTRY_BYTES: u64 = - (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; - -impl RoomDirectory { - pub fn tracked_bytes(&self) -> u64 { - self.rooms - .values() - .map(|entry| { - let map = if entry.locations.is_empty() { - 0 - } else { - LOCATION_MAP_BYTES - }; - DIRECTORY_ENTRY_BYTES.saturating_add(map).saturating_add( - (entry.locations.len() as u64).saturating_mul(LOCATION_ENTRY_BYTES), - ) - }) - .fold(0u64, u64::saturating_add) - .saturating_add((self.listings.len() as u64).saturating_mul(LOCATION_ENTRY_BYTES)) - } - - pub const fn new(limits: DirectoryLimits) -> Self { - Self { - limits, - rooms: BTreeMap::new(), - listings: BTreeMap::new(), - relocated: Vec::new(), - admitted_total: 0, - refused_total: 0, - forgotten_total: 0, - unreadable_lists_total: 0, - incomplete_lists_total: 0, - withdrawn_total: 0, - } - } - - pub const fn limits(&self) -> DirectoryLimits { - self.limits - } - - pub fn len(&self) -> usize { - self.rooms.len() - } - - pub fn is_empty(&self) -> bool { - self.rooms.is_empty() - } - - pub const fn admitted_total(&self) -> u64 { - self.admitted_total - } - - pub const fn refused_total(&self) -> u64 { - self.refused_total - } - - pub const fn forgotten_total(&self) -> u64 { - self.forgotten_total - } - - pub const fn unreadable_lists_total(&self) -> u64 { - self.unreadable_lists_total - } - - pub const fn incomplete_lists_total(&self) -> u64 { - self.incomplete_lists_total - } - - pub const fn withdrawn_total(&self) -> u64 { - self.withdrawn_total - } - - pub fn rooms(&self) -> impl Iterator { - self.rooms - .iter() - .map(|(room, entry)| (*room, &entry.provenance)) - } - - pub fn room_keys(&self) -> Vec { - self.rooms.keys().copied().collect() - } - - pub fn contains(&self, room: &RoomKey) -> bool { - self.rooms.contains_key(room) - } - - pub fn provenance(&self, room: &RoomKey) -> Option<&RoomProvenance> { - self.rooms.get(room).map(|entry| &entry.provenance) - } - - pub fn known_only_to(&self, source: RoomSource) -> Vec { - self.rooms - .iter() - .filter(|(_, entry)| entry.provenance.only(source)) - .map(|(room, _)| *room) - .collect() - } - - pub fn gateway_only_rooms(&self) -> Vec { - self.rooms - .iter() - .filter(|(_, entry)| entry.provenance.is_gateway_only()) - .map(|(room, _)| *room) - .collect() - } - - pub fn media_only_rooms(&self) -> Vec { - self.rooms - .iter() - .filter(|(_, entry)| entry.provenance.is_media_only()) - .map(|(room, _)| *room) - .collect() - } - - pub fn listed_completely_at(&self, location: &Location) -> Option { - self.listings.get(location).copied() - } - - pub fn room_is_absent_from(&self, location: &Location, room: &RoomKey) -> bool { - self.listings.contains_key(location) - && !self - .rooms - .get(room) - .is_some_and(|entry| entry.locations.contains_key(location)) - } - - pub fn forget_location(&mut self, location: &Location) { - self.listings.remove(location); - } - - pub fn locations_for(&self, room: &RoomKey) -> Vec { - self.rooms - .get(room) - .map(|entry| entry.locations.keys().cloned().collect()) - .unwrap_or_default() - } - - pub fn pinned_for(&self, room: &RoomKey) -> Option { - self.rooms.get(room).and_then(|entry| entry.pinned.clone()) - } - - pub fn first_seen_at(&self, room: &RoomKey) -> Option { - self.rooms.get(room).map(|entry| entry.first_at) - } - - pub fn last_seen_at(&self, room: &RoomKey) -> Option { - self.rooms.get(room).map(|entry| entry.last_at) - } - - pub fn note(&mut self, room: RoomKey, source: RoomSource, now: Millis) -> Admission { - if let Some(entry) = self.rooms.get_mut(&room) { - entry.provenance.note(source, now); - entry.last_at = now; - return Admission::Refreshed; - } - if self.rooms.len() >= self.limits.max_rooms { - self.refused_total = self.refused_total.saturating_add(1); - return Admission::Refused; - } - let mut provenance = RoomProvenance::new(); - provenance.note(source, now); - self.rooms.insert( - room, - RoomEntry { - provenance, - locations: BTreeMap::new(), - pinned: None, - first_at: now, - last_at: now, - }, - ); - self.admitted_total = self.admitted_total.saturating_add(1); - Admission::Admitted - } - - pub fn note_census(&mut self, snapshot: &CensusSnapshot, now: Millis) -> Vec { - snapshot - .room_keys() - .into_iter() - .filter(|room| self.note(*room, RoomSource::GatewayCensus, now) == Admission::Admitted) - .collect() - } - - pub fn drain_relocated(&mut self) -> Vec { - let mut moved = std::mem::take(&mut self.relocated); - moved.sort_unstable(); - moved.dedup(); - moved - } - - pub fn note_pinned(&mut self, room: RoomKey, location: Location, now: Millis) -> Admission { - let admission = self.note(room, RoomSource::PinnedServer, now); - if admission.is_tracked() - && let Some(entry) = self.rooms.get_mut(&room) - { - entry.locations.insert(location.clone(), now); - entry.pinned = Some(location); - } - admission - } - - pub fn note_ledger(&mut self, room: RoomKey, now: Millis) -> Admission { - self.note(room, RoomSource::Ledger, now) - } - - pub fn note_suspicion(&mut self, room: RoomKey, now: Millis) -> Admission { - self.note(room, RoomSource::Suspicion, now) - } - - pub fn apply_room_list( - &mut self, - location: &Location, - list: &ServerRoomList, - now: Millis, - ) -> RoomListEffect { - let (rooms, unparseable) = match list { - ServerRoomList::Unreadable(fault) => { - self.unreadable_lists_total = self.unreadable_lists_total.saturating_add(1); - self.listings.remove(location); - return RoomListEffect::Unreadable(*fault); - } - ServerRoomList::Listed { rooms, unparseable } => (rooms, *unparseable), - }; - - let listed: BTreeSet = rooms.iter().copied().collect(); - let mut added = 0usize; - for room in &listed { - if self.note(*room, RoomSource::MediaRoomList, now) == Admission::Admitted { - added = added.saturating_add(1); - } - if let Some(entry) = self.rooms.get_mut(room) { - let elsewhere = - !entry.locations.is_empty() && !entry.locations.contains_key(location); - entry.locations.insert(location.clone(), now); - if elsewhere { - self.relocated.push(*room); - } - } - } - - if unparseable > 0 { - self.incomplete_lists_total = self.incomplete_lists_total.saturating_add(1); - self.listings.remove(location); - return RoomListEffect::AdditiveOnly { - listed: listed.len(), - added, - unparseable, - }; - } - - let mut withdrawn = 0usize; - for (room, entry) in &mut self.rooms { - if listed.contains(room) { - continue; - } - if entry.pinned.as_ref() == Some(location) { - continue; - } - if entry.locations.remove(location).is_some() { - withdrawn = withdrawn.saturating_add(1); - } - } - self.withdrawn_total = self.withdrawn_total.saturating_add(withdrawn as u64); - self.listings.insert(location.clone(), now); - - RoomListEffect::Applied { - listed: listed.len(), - added, - withdrawn, - } - } - - pub fn apply_pass( - &mut self, - results: &[(Location, ServerRoomList)], - now: Millis, - ) -> DiscoveryPass { - let mut pass = DiscoveryPass::empty(now); - pass.servers = results.len(); - - for (location, list) in results { - match self.apply_room_list(location, list, now) { - RoomListEffect::Unreadable(fault) => { - pass.unreadable.push((location.clone(), fault)); - } - RoomListEffect::AdditiveOnly { added, .. } => { - pass.readable = pass.readable.saturating_add(1); - pass.rooms_added = pass.rooms_added.saturating_add(added); - } - RoomListEffect::Applied { - added, withdrawn, .. - } => { - pass.readable = pass.readable.saturating_add(1); - pass.complete = pass.complete.saturating_add(1); - pass.rooms_added = pass.rooms_added.saturating_add(added); - pass.locations_withdrawn = pass.locations_withdrawn.saturating_add(withdrawn); - } - } - } - - pass - } - - pub fn sources_for( - &self, - room: &RoomKey, - gateway_hints: Vec, - ledger_last_known: Vec, - ) -> CandidateSources { - CandidateSources { - discovered: self.locations_for(room), - gateway_hints, - pinned: self.pinned_for(room), - ledger_last_known, - } - } - - pub fn forget(&mut self, room: &RoomKey) -> bool { - let removed = self.rooms.remove(room).is_some(); - if removed { - self.forgotten_total = self.forgotten_total.saturating_add(1); - } - removed - } - - pub fn retain(&mut self, mut keep: F) - where - F: FnMut(&RoomKey, &RoomProvenance) -> bool, - { - let dropped: Vec = self - .rooms - .iter() - .filter(|(room, entry)| !keep(room, &entry.provenance)) - .map(|(room, _)| *room) - .collect(); - for room in dropped { - self.forget(&room); - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ReadClock { - pub now: Millis, - pub wall_now: WallMillis, -} - -impl ReadClock { - pub fn from_clock(clock: &dyn Clock) -> Self { - Self { - now: clock.now(), - wall_now: clock.wall_now(), - } - } - - pub const fn join_instant(&self, joined_at_unix_seconds: u64) -> Millis { - if joined_at_unix_seconds == 0 { - return self.now; - } - let joined_wall = WallMillis::new(joined_at_unix_seconds.saturating_mul(1_000)); - let age_ms = self.wall_now.saturating_since(joined_wall); - Millis::new(self.now.get().saturating_sub(age_ms)) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct RoomRoster { - participants: Vec, - unparseable_identities: usize, -} - -impl RoomRoster { - pub fn participants(&self) -> &[MediaParticipant] { - &self.participants - } - - pub const fn unparseable_identities(&self) -> usize { - self.unparseable_identities - } - - pub fn len(&self) -> usize { - self.participants.len() - } - - pub fn is_empty(&self) -> bool { - self.participants.is_empty() - } - - pub fn connections(&self) -> Vec { - let mut connections: Vec = self - .participants - .iter() - .map(|participant| participant.connection.clone()) - .collect(); - connections.sort(); - connections.dedup(); - connections - } - - fn digest(&self) -> u64 { - let mut digest = Digest::new().number(self.unparseable_identities as u64); - for participant in &self.participants { - digest = digest - .text(participant.connection.as_str()) - .number(participant.user_id.get()) - .number(participant.joined_at.get()) - .flag(participant.is_publisher); - } - digest.finish() - } -} - -pub fn roster_from_records( - records: &[ParticipantRecord], - location: &Location, - clock: ReadClock, -) -> RoomRoster { - let mut participants: Vec = Vec::new(); - let mut unparseable_identities = 0usize; - - for record in records { - match parse_participant_identity(&record.identity) { - Err(_) => unparseable_identities = unparseable_identities.saturating_add(1), - Ok(identity) => participants.push(MediaParticipant { - connection: identity.connection_id, - user_id: identity.user_id, - location: location.clone(), - joined_at: clock.join_instant(record.joined_at_unix_seconds), - is_publisher: record.is_publisher, - }), - } - } - - RoomRoster { - participants, - unparseable_identities, - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LocationReadout { - location: Location, - roster: ReadResult, - at: Millis, -} - -impl LocationReadout { - pub const fn read(location: Location, roster: RoomRoster, at: Millis) -> Self { - Self { - location, - roster: ReadResult::Read(roster), - at, - } - } - - pub const fn unreadable(location: Location, fault: LiveKitFault, at: Millis) -> Self { - Self { - location, - roster: ReadResult::Unreadable(fault), - at, - } - } - - pub const fn location(&self) -> &Location { - &self.location - } - - pub const fn at(&self) -> Millis { - self.at - } - - pub fn is_readable(&self) -> bool { - self.roster.is_readable() - } - - pub fn roster(&self) -> Option<&RoomRoster> { - self.roster.as_read() - } - - pub fn fault(&self) -> Option { - self.roster.fault() - } - - pub fn participants_seen(&self) -> u32 { - self.roster - .as_read() - .map_or(0, |roster| u32::try_from(roster.len()).unwrap_or(u32::MAX)) - } - - pub fn outcome(&self) -> ProbeOutcome { - match &self.roster { - ReadResult::Read(_) => ProbeOutcome::Success, - ReadResult::Unreadable(fault) if fault.is_auth_failure() => ProbeOutcome::AuthFailure, - ReadResult::Unreadable(_) => ProbeOutcome::Failure, - } - } - - pub fn probe(&self, health: ServerHealth, cliffed: bool) -> LocationProbe { - let result = match &self.roster { - ReadResult::Unreadable(fault) => ProbeResult::Failed(*fault), - ReadResult::Read(roster) => ProbeResult::Ok { - participants: roster.participants.clone(), - unparseable_identities: roster.unparseable_identities, - }, - }; - LocationProbe { - location: self.location.clone(), - result, - health, - cliffed, - } - } - - fn digest(&self) -> u64 { - let digest = Digest::new() - .text(self.location.region.as_str()) - .text(self.location.server.as_str()); - match &self.roster { - ReadResult::Unreadable(fault) => digest.text("unreadable").text(fault.label()).finish(), - ReadResult::Read(roster) => digest.text("read").number(roster.digest()).finish(), - } - } -} - -pub fn record_health(readouts: &[LocationReadout], health: &mut ServerHealthMap, now: Millis) { - for readout in readouts { - health.record(&readout.location, readout.outcome(), now); - } -} - -fn digest_gateway_read(read: &GatewayRead) -> u64 { - match read { - GatewayRead::Failed(fault) => Digest::new().text("failed").text(fault.label()).finish(), - GatewayRead::Ok { states } => { - let mut digest = Digest::new().text("ok"); - for state in states { - digest = digest - .text(state.connection.as_ref().map_or("", ConnectionId::as_str)) - .flag(state.connection.is_some()) - .number(state.user_id.get()) - .number(state.channel_id.get()); - } - digest.finish() - } - } -} - -fn digest_pending_joins(joins: &[PendingJoin]) -> u64 { - let mut digest = Digest::new().text("pending_joins"); - for join in joins { - digest = digest - .text(join.connection.as_str()) - .number(join.user_id.get()) - .number(join.expires_at.get()); - } - digest.finish() -} - -pub async fn list_server_rooms(livekit: &L, location: &Location) -> ServerRoomList -where - L: LiveKitApi, -{ - match livekit.list_rooms(location).await { - ReadResult::Unreadable(fault) => ServerRoomList::Unreadable(fault), - ReadResult::Read(names) => { - let mut rooms: Vec = Vec::new(); - let mut unparseable = 0usize; - for name in &names { - match parse_room_name(name) { - Err(_) => unparseable = unparseable.saturating_add(1), - Ok(room) => rooms.push(room), - } - } - rooms.sort(); - rooms.dedup(); - ServerRoomList::Listed { rooms, unparseable } - } - } -} - -pub async fn list_fleet_rooms( - livekit: &L, - locations: &[Location], - mut admit: F, -) -> Vec<(Location, ServerRoomList)> -where - L: LiveKitApi, - F: FnMut(&Location) -> bool, -{ - let mut results: Vec<(Location, ServerRoomList)> = Vec::new(); - for location in locations { - if !admit(location) { - continue; - } - let list = list_server_rooms(livekit, location).await; - results.push((location.clone(), list)); - } - results -} - -pub async fn read_location_roster( - livekit: &L, - location: &Location, - room: RoomKey, - clock: ReadClock, -) -> LocationReadout -where - L: LiveKitApi, -{ - match livekit.list_participants(location, room).await { - ReadResult::Unreadable(fault) => { - LocationReadout::unreadable(location.clone(), fault, clock.now) - } - ReadResult::Read(records) => LocationReadout::read( - location.clone(), - roster_from_records(&records, location, clock), - clock.now, - ), - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct CliffVerdict { - pub fraction: bool, - pub disputed: bool, -} - -impl CliffVerdict { - pub const fn is_cliffed(self) -> bool { - self.fraction || self.disputed - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ServerPass { - pub location: Location, - pub participants_seen: u32, - pub expected_participants: u32, - pub expectations_accounted: bool, - pub coverage_complete: bool, -} - -impl ServerPass { - pub const fn new( - location: Location, - participants_seen: u32, - expected_participants: u32, - ) -> Self { - Self { - location, - participants_seen, - expected_participants, - expectations_accounted: false, - coverage_complete: true, - } - } - - #[must_use] - pub const fn accounted(mut self) -> Self { - self.expectations_accounted = true; - self - } - - pub const fn says_nobody_is_expected(&self) -> bool { - self.expectations_accounted && self.expected_participants == 0 - } - - #[must_use] - pub const fn partially_covered(mut self) -> Self { - self.coverage_complete = false; - self - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ExpectationSource { - View, - Ledger, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct PassCounters { - seen: BTreeMap, - expected: BTreeMap, - seen_by_room: BTreeMap<(Location, RoomKey), u32>, - expected_by_room: BTreeMap<(Location, RoomKey, ExpectationSource), u32>, - expectations_accounted: bool, - skipped: BTreeSet, -} - -impl PassCounters { - pub const fn new() -> Self { - Self { - seen: BTreeMap::new(), - expected: BTreeMap::new(), - seen_by_room: BTreeMap::new(), - expected_by_room: BTreeMap::new(), - expectations_accounted: false, - skipped: BTreeSet::new(), - } - } - - pub const fn note_expectations_accounted(&mut self) { - self.expectations_accounted = true; - } - - pub const fn expectations_accounted(&self) -> bool { - self.expectations_accounted - } - - pub fn note_skipped(&mut self, location: &Location) { - self.skipped.insert(location.clone()); - } - - pub fn is_fully_covered(&self, location: &Location) -> bool { - !self.skipped.contains(location) - } - - pub fn note_seen(&mut self, location: &Location, count: u32) { - let entry = self.seen.entry(location.clone()).or_insert(0); - *entry = entry.saturating_add(count); - } - - pub fn note_expected(&mut self, location: &Location, count: u32) { - let entry = self.expected.entry(location.clone()).or_insert(0); - *entry = entry.saturating_add(count); - } - - pub fn note_room_seen(&mut self, location: &Location, room: RoomKey, count: u32) { - let previous = self - .seen_by_room - .insert((location.clone(), room), count) - .unwrap_or(0); - let entry = self.seen.entry(location.clone()).or_insert(0); - *entry = entry.saturating_sub(previous).saturating_add(count); - } - - pub fn note_room_expected( - &mut self, - location: &Location, - room: RoomKey, - source: ExpectationSource, - count: u32, - ) { - let previous = self - .expected_by_room - .insert((location.clone(), room, source), count) - .unwrap_or(0); - let entry = self.expected.entry(location.clone()).or_insert(0); - *entry = entry.saturating_sub(previous).saturating_add(count); - } - - pub fn note_view(&mut self, view: &RoomView) { - let room = view.observation().room; - let readable = view.readable_locations(); - for readout in view.readouts() { - if !readout.is_readable() { - continue; - } - self.note_room_seen(readout.location(), room, readout.participants_seen()); - } - for location in view.observation().candidates.locations() { - if !readable.contains(location) { - self.note_skipped(location); - } - } - for (location, expected) in view.expectations() { - self.note_room_expected(location, room, ExpectationSource::View, *expected); - } - } - - pub fn seen(&self, location: &Location) -> u32 { - self.seen.get(location).copied().unwrap_or(0) - } - - pub fn expected(&self, location: &Location) -> u32 { - self.expected.get(location).copied().unwrap_or(0) - } - - pub fn locations(&self) -> Vec { - let mut locations: Vec = self - .seen - .keys() - .chain(self.expected.keys()) - .cloned() - .collect(); - locations.sort(); - locations.dedup(); - locations - } - - pub fn passes(&self) -> Vec { - self.seen - .iter() - .map(|(location, participants_seen)| ServerPass { - location: location.clone(), - participants_seen: *participants_seen, - expected_participants: self.expected(location), - expectations_accounted: self.expectations_accounted, - coverage_complete: self.is_fully_covered(location), - }) - .collect() - } - - pub fn is_empty(&self) -> bool { - self.seen.is_empty() && self.expected.is_empty() - } - - pub fn clear(&mut self) { - self.seen.clear(); - self.expected.clear(); - self.seen_by_room.clear(); - self.expected_by_room.clear(); - self.expectations_accounted = false; - self.skipped.clear(); - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct Watch { - detector: ServerCliffDetector, - baseline: u32, - last_nonempty_at: Option, - dispute_since: Option, - expected_at_dispute: u32, -} - -#[derive(Clone, Debug)] -pub struct ServerCliffWatch { - config: ServerCliffConfig, - watches: BTreeMap, - tripped_total: u64, - disputed_total: u64, - released_total: u64, -} - -impl ServerCliffWatch { - pub const fn new(config: ServerCliffConfig) -> Self { - Self { - config, - watches: BTreeMap::new(), - tripped_total: 0, - disputed_total: 0, - released_total: 0, - } - } - - pub fn from_config(config: &ReconConfig) -> Self { - Self::new(ServerCliffConfig { - fraction: config.server_cliff_fraction, - hold_ms: config.server_cliff_hold_ms, - }) - } - - pub const fn config(&self) -> ServerCliffConfig { - self.config - } - - pub const fn tripped_total(&self) -> u64 { - self.tripped_total - } - - pub const fn disputed_total(&self) -> u64 { - self.disputed_total - } - - pub const fn released_total(&self) -> u64 { - self.released_total - } - - pub fn tracked(&self) -> usize { - self.watches.len() - } - - pub fn observe(&mut self, pass: &ServerPass, now: Millis) -> CliffVerdict { - let config = self.config; - let watch = self.watches.entry(pass.location.clone()).or_default(); - - let fraction = if pass.coverage_complete { - watch.detector.observe(pass.participants_seen, now, config) - } else { - watch.detector.is_cliffed(now, config) - }; - let mut armed = false; - let mut released = false; - - if pass.participants_seen > 0 { - watch.baseline = pass.participants_seen; - watch.last_nonempty_at = Some(now); - watch.expected_at_dispute = 0; - released = watch.dispute_since.take().is_some(); - } else if watch.baseline > 0 || pass.expected_participants > 0 { - if watch.dispute_since.is_none() { - watch.dispute_since = Some(now); - armed = true; - } - watch.expected_at_dispute = pass.expected_participants; - if let Some(since) = watch.dispute_since - && pass.says_nobody_is_expected() - && now.saturating_since(since) >= config.hold_ms - { - watch.dispute_since = None; - watch.expected_at_dispute = 0; - released = true; - } - } - - if pass.coverage_complete - && pass.says_nobody_is_expected() - && let Some(since) = watch.detector.cliffed_since() - && now.saturating_since(since) >= config.hold_ms - { - watch.detector.release(); - } - - let disputed = watch.dispute_since.is_some(); - if armed { - self.disputed_total = self.disputed_total.saturating_add(1); - } - if released { - self.released_total = self.released_total.saturating_add(1); - } - if fraction { - self.tripped_total = self.tripped_total.saturating_add(1); - } - - CliffVerdict { fraction, disputed } - } - - pub fn observe_pass(&mut self, counters: &PassCounters, now: Millis) -> Vec { - counters - .passes() - .into_iter() - .filter_map(|pass| { - let verdict = self.observe(&pass, now); - verdict.is_cliffed().then_some(pass.location) - }) - .collect() - } - - pub fn is_cliffed(&self, location: &Location, now: Millis) -> bool { - self.watches.get(location).is_some_and(|watch| { - watch.dispute_since.is_some() || watch.detector.is_cliffed(now, self.config) - }) - } - - pub fn cliffed_since(&self, location: &Location) -> Option { - let watch = self.watches.get(location)?; - match (watch.dispute_since, watch.detector.cliffed_since()) { - (Some(dispute), Some(fraction)) => Some(dispute.min(fraction)), - (Some(dispute), None) => Some(dispute), - (None, fraction) => fraction, - } - } - - pub fn is_disputed(&self, location: &Location) -> bool { - self.watches - .get(location) - .is_some_and(|watch| watch.dispute_since.is_some()) - } - - pub fn disputed(&self) -> Vec { - self.watches - .iter() - .filter(|(_, watch)| watch.dispute_since.is_some()) - .map(|(location, _)| location.clone()) - .collect() - } - - pub fn cliffed(&self, now: Millis) -> Vec { - self.watches - .keys() - .filter(|location| self.is_cliffed(location, now)) - .cloned() - .collect() - } - - pub fn forget(&mut self, location: &Location) -> bool { - self.watches.remove(location).is_some() - } - - pub fn retain(&mut self, mut keep: F) - where - F: FnMut(&Location) -> bool, - { - self.watches.retain(|location, _| keep(location)); - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomTurnReads { - pub gateway: GatewayRead, - pub pending_joins: Vec, - pub pending_joins_read: bool, - pub gateway_calls: u32, - pub readouts: Vec, -} - -impl RoomTurnReads { - pub fn gateway_connections(&self) -> Vec { - let mut connections: Vec = match &self.gateway { - GatewayRead::Failed(_) => Vec::new(), - GatewayRead::Ok { states } => states - .iter() - .filter_map(|state| state.connection.clone()) - .collect(), - }; - connections.sort(); - connections.dedup(); - connections - } - - pub fn media_connections(&self) -> Vec { - let mut connections: Vec = self - .readouts - .iter() - .filter_map(LocationReadout::roster) - .flat_map(RoomRoster::connections) - .collect(); - connections.sort(); - connections.dedup(); - connections - } - - pub fn gateway_states(&self) -> &[GatewayVoiceState] { - match &self.gateway { - GatewayRead::Ok { states } => states, - GatewayRead::Failed(_) => &[], - } - } - - pub fn media_only_connections(&self) -> Vec { - let gateway = self.gateway_connections(); - self.media_connections() - .into_iter() - .filter(|connection| !gateway.contains(connection)) - .collect() - } -} - -#[derive(Clone, Copy, Debug)] -pub struct RoomContext<'a> { - pub room: RoomKey, - pub turn: TurnId, - pub at: Millis, - pub wall_at: WallMillis, - pub candidates: &'a CandidateSet, - pub health: &'a ServerHealthMap, - pub cliffs: &'a ServerCliffWatch, - pub holed: &'a [Location], - pub believed_homes: &'a [Location], - pub room_cliffed: bool, - pub census: CensusCrossCheck, - pub topology: TopologyFreshness, - pub census_epoch: Epoch, - pub topology_epoch: Epoch, - pub max_connections_per_room: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomView { - observation: RoomObservation, - gateway: GatewayRead, - readouts: Vec, - probes: Vec, - expectations: BTreeMap, - pending_joins_read: bool, -} - -impl RoomView { - pub const fn observation(&self) -> &RoomObservation { - &self.observation - } - - pub const fn room(&self) -> RoomKey { - self.observation.room - } - - pub const fn turn(&self) -> TurnId { - self.observation.turn - } - - pub const fn at(&self) -> Millis { - self.observation.at - } - - pub const fn authority(&self) -> &Side { - &self.observation.authority - } - - pub const fn gateway_read(&self) -> &GatewayRead { - &self.gateway - } - - pub const fn gateway_readable(&self) -> bool { - matches!(self.gateway, GatewayRead::Ok { .. }) - } - - pub const fn pending_joins_read(&self) -> bool { - self.pending_joins_read - } - - pub fn readouts(&self) -> &[LocationReadout] { - &self.readouts - } - - pub fn probes(&self) -> &[LocationProbe] { - &self.probes - } - - pub const fn expectations(&self) -> &BTreeMap { - &self.expectations - } - - pub fn readable_locations(&self) -> Vec { - self.readouts - .iter() - .filter(|readout| readout.is_readable()) - .map(|readout| readout.location().clone()) - .collect() - } - - pub fn unreadable_locations(&self) -> Vec<(Location, LiveKitFault)> { - self.readouts - .iter() - .filter_map(|readout| { - readout - .fault() - .map(|fault| (readout.location().clone(), fault)) - }) - .collect() - } - - pub fn unprobed_candidates(&self) -> Vec { - self.observation - .candidates - .locations() - .iter() - .filter(|location| { - !self - .readouts - .iter() - .any(|readout| readout.location() == *location) - }) - .cloned() - .collect() - } - - pub fn participants_seen(&self) -> u32 { - self.readouts - .iter() - .map(LocationReadout::participants_seen) - .fold(0u32, u32::saturating_add) - } - - pub fn locations_seen(&self) -> Vec { - let mut locations: Vec = self - .readouts - .iter() - .filter(|readout| readout.participants_seen() > 0) - .map(|readout| readout.location().clone()) - .collect(); - locations.sort(); - locations.dedup(); - locations - } -} - -fn expectations_from(reads: &RoomTurnReads) -> BTreeMap { - let media = reads.media_connections(); - let mut expectations: BTreeMap = BTreeMap::new(); - - for state in reads.gateway_states() { - let Some(connection) = state.connection.as_ref() else { - continue; - }; - if media.contains(connection) { - continue; - } - let Some(hint) = state.hint.as_ref() else { - continue; - }; - let entry = expectations.entry(hint.clone()).or_insert(0); - *entry = entry.saturating_add(1); - } - - expectations -} - -pub fn observe_room(reads: RoomTurnReads, context: &RoomContext<'_>) -> RoomView { - let probes: Vec = reads - .readouts - .iter() - .map(|readout| { - let location = readout.location(); - let cliffed = - context.cliffs.is_cliffed(location, context.at) || context.holed.contains(location); - readout.probe(context.health.health(location), cliffed) - }) - .collect(); - - let mut connections = reads.gateway_connections(); - connections.extend(reads.media_connections()); - connections.sort(); - connections.dedup(); - - let gateway_side = gateway_authority(&GatewayAuthorityInput { - read: &reads.gateway, - room_cliffed: context.room_cliffed, - census: context.census, - topology: context.topology, - }); - let media_side = match unread_believed_home(context.believed_homes, context.candidates, &probes) - { - Some(_) => SideAuthority::Unknown(UnknownReason::MediaLocationErrored( - LiveKitFault::ServerMissing, - )), - None => media_authority(&MediaAuthorityInput { - candidates: context.candidates, - probes: &probes, - connections_in_room: connections.len(), - max_connections_per_room: context.max_connections_per_room, - }), - }; - - let observation = RoomObservation::assemble( - &RoomReads { - room: context.room, - turn: context.turn, - at: context.at, - wall_at: context.wall_at, - gateway: &reads.gateway, - probes: &probes, - pending_joins: reads.pending_joins.clone(), - candidates: context.candidates.clone(), - census_epoch: context.census_epoch, - topology_epoch: context.topology_epoch, - }, - Side::new(gateway_side, media_side), - ); - - RoomView { - expectations: expectations_from(&reads), - observation, - gateway: reads.gateway, - readouts: reads.readouts, - probes, - pending_joins_read: reads.pending_joins_read, - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct RoomReadPlan<'a> { - pub room: RoomKey, - pub locations: &'a [Location], -} - -pub async fn read_room( - gateway: &G, - livekit: &L, - plan: &RoomReadPlan<'_>, - clock: &dyn Clock, - mut admit: F, - token: &mut TurnToken, -) -> Result, TurnError> -where - G: GatewayApi, - L: LiveKitApi, - F: FnMut(&Location) -> bool, -{ - let states = gateway.voice_states_for_channel(plan.room).await; - let gateway_read = match states { - Ok(states) => GatewayRead::Ok { states }, - Err(fault) => GatewayRead::Failed(fault), - }; - let fresh_gateway = token.read( - ReadSource::GatewayVoiceStates, - clock.now(), - digest_gateway_read(&gateway_read), - gateway_read, - ); - - let mut readouts: Vec = Vec::new(); - for location in plan.locations { - if !admit(location) { - continue; - } - let readout = - read_location_roster(livekit, location, plan.room, ReadClock::from_clock(clock)).await; - readouts.push(readout); - } - - let mut fresh_readouts = match readouts.first() { - None => token.read( - ReadSource::MediaParticipants, - clock.now(), - Digest::new().text("no_locations_read").finish(), - Vec::new(), - ), - Some(first) => token.read( - ReadSource::MediaParticipants, - first.at(), - first.digest(), - vec![first.clone()], - ), - }; - for readout in readouts.iter().skip(1) { - let next = token.read( - ReadSource::MediaParticipants, - readout.at(), - readout.digest(), - readout.clone(), - ); - fresh_readouts = fresh_readouts.zip(next)?.map(|(mut all, one)| { - all.push(one); - all - }); - } - - let reads = fresh_gateway - .zip(fresh_readouts)? - .map(|(gateway, readouts)| RoomTurnReads { - gateway, - pending_joins: Vec::new(), - pending_joins_read: false, - gateway_calls: 1, - readouts, - }); - - let gateway_readable = matches!(reads.peek().gateway, GatewayRead::Ok { .. }); - if !gateway_readable || reads.peek().media_only_connections().is_empty() { - return Ok(reads); - } - - let joins = gateway.pending_joins_for_channel(plan.room).await; - let at = clock.now(); - match joins { - Ok(joins) => { - let fresh_joins = token.read( - ReadSource::GatewayPendingJoins, - at, - digest_pending_joins(&joins), - joins, - ); - Ok(reads.zip(fresh_joins)?.map(|(mut reads, joins)| { - reads.pending_joins = joins; - reads.pending_joins_read = true; - reads.gateway_calls = 2; - reads - })) - } - Err(fault) => { - let fresh_fault = token.read( - ReadSource::GatewayPendingJoins, - at, - Digest::new().text("failed").text(fault.label()).finish(), - fault, - ); - Ok(reads.zip(fresh_fault)?.map(|(mut reads, fault)| { - reads.gateway = GatewayRead::Failed(fault); - reads.pending_joins = Vec::new(); - reads.pending_joins_read = false; - reads.gateway_calls = 2; - reads - })) - } - } -} diff --git a/fluxer_recon/src/evidence.rs b/fluxer_recon/src/evidence.rs deleted file mode 100644 index d1c38b345..000000000 --- a/fluxer_recon/src/evidence.rs +++ /dev/null @@ -1,758 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::gateway::{GatewayFault, Nonce}; -use crate::health::ServerHealth; -use crate::ids::{ - ChannelId, ConnectionId, Epoch, GuildId, Location, Millis, RoomKey, TurnId, UserId, WallMillis, -}; -use crate::livekit::LiveKitFault; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SideKind { - Gateway, - Media, -} - -impl SideKind { - pub const ALL: [Self; 2] = [Self::Gateway, Self::Media]; - - pub const fn label(self) -> &'static str { - match self { - Self::Gateway => "gateway", - Self::Media => "media", - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub struct Side { - pub gateway: T, - pub media: T, -} - -impl Side { - pub const fn new(gateway: T, media: T) -> Self { - Self { gateway, media } - } - - pub const fn get(&self, kind: SideKind) -> &T { - match kind { - SideKind::Gateway => &self.gateway, - SideKind::Media => &self.media, - } - } - - pub fn map U>(&self, f: F) -> Side { - Side { - gateway: f(SideKind::Gateway, &self.gateway), - media: f(SideKind::Media, &self.media), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum UnknownReason { - GatewayReadFailed(GatewayFault), - GatewayCliff, - CensusDisagrees, - MediaLocationErrored(LiveKitFault), - MediaServerUnhealthy, - MediaServerCliff, - NoCandidateLocations, - TopologyStale, - UnparseableIdentityInRoom, - RoomPartiallyUnreadable, -} - -impl UnknownReason { - pub const fn label(&self) -> &'static str { - match self { - Self::GatewayReadFailed(_) => "gateway_read_failed", - Self::GatewayCliff => "gateway_cliff", - Self::CensusDisagrees => "census_disagrees", - Self::MediaLocationErrored(_) => "media_location_errored", - Self::MediaServerUnhealthy => "media_server_unhealthy", - Self::MediaServerCliff => "media_server_cliff", - Self::NoCandidateLocations => "no_candidate_locations", - Self::TopologyStale => "topology_stale", - Self::UnparseableIdentityInRoom => "unparseable_identity_in_room", - Self::RoomPartiallyUnreadable => "room_partially_unreadable", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Presence { - Present, - Absent, - Unknown(UnknownReason), -} - -impl Presence { - pub const fn is_present(&self) -> bool { - matches!(self, Self::Present) - } - - pub const fn is_absent(&self) -> bool { - matches!(self, Self::Absent) - } - - pub const fn is_unknown(&self) -> bool { - matches!(self, Self::Unknown(_)) - } - - pub const fn unknown_reason(&self) -> Option<&UnknownReason> { - match self { - Self::Unknown(reason) => Some(reason), - Self::Present | Self::Absent => None, - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Present => "present", - Self::Absent => "absent", - Self::Unknown(_) => "unknown", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SideAuthority { - Authoritative, - Unknown(UnknownReason), -} - -impl SideAuthority { - pub const fn is_authoritative(&self) -> bool { - matches!(self, Self::Authoritative) - } - - pub const fn unknown_reason(&self) -> Option<&UnknownReason> { - match self { - Self::Authoritative => None, - Self::Unknown(reason) => Some(reason), - } - } - - pub fn project(&self, sighted: bool) -> Presence { - match self { - Self::Unknown(reason) => Presence::Unknown(reason.clone()), - Self::Authoritative => { - if sighted { - Presence::Present - } else { - Presence::Absent - } - } - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct RoomCliffInput { - pub dropped: usize, - pub remaining: usize, -} - -pub const fn room_cliff_trips(input: RoomCliffInput) -> bool { - input.dropped >= 2 || (input.dropped >= 1 && input.remaining == 0) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PriorConnection { - pub connection: ConnectionId, - pub was_consistent: bool, -} - -pub fn room_cliff_input( - prior: &[PriorConnection], - gateway_now: &[ConnectionId], - media_now: &[ConnectionId], -) -> RoomCliffInput { - let dropped = prior - .iter() - .filter(|entry| entry.was_consistent) - .filter(|entry| !gateway_now.contains(&entry.connection)) - .filter(|entry| media_now.contains(&entry.connection)) - .count(); - - RoomCliffInput { - dropped, - remaining: gateway_now.len(), - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct CliffWindow { - since: Option, - last: Option, -} - -impl CliffWindow { - pub const fn clear() -> Self { - Self { - since: None, - last: None, - } - } - - pub const fn since(&self) -> Option { - self.since - } - - pub const fn trip(&mut self, now: Millis) { - if self.since.is_none() { - self.since = Some(now); - } - self.last = Some(now); - } - - pub const fn release(&mut self) { - self.since = None; - self.last = None; - } - - pub const fn is_held(&self, now: Millis, hold_ms: u64) -> bool { - match self.last { - None => false, - Some(last) => now.saturating_since(last) < hold_ms, - } - } - - pub const fn expire(&mut self, now: Millis, hold_ms: u64) { - if !self.is_held(now, hold_ms) { - self.release(); - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct ServerCliffConfig { - pub fraction: f64, - pub hold_ms: u64, -} - -pub const SERVER_CLIFF_TRAILING_PASSES: usize = 5; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct ServerCliffDetector { - trailing: [u32; SERVER_CLIFF_TRAILING_PASSES], - filled: usize, - cursor: usize, - window: CliffWindow, -} - -impl ServerCliffDetector { - pub const fn new() -> Self { - Self { - trailing: [0; SERVER_CLIFF_TRAILING_PASSES], - filled: 0, - cursor: 0, - window: CliffWindow::clear(), - } - } - - pub fn observe( - &mut self, - participants_seen: u32, - now: Millis, - config: ServerCliffConfig, - ) -> bool { - let trailing_max = self.trailing_max(); - let tripped = if trailing_max == 0 { - false - } else { - let drop = - f64::from(trailing_max.saturating_sub(participants_seen)) / f64::from(trailing_max); - drop > config.fraction - }; - - if tripped { - self.window.trip(now); - } else { - self.record(participants_seen); - self.window.expire(now, config.hold_ms); - } - - self.is_cliffed(now, config) - } - - pub const fn release(&mut self) { - self.trailing = [0; SERVER_CLIFF_TRAILING_PASSES]; - self.filled = 0; - self.cursor = 0; - self.window.release(); - } - - fn record(&mut self, participants_seen: u32) { - self.trailing[self.cursor] = participants_seen; - self.cursor = (self.cursor + 1) % SERVER_CLIFF_TRAILING_PASSES; - self.filled = self - .filled - .saturating_add(1) - .min(SERVER_CLIFF_TRAILING_PASSES); - } - - pub const fn is_cliffed(&self, now: Millis, config: ServerCliffConfig) -> bool { - self.window.is_held(now, config.hold_ms) - } - - pub const fn cliffed_since(&self) -> Option { - self.window.since() - } - - fn trailing_max(&self) -> u32 { - self.trailing - .iter() - .take(self.filled) - .copied() - .max() - .unwrap_or(0) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum GatewayRead { - Ok { states: Vec }, - Failed(GatewayFault), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GatewayVoiceState { - pub connection: Option, - pub user_id: UserId, - pub channel_id: ChannelId, - pub guild_id: Option, - pub hint: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PendingJoin { - pub connection: ConnectionId, - pub user_id: UserId, - pub expires_at: WallMillis, - pub nonce: Nonce, -} - -impl PendingJoin { - pub const fn is_unexpired(&self, wall_now: WallMillis, skew_ms: u64) -> bool { - wall_now.get() <= self.expires_at.get().saturating_add(skew_ms) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum CensusCrossCheck { - Fresh { voice_state_count: u32 }, - Stale, - Missing, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TopologyFreshness { - pub age_ms: u64, - pub max_age_ms: u64, -} - -impl TopologyFreshness { - pub const fn is_stale(self) -> bool { - self.age_ms > self.max_age_ms - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GatewayAuthorityInput<'a> { - pub read: &'a GatewayRead, - pub room_cliffed: bool, - pub census: CensusCrossCheck, - pub topology: TopologyFreshness, -} - -pub fn gateway_authority(input: &GatewayAuthorityInput<'_>) -> SideAuthority { - let states = match input.read { - GatewayRead::Failed(fault) => { - return SideAuthority::Unknown(UnknownReason::GatewayReadFailed(fault.clone())); - } - GatewayRead::Ok { states } => states, - }; - - if input.room_cliffed { - return SideAuthority::Unknown(UnknownReason::GatewayCliff); - } - - if let CensusCrossCheck::Fresh { voice_state_count } = input.census - && voice_state_count > 0 - && states.is_empty() - { - return SideAuthority::Unknown(UnknownReason::CensusDisagrees); - } - - if input.topology.is_stale() { - return SideAuthority::Unknown(UnknownReason::TopologyStale); - } - - SideAuthority::Authoritative -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ProbeResult { - Ok { - participants: Vec, - unparseable_identities: usize, - }, - Failed(LiveKitFault), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MediaParticipant { - pub connection: ConnectionId, - pub user_id: UserId, - pub location: Location, - pub joined_at: Millis, - pub is_publisher: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MediaSighting { - pub connection: ConnectionId, - pub user_id: UserId, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LocationProbe { - pub location: Location, - pub result: ProbeResult, - pub health: ServerHealth, - pub cliffed: bool, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CandidateSet { - locations: Vec, -} - -impl CandidateSet { - pub fn from_locations(mut locations: Vec) -> Self { - locations.sort(); - locations.dedup(); - Self { locations } - } - - pub fn locations(&self) -> &[Location] { - &self.locations - } - - pub fn is_empty(&self) -> bool { - self.locations.is_empty() - } - - pub fn len(&self) -> usize { - self.locations.len() - } - - pub fn contains(&self, location: &Location) -> bool { - self.locations.binary_search(location).is_ok() - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CandidateSources { - pub discovered: Vec, - pub gateway_hints: Vec, - pub pinned: Option, - pub ledger_last_known: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ServerRecord { - pub location: Location, - pub removed_at: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TopologyLens<'a> { - pub servers: &'a [ServerRecord], - pub now: Millis, - pub drain_grace_ms: u64, -} - -impl TopologyLens<'_> { - pub fn live_locations(&self) -> Vec { - let mut locations: Vec = self - .servers - .iter() - .filter(|record| !self.is_drained(&record.location)) - .map(|record| record.location.clone()) - .collect(); - locations.sort(); - locations.dedup(); - locations - } - - fn is_drained(&self, location: &Location) -> bool { - self.servers - .iter() - .find(|record| &record.location == location) - .and_then(|record| record.removed_at) - .is_some_and(|removed_at| self.now.saturating_since(removed_at) > self.drain_grace_ms) - } -} - -pub fn candidate_set(sources: &CandidateSources, topology: &TopologyLens<'_>) -> CandidateSet { - let mut locations: Vec = sources - .discovered - .iter() - .chain(sources.gateway_hints.iter()) - .chain(sources.pinned.iter()) - .chain(sources.ledger_last_known.iter()) - .filter(|location| !topology.is_drained(location)) - .cloned() - .collect(); - - locations.sort(); - locations.dedup(); - CandidateSet { locations } -} - -pub fn unread_believed_home<'a>( - homes: &'a [Location], - candidates: &CandidateSet, - probes: &[LocationProbe], -) -> Option<&'a Location> { - homes.iter().find(|home| { - !candidates.contains(home) - || !probes.iter().any(|probe| { - &&probe.location == home && matches!(probe.result, ProbeResult::Ok { .. }) - }) - }) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MediaAuthorityInput<'a> { - pub candidates: &'a CandidateSet, - pub probes: &'a [LocationProbe], - pub connections_in_room: usize, - pub max_connections_per_room: usize, -} - -pub fn media_authority(input: &MediaAuthorityInput<'_>) -> SideAuthority { - if input.candidates.is_empty() { - return SideAuthority::Unknown(UnknownReason::NoCandidateLocations); - } - - for location in input.candidates.locations() { - let Some(probe) = input - .probes - .iter() - .find(|probe| &probe.location == location) - else { - return SideAuthority::Unknown(UnknownReason::MediaLocationErrored( - LiveKitFault::ServerMissing, - )); - }; - - match &probe.result { - ProbeResult::Failed(fault) => { - return SideAuthority::Unknown(UnknownReason::MediaLocationErrored(*fault)); - } - ProbeResult::Ok { - unparseable_identities, - participants: _, - } => { - if *unparseable_identities > 0 { - return SideAuthority::Unknown(UnknownReason::UnparseableIdentityInRoom); - } - } - } - - if !probe.health.is_healthy() { - return SideAuthority::Unknown(UnknownReason::MediaServerUnhealthy); - } - - if probe.cliffed { - return SideAuthority::Unknown(UnknownReason::MediaServerCliff); - } - } - - if input.connections_in_room > input.max_connections_per_room { - return SideAuthority::Unknown(UnknownReason::RoomPartiallyUnreadable); - } - - SideAuthority::Authoritative -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ConnectionObservation { - pub connection: ConnectionId, - pub user_id: UserId, - pub presence: Side, - pub gateway_siblings: Vec, - pub media_locations: Vec, - pub participant_joined_at: Option, - pub gateway_state_has_connection_id: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomObservation { - pub room: RoomKey, - pub turn: TurnId, - pub at: Millis, - pub wall_at: WallMillis, - pub authority: Side, - pub connections: Vec, - pub pending_joins: Vec, - pub candidates: CandidateSet, - pub census_epoch: Epoch, - pub topology_epoch: Epoch, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomReads<'a> { - pub room: RoomKey, - pub turn: TurnId, - pub at: Millis, - pub wall_at: WallMillis, - pub gateway: &'a GatewayRead, - pub probes: &'a [LocationProbe], - pub pending_joins: Vec, - pub candidates: CandidateSet, - pub census_epoch: Epoch, - pub topology_epoch: Epoch, -} - -impl RoomObservation { - pub fn assemble(reads: &RoomReads<'_>, authority: Side) -> Self { - let gateway_states: &[GatewayVoiceState] = match reads.gateway { - GatewayRead::Ok { states } => states, - GatewayRead::Failed(_) => &[], - }; - - let participants: Vec<&MediaParticipant> = reads - .probes - .iter() - .filter_map(|probe| match &probe.result { - ProbeResult::Ok { participants, .. } => Some(participants.iter()), - ProbeResult::Failed(_) => None, - }) - .flatten() - .collect(); - - let mut keys: Vec<(ConnectionId, UserId)> = gateway_states - .iter() - .filter_map(|state| { - state - .connection - .as_ref() - .map(|connection| (connection.clone(), state.user_id)) - }) - .chain( - participants - .iter() - .map(|participant| (participant.connection.clone(), participant.user_id)), - ) - .collect(); - keys.sort(); - keys.dedup(); - - let connections = keys - .into_iter() - .map(|(connection, user_id)| { - let gateway_sighted = gateway_states - .iter() - .any(|state| state.connection.as_ref() == Some(&connection)); - - let media_sightings: Vec<&&MediaParticipant> = participants - .iter() - .filter(|participant| participant.connection == connection) - .collect(); - - let media_sighted = !media_sightings.is_empty(); - - let mut gateway_siblings: Vec = gateway_states - .iter() - .filter(|state| state.user_id == user_id) - .filter_map(|state| state.connection.clone()) - .collect(); - gateway_siblings.sort(); - gateway_siblings.dedup(); - - let mut media_locations: Vec = media_sightings - .iter() - .map(|participant| participant.location.clone()) - .collect(); - media_locations.sort(); - media_locations.dedup(); - - let participant_joined_at = media_sightings - .iter() - .map(|participant| participant.joined_at) - .min(); - - let gateway_state_has_connection_id = !gateway_states - .iter() - .any(|state| state.user_id == user_id && state.connection.is_none()); - - ConnectionObservation { - presence: Side::new( - authority.gateway.project(gateway_sighted), - authority.media.project(media_sighted), - ), - connection, - user_id, - gateway_siblings, - media_locations, - participant_joined_at, - gateway_state_has_connection_id, - } - }) - .collect(); - - Self { - room: reads.room, - turn: reads.turn, - at: reads.at, - wall_at: reads.wall_at, - authority, - connections, - pending_joins: reads.pending_joins.clone(), - candidates: reads.candidates.clone(), - census_epoch: reads.census_epoch, - topology_epoch: reads.topology_epoch, - } - } - - pub fn observation_for(&self, connection: &ConnectionId) -> Option<&ConnectionObservation> { - self.connections - .iter() - .find(|entry| &entry.connection == connection) - } - - pub fn unsighted_presence(&self) -> Side { - Side::new( - self.authority.gateway.project(false), - self.authority.media.project(false), - ) - } - - pub fn pending_join_for( - &self, - connection: &ConnectionId, - wall_now: WallMillis, - skew_ms: u64, - ) -> Option<&PendingJoin> { - self.pending_joins - .iter() - .find(|join| &join.connection == connection && join.is_unexpired(wall_now, skew_ms)) - } - - pub fn user_has_pending_join( - &self, - user_id: UserId, - wall_now: WallMillis, - skew_ms: u64, - ) -> bool { - self.pending_joins - .iter() - .any(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) - } -} diff --git a/fluxer_recon/src/gateway/codes.rs b/fluxer_recon/src/gateway/codes.rs deleted file mode 100644 index 631659385..000000000 --- a/fluxer_recon/src/gateway/codes.rs +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Confidence { - Transient, - Definitive, - Refused, - Unknown, -} - -impl Confidence { - pub const ALL: [Self; 4] = [ - Self::Transient, - Self::Definitive, - Self::Refused, - Self::Unknown, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::Transient => "transient", - Self::Definitive => "definitive", - Self::Refused => "refused", - Self::Unknown => "unknown", - } - } - - pub const fn is_definitive(self) -> bool { - matches!(self, Self::Definitive) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum GatewayErrorCode { - Timeout, - NoResponders, - Overloaded, - GuildNotFound, - VoiceStatesError, - PendingJoinsError, - CallLookupError, - CallStateError, - CallPendingJoinsError, - DisconnectUserError, - ConfirmConnectionError, - ConnectionNotFound, - VoiceStateMismatch, - VoiceInvalidState, - VoiceNotSupported, - EventMutationsPaused, - InvalidParams, -} - -impl GatewayErrorCode { - pub const ALL: [Self; 17] = [ - Self::Timeout, - Self::NoResponders, - Self::Overloaded, - Self::GuildNotFound, - Self::VoiceStatesError, - Self::PendingJoinsError, - Self::CallLookupError, - Self::CallStateError, - Self::CallPendingJoinsError, - Self::DisconnectUserError, - Self::ConfirmConnectionError, - Self::ConnectionNotFound, - Self::VoiceStateMismatch, - Self::VoiceInvalidState, - Self::VoiceNotSupported, - Self::EventMutationsPaused, - Self::InvalidParams, - ]; - - pub const fn wire(self) -> &'static str { - match self { - Self::Timeout => "timeout", - Self::NoResponders => "no_responders", - Self::Overloaded => "overloaded", - Self::GuildNotFound => "guild_not_found", - Self::VoiceStatesError => "voice_states_error", - Self::PendingJoinsError => "pending_joins_error", - Self::CallLookupError => "call_lookup_error", - Self::CallStateError => "call_state_error", - Self::CallPendingJoinsError => "call_pending_joins_error", - Self::DisconnectUserError => "disconnect_user_error", - Self::ConfirmConnectionError => "confirm_connection_error", - Self::ConnectionNotFound => "connection_not_found", - Self::VoiceStateMismatch => "voice_state_mismatch", - Self::VoiceInvalidState => "voice_invalid_state", - Self::VoiceNotSupported => "voice_not_supported", - Self::EventMutationsPaused => "event_mutations_paused", - Self::InvalidParams => "invalid_params", - } - } - - pub const fn confidence(self) -> Confidence { - match self { - Self::Timeout - | Self::NoResponders - | Self::Overloaded - | Self::GuildNotFound - | Self::VoiceStatesError - | Self::PendingJoinsError - | Self::CallLookupError - | Self::CallStateError - | Self::CallPendingJoinsError - | Self::DisconnectUserError - | Self::ConfirmConnectionError => Confidence::Transient, - Self::ConnectionNotFound - | Self::VoiceStateMismatch - | Self::VoiceInvalidState - | Self::VoiceNotSupported => Confidence::Definitive, - Self::EventMutationsPaused | Self::InvalidParams => Confidence::Refused, - } - } -} - -pub const MISSING_ERROR_LABEL: &str = "missing_error"; - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum GatewayError { - Known(GatewayErrorCode), - Unrecognised(Box), - Missing, -} - -impl GatewayError { - pub fn from_wire(value: Option<&str>) -> Self { - match value { - None => Self::Missing, - Some(raw) => Self::from_code(raw), - } - } - - fn from_code(raw: &str) -> Self { - GatewayErrorCode::ALL - .into_iter() - .find(|code| code.wire() == raw) - .map_or_else(|| Self::Unrecognised(Box::from(raw)), Self::Known) - } - - pub const fn confidence(&self) -> Confidence { - match self { - Self::Known(code) => code.confidence(), - Self::Unrecognised(_) | Self::Missing => Confidence::Unknown, - } - } - - pub fn label(&self) -> &str { - match self { - Self::Known(code) => code.wire(), - Self::Unrecognised(raw) => raw, - Self::Missing => MISSING_ERROR_LABEL, - } - } - - pub const fn code(&self) -> Option { - match self { - Self::Known(code) => Some(*code), - Self::Unrecognised(_) | Self::Missing => None, - } - } -} diff --git a/fluxer_recon/src/gateway/mod.rs b/fluxer_recon/src/gateway/mod.rs deleted file mode 100644 index 323f4578f..000000000 --- a/fluxer_recon/src/gateway/mod.rs +++ /dev/null @@ -1,316 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -pub mod codes; -pub mod nats; - -use std::future::Future; -use std::time::Duration; - -use crate::evidence::{GatewayVoiceState, PendingJoin}; -use crate::gateway::codes::{Confidence, GatewayError}; -use crate::ids::{ConnectionId, RoomKey, UserId}; -use crate::ledger::RepairVerdict; - -pub const SUBJECT_PREFIX: &str = "rpc.gateway."; - -pub const MUTATION_DEADLINE: Duration = Duration::from_secs(4); -pub const CHANNEL_READ_DEADLINE: Duration = Duration::from_secs(5); -pub const CENSUS_DEADLINE: Duration = Duration::from_secs(12); - -pub const GATEWAY_GUILD_CALL_BUDGET: Duration = Duration::from_secs(4); -pub const GATEWAY_CENSUS_NODE_RPC_BUDGET: Duration = Duration::from_secs(10); - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum MethodClass { - Read, - Constructive, - Destructive, -} - -impl MethodClass { - pub const fn label(self) -> &'static str { - match self { - Self::Read => "read", - Self::Constructive => "constructive", - Self::Destructive => "destructive", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum GatewayMethod { - ActiveVoiceRooms, - VoiceStatesForChannel, - PendingJoinsForChannel, - ConfirmConnection, - RepairStateFromCache, - DisconnectUserIfInChannel, -} - -impl GatewayMethod { - pub const ALL: [Self; 6] = [ - Self::ActiveVoiceRooms, - Self::VoiceStatesForChannel, - Self::PendingJoinsForChannel, - Self::ConfirmConnection, - Self::RepairStateFromCache, - Self::DisconnectUserIfInChannel, - ]; - - pub const fn name(self) -> &'static str { - match self { - Self::ActiveVoiceRooms => "process.active_voice_rooms", - Self::VoiceStatesForChannel => "voice.get_voice_states_for_channel", - Self::PendingJoinsForChannel => "voice.get_pending_joins_for_channel", - Self::ConfirmConnection => "voice.confirm_connection", - Self::RepairStateFromCache => "voice.repair_state_from_cache", - Self::DisconnectUserIfInChannel => "voice.disconnect_user_if_in_channel", - } - } - - pub const fn deadline(self) -> Duration { - match self { - Self::ActiveVoiceRooms => CENSUS_DEADLINE, - Self::VoiceStatesForChannel | Self::PendingJoinsForChannel => CHANNEL_READ_DEADLINE, - Self::ConfirmConnection - | Self::RepairStateFromCache - | Self::DisconnectUserIfInChannel => MUTATION_DEADLINE, - } - } - - pub const fn class(self) -> MethodClass { - match self { - Self::ActiveVoiceRooms | Self::VoiceStatesForChannel | Self::PendingJoinsForChannel => { - MethodClass::Read - } - Self::ConfirmConnection | Self::RepairStateFromCache => MethodClass::Constructive, - Self::DisconnectUserIfInChannel => MethodClass::Destructive, - } - } - - pub fn subject(self) -> String { - let mut subject = String::with_capacity(SUBJECT_PREFIX.len() + self.name().len()); - subject.push_str(SUBJECT_PREFIX); - subject.push_str(self.name()); - subject - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum FaultOrigin { - CouldNotAsk, - Unintelligible, - GatewaySaidNo, -} - -impl FaultOrigin { - pub const fn label(self) -> &'static str { - match self { - Self::CouldNotAsk => "could_not_ask", - Self::Unintelligible => "unintelligible", - Self::GatewaySaidNo => "gateway_said_no", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum GatewayFault { - Timeout, - NoResponders, - TransportFailed, - DecodeFailed, - NotOk(GatewayError), -} - -impl GatewayFault { - pub const fn label(&self) -> &'static str { - match self { - Self::Timeout => "timeout", - Self::NoResponders => "no_responders", - Self::TransportFailed => "transport_failed", - Self::DecodeFailed => "decode_failed", - Self::NotOk(_) => "not_ok", - } - } - - pub const fn origin(&self) -> FaultOrigin { - match self { - Self::Timeout | Self::NoResponders | Self::TransportFailed => FaultOrigin::CouldNotAsk, - Self::DecodeFailed => FaultOrigin::Unintelligible, - Self::NotOk(_) => FaultOrigin::GatewaySaidNo, - } - } - - pub const fn is_evidence(&self) -> bool { - matches!(self.origin(), FaultOrigin::GatewaySaidNo) - } - - pub const fn confidence(&self) -> Confidence { - match self { - Self::Timeout | Self::NoResponders | Self::TransportFailed => Confidence::Transient, - Self::DecodeFailed => Confidence::Unknown, - Self::NotOk(error) => error.confidence(), - } - } - - pub const fn error(&self) -> Option<&GatewayError> { - match self { - Self::Timeout | Self::NoResponders | Self::TransportFailed | Self::DecodeFailed => None, - Self::NotOk(error) => Some(error), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] -pub enum Nonce { - #[default] - Empty, - Value(Box), -} - -impl Nonce { - pub fn from_wire(value: Option<&str>) -> Self { - match value { - None => Self::Empty, - Some("") => Self::Empty, - Some(raw) => Self::Value(Box::from(raw)), - } - } - - pub fn as_str(&self) -> Option<&str> { - match self { - Self::Empty => None, - Self::Value(value) => Some(value), - } - } - - pub const fn is_empty(&self) -> bool { - matches!(self, Self::Empty) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct ActiveVoiceRoom { - pub room: RoomKey, - pub voice_state_count: u32, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ActiveVoiceRooms { - pub node_count: u32, - pub rooms: Vec, - pub unparseable_rooms: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConfirmOutcome { - Confirmed, - AlreadyConfirmed, - CallNotFound, - Failed(GatewayError), -} - -impl ConfirmOutcome { - pub const fn succeeded(&self) -> bool { - matches!(self, Self::Confirmed | Self::AlreadyConfirmed) - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Confirmed => "confirmed", - Self::AlreadyConfirmed => "already_confirmed", - Self::CallNotFound => "call_not_found", - Self::Failed(_) => "failed", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RepairOutcome { - Repaired, - NoChange, - Failed(GatewayError), -} - -impl RepairOutcome { - pub const fn verdict(&self) -> RepairVerdict { - match self { - Self::Repaired => RepairVerdict::Repaired, - Self::NoChange => RepairVerdict::NoChange, - Self::Failed(error) => match error.confidence() { - Confidence::Definitive => RepairVerdict::NotRepairable, - Confidence::Transient | Confidence::Refused | Confidence::Unknown => { - RepairVerdict::NoChange - } - }, - } - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Repaired => "repaired", - Self::NoChange => "no_change", - Self::Failed(_) => "failed", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum DisconnectOutcome { - Removed, - Ignored { reason: Option> }, - CallNotFound, - Failed(GatewayError), -} - -impl DisconnectOutcome { - pub const fn removed_state(&self) -> bool { - matches!(self, Self::Removed) - } - - pub const fn label(&self) -> &'static str { - match self { - Self::Removed => "removed", - Self::Ignored { .. } => "ignored", - Self::CallNotFound => "call_not_found", - Self::Failed(_) => "failed", - } - } -} - -pub trait GatewayApi { - fn active_voice_rooms( - &self, - ) -> impl Future> + Send; - - fn voice_states_for_channel( - &self, - room: RoomKey, - ) -> impl Future, GatewayFault>> + Send; - - fn pending_joins_for_channel( - &self, - room: RoomKey, - ) -> impl Future, GatewayFault>> + Send; - - fn confirm_connection( - &self, - room: RoomKey, - connection: &ConnectionId, - nonce: &Nonce, - ) -> impl Future> + Send; - - fn repair_state_from_cache( - &self, - room: RoomKey, - user_id: UserId, - connection: &ConnectionId, - ) -> impl Future> + Send; - - fn disconnect_user_if_in_channel( - &self, - room: RoomKey, - user_id: UserId, - connection: Option<&ConnectionId>, - ) -> impl Future> + Send; -} diff --git a/fluxer_recon/src/gateway/nats.rs b/fluxer_recon/src/gateway/nats.rs deleted file mode 100644 index d225d5480..000000000 --- a/fluxer_recon/src/gateway/nats.rs +++ /dev/null @@ -1,468 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use serde::Serialize; -use serde_json::Value; - -use fluxer_svc::transport::Transport; - -use crate::evidence::{GatewayVoiceState, PendingJoin}; -use crate::gateway::codes::GatewayError; -use crate::gateway::{ - ActiveVoiceRoom, ActiveVoiceRooms, ConfirmOutcome, DisconnectOutcome, GatewayApi, GatewayFault, - GatewayMethod, Nonce, RepairOutcome, -}; -use crate::ids::{ - ChannelId, ConnectionId, GuildId, Location, RegionId, RoomKey, ServerId, UserId, WallMillis, -}; - -#[derive(Debug, Serialize)] -struct NoParams {} - -#[derive(Debug, Serialize)] -struct ChannelReadParams { - channel_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - guild_id: Option, -} - -#[derive(Debug, Serialize)] -struct ConfirmParams { - channel_id: String, - connection_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - guild_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - token_nonce: Option, -} - -#[derive(Debug, Serialize)] -struct RepairParams { - channel_id: String, - user_id: String, - connection_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - guild_id: Option, -} - -#[derive(Debug, Serialize)] -struct DisconnectParams { - channel_id: String, - user_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - guild_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - connection_id: Option, -} - -fn guild_param(room: RoomKey) -> Option { - room.guild_id().map(|guild_id| guild_id.to_string()) -} - -fn channel_read_params(room: RoomKey) -> ChannelReadParams { - ChannelReadParams { - channel_id: room.channel_id().to_string(), - guild_id: guild_param(room), - } -} - -fn confirm_params(room: RoomKey, connection: &ConnectionId, nonce: &Nonce) -> ConfirmParams { - ConfirmParams { - channel_id: room.channel_id().to_string(), - connection_id: connection.as_str().to_owned(), - guild_id: guild_param(room), - token_nonce: nonce.as_str().map(str::to_owned), - } -} - -fn repair_params(room: RoomKey, user_id: UserId, connection: &ConnectionId) -> RepairParams { - RepairParams { - channel_id: room.channel_id().to_string(), - user_id: user_id.to_string(), - connection_id: connection.as_str().to_owned(), - guild_id: guild_param(room), - } -} - -fn disconnect_params( - room: RoomKey, - user_id: UserId, - connection: Option<&ConnectionId>, -) -> DisconnectParams { - DisconnectParams { - channel_id: room.channel_id().to_string(), - user_id: user_id.to_string(), - guild_id: guild_param(room), - connection_id: connection.map(|id| id.as_str().to_owned()), - } -} - -pub fn classify_transport_error(error: &anyhow::Error) -> GatewayFault { - if error.chain().any(|cause| { - cause - .to_string() - .to_ascii_lowercase() - .contains("no responders") - }) { - return GatewayFault::NoResponders; - } - if error - .chain() - .any(|cause| cause.is::()) - { - return GatewayFault::Timeout; - } - if error.chain().any(|cause| { - let text = cause.to_string().to_ascii_lowercase(); - text.contains("timed out") || text.contains("timeout") || text.contains("deadline") - }) { - return GatewayFault::Timeout; - } - GatewayFault::TransportFailed -} - -pub fn decode_envelope(bytes: &[u8]) -> Result { - let Ok(body) = serde_json::from_slice::(bytes) else { - return Err(GatewayFault::DecodeFailed); - }; - let Some(object) = body.as_object() else { - return Err(GatewayFault::DecodeFailed); - }; - let Some(ok) = object.get("ok").and_then(Value::as_bool) else { - return Err(GatewayFault::DecodeFailed); - }; - if ok { - return Ok(object.get("result").cloned().unwrap_or(Value::Null)); - } - let error = object.get("error").and_then(Value::as_str); - Err(GatewayFault::NotOk(GatewayError::from_wire(error))) -} - -fn required<'a>(result: &'a Value, key: &str) -> Result<&'a Value, GatewayFault> { - result.get(key).ok_or(GatewayFault::DecodeFailed) -} - -fn optional<'a>(result: &'a Value, key: &str) -> Option<&'a Value> { - match result.get(key) { - None | Some(Value::Null) => None, - Some(value) => Some(value), - } -} - -fn entries(result: &Value, key: &str) -> Result, GatewayFault> { - required(result, key)? - .as_array() - .cloned() - .ok_or(GatewayFault::DecodeFailed) -} - -fn decimal_u64(value: &str) -> Option { - if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { - return None; - } - value.parse::().ok() -} - -fn snowflake(value: &Value) -> Result { - match value { - Value::String(raw) => decimal_u64(raw).ok_or(GatewayFault::DecodeFailed), - Value::Number(number) => number.as_u64().ok_or(GatewayFault::DecodeFailed), - Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => { - Err(GatewayFault::DecodeFailed) - } - } -} - -fn text(value: &Value) -> Result<&str, GatewayFault> { - value.as_str().ok_or(GatewayFault::DecodeFailed) -} - -fn count(value: &Value) -> Result { - let number = value.as_u64().ok_or(GatewayFault::DecodeFailed)?; - u32::try_from(number).map_err(|_| GatewayFault::DecodeFailed) -} - -fn flag(result: &Value, key: &str) -> Result { - match optional(result, key) { - None => Ok(false), - Some(value) => value.as_bool().ok_or(GatewayFault::DecodeFailed), - } -} - -fn reason(result: &Value) -> Result>, GatewayFault> { - match optional(result, "reason") { - None => Ok(None), - Some(value) => Ok(Some(Box::from(text(value)?))), - } -} - -fn error_of(result: &Value) -> GatewayError { - GatewayError::from_wire(optional(result, "error").and_then(Value::as_str)) -} - -fn connection_of(value: &Value) -> Result, GatewayFault> { - let raw = text(value)?; - if raw.is_empty() { - return Ok(None); - } - ConnectionId::new(raw) - .map(Some) - .map_err(|_| GatewayFault::DecodeFailed) -} - -fn location_hint(entry: &Value) -> Result, GatewayFault> { - let region = match optional(entry, "region_id") { - None => None, - Some(value) => Some(RegionId::new(text(value)?).map_err(|_| GatewayFault::DecodeFailed)?), - }; - let server = match optional(entry, "server_id") { - None => None, - Some(value) => Some(ServerId::new(text(value)?).map_err(|_| GatewayFault::DecodeFailed)?), - }; - match (region, server) { - (Some(region), Some(server)) => Ok(Some(Location::new(region, server))), - (Some(_), None) | (None, Some(_)) | (None, None) => Ok(None), - } -} - -fn room_of(entry: &Value) -> Result { - let channel_id = ChannelId::new(snowflake(required(entry, "channel_id")?)?); - match optional(entry, "guild_id") { - None => Ok(RoomKey::Dm { channel_id }), - Some(value) => Ok(RoomKey::Guild { - guild_id: GuildId::new(snowflake(value)?), - channel_id, - }), - } -} - -fn decode_active_voice_room(entry: &Value) -> Result { - Ok(ActiveVoiceRoom { - room: room_of(entry)?, - voice_state_count: count(required(entry, "voice_state_count")?)?, - }) -} - -pub fn decode_active_voice_rooms(result: &Value) -> Result { - let node_count = count(required(result, "node_count")?)?; - let mut rooms = Vec::new(); - let mut unparseable_rooms = 0u32; - - for entry in entries(result, "rooms")? { - if let Ok(room) = decode_active_voice_room(&entry) { - rooms.push(room); - } else { - unparseable_rooms = unparseable_rooms.saturating_add(1); - } - } - - Ok(ActiveVoiceRooms { - node_count, - rooms, - unparseable_rooms, - }) -} - -fn decode_voice_state(entry: &Value) -> Result { - let connection = match optional(entry, "connection_id") { - None => None, - Some(value) => connection_of(value)?, - }; - let guild_id = match optional(entry, "guild_id") { - None => None, - Some(value) => Some(GuildId::new(snowflake(value)?)), - }; - - Ok(GatewayVoiceState { - connection, - user_id: UserId::new(snowflake(required(entry, "user_id")?)?), - channel_id: ChannelId::new(snowflake(required(entry, "channel_id")?)?), - guild_id, - hint: location_hint(entry)?, - }) -} - -pub fn decode_voice_states(result: &Value) -> Result, GatewayFault> { - entries(result, "voice_states")? - .iter() - .map(decode_voice_state) - .collect() -} - -fn decode_pending_join(entry: &Value) -> Result { - let connection = ConnectionId::new(text(required(entry, "connection_id")?)?) - .map_err(|_| GatewayFault::DecodeFailed)?; - let expires_at = required(entry, "expires_at")? - .as_i64() - .ok_or(GatewayFault::DecodeFailed)?; - let nonce = match optional(entry, "token_nonce") { - None => Nonce::Empty, - Some(value) => Nonce::from_wire(Some(text(value)?)), - }; - - Ok(PendingJoin { - connection, - user_id: UserId::new(snowflake(required(entry, "user_id")?)?), - expires_at: WallMillis::new(u64::try_from(expires_at).unwrap_or(0)), - nonce, - }) -} - -pub fn decode_pending_joins(result: &Value) -> Result, GatewayFault> { - entries(result, "pending_joins")? - .iter() - .map(decode_pending_join) - .collect() -} - -pub fn decode_confirm(result: &Value) -> Result { - if !flag(result, "success")? { - return Ok(ConfirmOutcome::Failed(error_of(result))); - } - if flag(result, "call_not_found")? { - return Ok(ConfirmOutcome::CallNotFound); - } - if flag(result, "already_confirmed")? { - return Ok(ConfirmOutcome::AlreadyConfirmed); - } - Ok(ConfirmOutcome::Confirmed) -} - -pub fn decode_repair(result: &Value) -> Result { - if !flag(result, "success")? { - return Ok(RepairOutcome::Failed(error_of(result))); - } - match optional(result, "repaired") { - None => Ok(RepairOutcome::NoChange), - Some(value) => match value.as_bool().ok_or(GatewayFault::DecodeFailed)? { - true => Ok(RepairOutcome::Repaired), - false => Ok(RepairOutcome::NoChange), - }, - } -} - -pub fn decode_disconnect(result: &Value) -> Result { - if !flag(result, "success")? { - return Ok(DisconnectOutcome::Failed(error_of(result))); - } - if flag(result, "ignored")? { - return Ok(DisconnectOutcome::Ignored { - reason: reason(result)?, - }); - } - if flag(result, "call_not_found")? { - return Ok(DisconnectOutcome::CallNotFound); - } - Ok(DisconnectOutcome::Removed) -} - -pub struct NatsGateway { - transport: T, -} - -impl NatsGateway { - pub const fn new(transport: T) -> Self { - Self { transport } - } - - pub const fn transport(&self) -> &T { - &self.transport - } -} - -impl NatsGateway { - async fn call( - &self, - method: GatewayMethod, - params: &P, - ) -> Result { - let Ok(payload) = serde_json::to_vec(params) else { - return Err(GatewayFault::TransportFailed); - }; - let bytes = self - .transport - .request(&method.subject(), &payload, method.deadline()) - .await - .map_err(|error| classify_transport_error(&error))?; - decode_envelope(&bytes) - } -} - -impl GatewayApi for NatsGateway { - async fn active_voice_rooms(&self) -> Result { - let result = self - .call(GatewayMethod::ActiveVoiceRooms, &NoParams {}) - .await?; - decode_active_voice_rooms(&result) - } - - async fn voice_states_for_channel( - &self, - room: RoomKey, - ) -> Result, GatewayFault> { - let result = self - .call( - GatewayMethod::VoiceStatesForChannel, - &channel_read_params(room), - ) - .await?; - decode_voice_states(&result) - } - - async fn pending_joins_for_channel( - &self, - room: RoomKey, - ) -> Result, GatewayFault> { - let result = self - .call( - GatewayMethod::PendingJoinsForChannel, - &channel_read_params(room), - ) - .await?; - decode_pending_joins(&result) - } - - async fn confirm_connection( - &self, - room: RoomKey, - connection: &ConnectionId, - nonce: &Nonce, - ) -> Result { - let result = self - .call( - GatewayMethod::ConfirmConnection, - &confirm_params(room, connection, nonce), - ) - .await?; - decode_confirm(&result) - } - - async fn repair_state_from_cache( - &self, - room: RoomKey, - user_id: UserId, - connection: &ConnectionId, - ) -> Result { - let result = self - .call( - GatewayMethod::RepairStateFromCache, - &repair_params(room, user_id, connection), - ) - .await?; - decode_repair(&result) - } - - async fn disconnect_user_if_in_channel( - &self, - room: RoomKey, - user_id: UserId, - connection: Option<&ConnectionId>, - ) -> Result { - let result = self - .call( - GatewayMethod::DisconnectUserIfInChannel, - &disconnect_params(room, user_id, connection), - ) - .await?; - decode_disconnect(&result) - } -} diff --git a/fluxer_recon/src/guards.rs b/fluxer_recon/src/guards.rs deleted file mode 100644 index 15fb0cd47..000000000 --- a/fluxer_recon/src/guards.rs +++ /dev/null @@ -1,572 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::config::ConnectionIdGuard; -use crate::evidence::{MediaSighting, PendingJoin}; -use crate::health::ServerHealth; -use crate::ids::{ConnectionId, Location, Millis, RoomKey, TurnId, UserId, WallMillis}; -use crate::ledger::RepairVerdict; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum AbortReason { - SiblingConnection, - NoConnectionId, - PendingJoin, - PreflightStale, - PreflightDisagreed, - JoinedAfterDivergence, - UnknownGateway, - UnknownMedia, - ServerUnhealthy, - Budget, - Mode, - Warmup, - DmPosture, - Breaker, -} - -impl AbortReason { - pub const ALL: [Self; 14] = [ - Self::SiblingConnection, - Self::NoConnectionId, - Self::PendingJoin, - Self::PreflightStale, - Self::PreflightDisagreed, - Self::JoinedAfterDivergence, - Self::UnknownGateway, - Self::UnknownMedia, - Self::ServerUnhealthy, - Self::Budget, - Self::Mode, - Self::Warmup, - Self::DmPosture, - Self::Breaker, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::SiblingConnection => "sibling_connection", - Self::NoConnectionId => "no_connection_id", - Self::PendingJoin => "pending_join", - Self::PreflightStale => "preflight_stale", - Self::PreflightDisagreed => "preflight_disagreed", - Self::JoinedAfterDivergence => "joined_after_divergence", - Self::UnknownGateway => "unknown_gateway", - Self::UnknownMedia => "unknown_media", - Self::ServerUnhealthy => "server_unhealthy", - Self::Budget => "budget", - Self::Mode => "mode", - Self::Warmup => "warmup", - Self::DmPosture => "dm_posture", - Self::Breaker => "breaker", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PreflightVoiceState { - pub user_id: UserId, - pub connection: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GatewayPreflight { - pub turn: TurnId, - pub taken_at: Millis, - pub voice_states: Vec, - pub pending_joins: Vec, - pub media_present: Vec, -} - -impl GatewayPreflight { - pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { - now.saturating_since(self.taken_at) <= max_age_ms - } - - pub fn states_for(&self, user_id: UserId) -> impl Iterator { - self.voice_states - .iter() - .filter(move |state| state.user_id == user_id) - } - - pub fn user_has_pending_join( - &self, - user_id: UserId, - wall_now: WallMillis, - skew_ms: u64, - ) -> bool { - self.pending_joins - .iter() - .any(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) - } - - pub fn media_holds(&self, connection: &ConnectionId) -> bool { - self.media_present - .iter() - .any(|sighting| &sighting.connection == connection) - } - - pub fn media_legs_of(&self, user_id: UserId) -> Vec { - self.media_present - .iter() - .filter(|sighting| sighting.user_id == user_id) - .map(|sighting| sighting.connection.clone()) - .collect() - } - - pub fn joining_connections_of( - &self, - user_id: UserId, - wall_now: WallMillis, - skew_ms: u64, - ) -> Vec { - self.pending_joins - .iter() - .filter(|join| join.user_id == user_id && join.is_unexpired(wall_now, skew_ms)) - .map(|join| join.connection.clone()) - .collect() - } - - pub fn live_connections_of( - &self, - user_id: UserId, - window: PreflightWindow, - ) -> Vec { - let joining = - self.joining_connections_of(user_id, window.wall_now, window.pending_join_skew_ms); - let mut live: Vec = self - .states_for(user_id) - .filter_map(|state| state.connection.clone()) - .collect(); - live.extend(self.media_legs_of(user_id)); - live.retain(|connection| !joining.contains(connection)); - live.sort(); - live.dedup(); - live - } - - pub fn gateway_holds(&self, connection: &ConnectionId) -> bool { - self.voice_states - .iter() - .any(|state| state.connection.as_ref() == Some(connection)) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum UnknownMediaCause { - UnreadRequired, - Unasked, - NoCustody, - UserHomeSilent, - NoStamp, - LiveServerSilent, -} - -impl UnknownMediaCause { - pub const ALL: [Self; 6] = [ - Self::UnreadRequired, - Self::Unasked, - Self::NoCustody, - Self::UserHomeSilent, - Self::NoStamp, - Self::LiveServerSilent, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::UnreadRequired => "unread_required", - Self::Unasked => "unasked", - Self::NoCustody => "no_custody", - Self::UserHomeSilent => "user_home_silent", - Self::NoStamp => "no_stamp", - Self::LiveServerSilent => "live_server_silent", - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct MediaCoverage { - fleet: Vec, - required: Vec, - read: Vec, - absent: Vec, - unreachable: Vec, -} - -impl MediaCoverage { - pub fn over(fleet: Vec, required: Vec) -> Self { - let mut fleet = fleet; - fleet.sort(); - fleet.dedup(); - let mut required = required; - required.sort(); - required.dedup(); - required.retain(|location| fleet.contains(location)); - Self { - fleet, - required, - read: Vec::new(), - absent: Vec::new(), - unreachable: Vec::new(), - } - } - - pub fn note_read(&mut self, location: &Location) { - if self.fleet.contains(location) && !self.read.contains(location) { - self.read.push(location.clone()); - } - } - - pub fn note_absent(&mut self, location: &Location) { - if self.fleet.contains(location) - && !self.read.contains(location) - && !self.absent.contains(location) - { - self.absent.push(location.clone()); - } - } - - pub fn note_unreachable(&mut self, location: &Location) { - if self.fleet.contains(location) && !self.unreachable.contains(location) { - self.unreachable.push(location.clone()); - } - } - - pub fn fleet(&self) -> &[Location] { - &self.fleet - } - - pub fn required(&self) -> &[Location] { - &self.required - } - - pub fn read(&self) -> &[Location] { - &self.read - } - - pub fn absent(&self) -> &[Location] { - &self.absent - } - - pub fn unreachable(&self) -> &[Location] { - &self.unreachable - } - - pub fn answered(&self) -> Vec { - let mut answered = self.read.clone(); - for location in &self.absent { - if !answered.contains(location) { - answered.push(location.clone()); - } - } - answered.sort(); - answered - } - - pub fn silent(&self) -> Vec { - self.fleet - .iter() - .filter(|location| !self.read.contains(location) && !self.absent.contains(location)) - .cloned() - .collect() - } - - pub fn unasked(&self) -> Option<&Location> { - self.fleet.iter().find(|location| { - !self.read.contains(location) - && !self.absent.contains(location) - && !self.unreachable.contains(location) - }) - } - - pub fn unread(&self) -> Option<&Location> { - self.required - .iter() - .find(|location| !self.read.contains(location) && !self.absent.contains(location)) - } - - pub fn is_complete(&self) -> bool { - !self.read.is_empty() && self.unasked().is_none() && self.unread().is_none() - } -} - -pub const fn connection_id_is_honoured(posture: ConnectionIdGuard, room: RoomKey) -> bool { - match room { - RoomKey::Guild { .. } => matches!(posture, ConnectionIdGuard::Honoured), - RoomKey::Dm { .. } => false, - } -} - -pub fn media_coverage_cause( - coverage: &MediaCoverage, - posture: ConnectionIdGuard, - room: RoomKey, -) -> Option { - if connection_id_is_honoured(posture, room) { - return None; - } - if coverage.unread().is_some() { - return Some(UnknownMediaCause::UnreadRequired); - } - if coverage.is_complete() { - None - } else { - Some(UnknownMediaCause::Unasked) - } -} - -pub fn media_coverage_refusal( - coverage: &MediaCoverage, - posture: ConnectionIdGuard, - room: RoomKey, -) -> Option { - media_coverage_cause(coverage, posture, room).map(|_| AbortReason::UnknownMedia) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MediaCustody<'a> { - pub room: RoomKey, - pub posture: ConnectionIdGuard, - pub fleet: &'a [Location], - pub answered: &'a [Location], - pub read: &'a [Location], - pub target_home: Option<&'a Location>, - pub user_homes: &'a [Location], - pub stamped: &'a [Location], -} - -impl MediaCustody<'_> { - fn silent(&self) -> Vec<&Location> { - self.fleet - .iter() - .filter(|location| !self.answered.contains(location)) - .collect() - } - - fn target_is_held(&self) -> bool { - self.target_home - .is_some_and(|home| self.fleet.contains(home) && self.read.contains(home)) - } -} - -pub fn media_custody_cause(custody: &MediaCustody<'_>) -> Option { - let silent = custody.silent(); - if silent.is_empty() { - return None; - } - if connection_id_is_honoured(custody.posture, custody.room) { - return if silent - .iter() - .all(|location| custody.stamped.contains(location)) - { - None - } else { - Some(UnknownMediaCause::NoStamp) - }; - } - if !custody.target_is_held() { - return Some(UnknownMediaCause::NoCustody); - } - if custody.user_homes.iter().any(|home| silent.contains(&home)) { - return Some(UnknownMediaCause::UserHomeSilent); - } - None -} - -pub fn media_custody_refusal(custody: &MediaCustody<'_>) -> Option { - media_custody_cause(custody).map(|_| AbortReason::UnknownMedia) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FleetSilence<'a> { - pub room: RoomKey, - pub posture: ConnectionIdGuard, - pub fleet: &'a [Location], - pub answered: &'a [Location], - pub written_off: &'a [Location], -} - -impl FleetSilence<'_> { - fn silent_and_still_expected(&self) -> Option<&Location> { - self.fleet.iter().find(|location| { - !self.answered.contains(location) && !self.written_off.contains(location) - }) - } -} - -pub fn fleet_silence_cause(silence: &FleetSilence<'_>) -> Option { - if connection_id_is_honoured(silence.posture, silence.room) { - return None; - } - silence - .silent_and_still_expected() - .map(|_| UnknownMediaCause::LiveServerSilent) -} - -pub fn fleet_silence_refusal(silence: &FleetSilence<'_>) -> Option { - fleet_silence_cause(silence).map(|_| AbortReason::UnknownMedia) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct PreflightWindow { - pub now: Millis, - pub wall_now: WallMillis, - pub preflight_max_age_ms: u64, - pub pending_join_skew_ms: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct DmPosture { - pub dm_gateway_eviction: bool, - pub dm_media_eviction: bool, -} - -impl Default for DmPosture { - fn default() -> Self { - Self { - dm_gateway_eviction: true, - dm_media_eviction: false, - } - } -} - -impl DmPosture { - pub const fn allows_gateway_removal(self, room: RoomKey) -> bool { - match room { - RoomKey::Guild { .. } => true, - RoomKey::Dm { .. } => self.dm_gateway_eviction, - } - } - - pub const fn allows_media_removal(self, room: RoomKey) -> bool { - match room { - RoomKey::Guild { .. } => true, - RoomKey::Dm { .. } => self.dm_media_eviction, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GatewayRemovalGuard<'a> { - pub room: RoomKey, - pub target: &'a ConnectionId, - pub user_id: UserId, - pub posture: ConnectionIdGuard, - pub dm_posture: DmPosture, - pub preflight: &'a GatewayPreflight, - pub window: PreflightWindow, -} - -pub fn gateway_removal_preflight(guard: &GatewayRemovalGuard<'_>) -> Result<(), AbortReason> { - if !guard - .preflight - .is_fresh(guard.window.now, guard.window.preflight_max_age_ms) - { - return Err(AbortReason::PreflightStale); - } - - if !guard.dm_posture.allows_gateway_removal(guard.room) { - return Err(AbortReason::DmPosture); - } - - let states: Vec<&PreflightVoiceState> = guard.preflight.states_for(guard.user_id).collect(); - - if states.iter().any(|state| state.connection.is_none()) { - return Err(AbortReason::NoConnectionId); - } - - if connection_id_is_honoured(guard.posture, guard.room) { - if !states - .iter() - .any(|state| state.connection.as_ref() == Some(guard.target)) - { - return Err(AbortReason::PreflightDisagreed); - } - } else { - let live = guard - .preflight - .live_connections_of(guard.user_id, guard.window); - if live.len() != 1 { - return Err(AbortReason::SiblingConnection); - } - if &live[0] != guard.target { - return Err(AbortReason::SiblingConnection); - } - } - - if guard.preflight.user_has_pending_join( - guard.user_id, - guard.window.wall_now, - guard.window.pending_join_skew_ms, - ) { - return Err(AbortReason::PendingJoin); - } - - if guard.preflight.media_holds(guard.target) { - return Err(AbortReason::PreflightDisagreed); - } - - Ok(()) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MediaRemovalGuard<'a> { - pub room: RoomKey, - pub target: &'a ConnectionId, - pub user_id: UserId, - pub dm_posture: DmPosture, - pub divergence_since: Millis, - pub participant_joined_at: Millis, - pub repair_verdict: Option, - pub server_health: ServerHealth, - pub preflight: &'a GatewayPreflight, - pub window: PreflightWindow, -} - -pub fn media_removal_preflight(guard: &MediaRemovalGuard<'_>) -> Result<(), AbortReason> { - if !guard - .preflight - .is_fresh(guard.window.now, guard.window.preflight_max_age_ms) - { - return Err(AbortReason::PreflightStale); - } - - if !guard.dm_posture.allows_media_removal(guard.room) { - return Err(AbortReason::DmPosture); - } - - if guard - .preflight - .states_for(guard.user_id) - .any(|state| state.connection.is_none()) - { - return Err(AbortReason::NoConnectionId); - } - - if guard.participant_joined_at > guard.divergence_since { - return Err(AbortReason::JoinedAfterDivergence); - } - - if matches!(guard.room, RoomKey::Guild { .. }) - && guard.repair_verdict != Some(RepairVerdict::NotRepairable) - { - return Err(AbortReason::PreflightDisagreed); - } - - if guard.preflight.gateway_holds(guard.target) { - return Err(AbortReason::PreflightDisagreed); - } - - if guard.preflight.user_has_pending_join( - guard.user_id, - guard.window.wall_now, - guard.window.pending_join_skew_ms, - ) { - return Err(AbortReason::PendingJoin); - } - - if !guard.server_health.is_healthy() { - return Err(AbortReason::ServerUnhealthy); - } - - Ok(()) -} diff --git a/fluxer_recon/src/health.rs b/fluxer_recon/src/health.rs deleted file mode 100644 index 4292607df..000000000 --- a/fluxer_recon/src/health.rs +++ /dev/null @@ -1,502 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; - -use crate::ids::{Location, Millis}; - -pub const DEGRADED_AT_FAILURES: u32 = 3; -pub const UNREACHABLE_AT_FAILURES: u32 = 10; -pub const QUARANTINED_AT_FAILURES: u32 = 30; -pub const SUCCESSES_TO_RECOVER: u32 = 3; -pub const UNREACHABLE_PROBE_INTERVAL_MS: u64 = 10_000; -pub const QUARANTINED_PROBE_INTERVAL_MS: u64 = 60_000; -pub const MUTATIONS_PAUSED_BACKOFF_MS: u64 = 15_000; -pub const MUTATIONS_PAUSED_MAX_BACKOFF_MS: u64 = 120_000; -pub const MUTATIONS_PAUSED_QUIET_MS: u64 = 300_000; -pub const MUTATIONS_PAUSED_MAX_DOUBLINGS: u32 = 3; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ReconMode { - Halted, - Observing, - Constructive, - Enforcing, -} - -impl ReconMode { - pub fn parse(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "halted" => Some(Self::Halted), - "observing" => Some(Self::Observing), - "constructive" => Some(Self::Constructive), - "enforcing" => Some(Self::Enforcing), - _ => None, - } - } - - pub const fn as_str(self) -> &'static str { - match self { - Self::Halted => "halted", - Self::Observing => "observing", - Self::Constructive => "constructive", - Self::Enforcing => "enforcing", - } - } - - pub const fn allows_constructive(self) -> bool { - matches!(self, Self::Constructive | Self::Enforcing) - } - - pub const fn allows_destructive(self) -> bool { - matches!(self, Self::Enforcing) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ModeClamp { - Warmup, - ColdStart, - BreakerTripped, - SingletonConflict, - TopologyStale, - NatsReconnect, - MutationsPaused, - CoverageOverrun, -} - -impl ModeClamp { - pub const ALL: [Self; 8] = [ - Self::Warmup, - Self::ColdStart, - Self::BreakerTripped, - Self::SingletonConflict, - Self::TopologyStale, - Self::NatsReconnect, - Self::MutationsPaused, - Self::CoverageOverrun, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::Warmup => "warmup", - Self::ColdStart => "cold_start", - Self::BreakerTripped => "breaker_tripped", - Self::SingletonConflict => "singleton_conflict", - Self::TopologyStale => "topology_stale", - Self::NatsReconnect => "nats_reconnect", - Self::MutationsPaused => "mutations_paused", - Self::CoverageOverrun => "coverage_overrun", - } - } - - pub fn parse(value: &str) -> Option { - let wanted = value.trim().to_ascii_lowercase(); - Self::ALL.into_iter().find(|clamp| clamp.label() == wanted) - } - - pub const fn is_self_clearing(self) -> bool { - match self { - Self::Warmup - | Self::ColdStart - | Self::SingletonConflict - | Self::TopologyStale - | Self::NatsReconnect - | Self::MutationsPaused - | Self::CoverageOverrun => true, - Self::BreakerTripped => false, - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ModeClamps { - active: BTreeMap>, -} - -impl ModeClamps { - pub fn new() -> Self { - Self::default() - } - - pub fn set(&mut self, clamp: ModeClamp, engaged: bool) { - if engaged { - self.active.insert(clamp, None); - } else { - self.active.remove(&clamp); - } - } - - pub fn engage_until(&mut self, clamp: ModeClamp, until: Millis) { - self.active.insert(clamp, Some(until)); - } - - pub fn release_expired(&mut self, now: Millis) { - self.active - .retain(|_, until| until.is_none_or(|deadline| now < deadline)); - } - - pub fn is_engaged(&self, clamp: ModeClamp) -> bool { - self.active.contains_key(&clamp) - } - - pub fn deadline(&self, clamp: ModeClamp) -> Option { - self.active.get(&clamp).copied().flatten() - } - - pub fn release(&mut self, clamp: ModeClamp) -> bool { - self.active.remove(&clamp).is_some() - } - - pub fn engaged(&self) -> impl Iterator + '_ { - self.active.keys().copied() - } - - pub fn is_empty(&self) -> bool { - self.active.is_empty() - } - - pub fn ceiling(&self) -> ReconMode { - if self.active.is_empty() { - ReconMode::Enforcing - } else { - ReconMode::Constructive - } - } -} - -pub fn effective_mode( - configured: ReconMode, - runtime_override: Option, - clamps: &ModeClamps, -) -> ReconMode { - let requested = match runtime_override { - None => configured, - Some(override_mode) => configured.min(override_mode), - }; - requested.min(clamps.ceiling()) -} - -pub const fn mutations_paused_hold_ms(windows: u32) -> u64 { - if windows == 0 { - return 0; - } - let doublings = if windows - 1 > MUTATIONS_PAUSED_MAX_DOUBLINGS { - MUTATIONS_PAUSED_MAX_DOUBLINGS - } else { - windows - 1 - }; - let hold = MUTATIONS_PAUSED_BACKOFF_MS.saturating_mul(1u64 << doublings); - if hold > MUTATIONS_PAUSED_MAX_BACKOFF_MS { - MUTATIONS_PAUSED_MAX_BACKOFF_MS - } else { - hold - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct PauseBackoff { - windows: u32, - until: Option, -} - -impl PauseBackoff { - pub const fn new() -> Self { - Self { - windows: 0, - until: None, - } - } - - pub const fn windows(&self) -> u32 { - self.windows - } - - pub const fn until(&self) -> Option { - self.until - } - - pub const fn hold_ms(&self) -> u64 { - mutations_paused_hold_ms(self.windows) - } - - pub const fn is_holding(&self, now: Millis) -> bool { - match self.until { - None => false, - Some(until) => now.get() < until.get(), - } - } - - pub const fn remaining_ms(&self, now: Millis) -> u64 { - match self.until { - None => 0, - Some(until) => until.saturating_since(now), - } - } - - pub const fn note_refused(&mut self, now: Millis) -> Option { - if self.is_holding(now) { - return None; - } - self.windows = self.windows.saturating_add(1); - let until = now.saturating_add_millis(mutations_paused_hold_ms(self.windows)); - self.until = Some(until); - Some(until) - } - - pub const fn clear(&mut self) { - self.windows = 0; - self.until = None; - } - - pub const fn decay(&mut self, now: Millis) -> bool { - let Some(until) = self.until else { - return false; - }; - if now.get() < until.get() { - return false; - } - if now.saturating_since(until) < MUTATIONS_PAUSED_QUIET_MS { - return false; - } - self.clear(); - true - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct WarmupGate { - started_at: Millis, - warmup_ms: u64, - coverage_pass_complete: bool, -} - -impl WarmupGate { - pub const fn new(started_at: Millis, warmup_ms: u64) -> Self { - Self { - started_at, - warmup_ms, - coverage_pass_complete: false, - } - } - - pub const fn note_coverage_pass(&mut self) { - self.coverage_pass_complete = true; - } - - pub const fn coverage_pass_complete(&self) -> bool { - self.coverage_pass_complete - } - - pub const fn elapsed_complete(&self, now: Millis) -> bool { - now.saturating_since(self.started_at) >= self.warmup_ms - } - - pub const fn is_complete(&self, now: Millis) -> bool { - self.elapsed_complete(now) && self.coverage_pass_complete - } - - pub const fn force_complete(&mut self) { - self.warmup_ms = 0; - self.coverage_pass_complete = true; - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ServerHealth { - Healthy, - Degraded, - Unreachable, - Quarantined, -} - -impl ServerHealth { - pub const fn label(self) -> &'static str { - match self { - Self::Healthy => "healthy", - Self::Degraded => "degraded", - Self::Unreachable => "unreachable", - Self::Quarantined => "quarantined", - } - } - - pub const fn is_healthy(self) -> bool { - matches!(self, Self::Healthy) - } - - pub const fn is_written_off(self) -> bool { - matches!(self, Self::Unreachable | Self::Quarantined) - } - - pub const fn severity(self) -> u8 { - match self { - Self::Healthy => 0, - Self::Degraded => 1, - Self::Unreachable => 2, - Self::Quarantined => 3, - } - } - - pub const fn worse_of(self, other: Self) -> Self { - if self.severity() >= other.severity() { - self - } else { - other - } - } - - pub const fn probe_interval_ms(self) -> Option { - match self { - Self::Healthy | Self::Degraded => None, - Self::Unreachable => Some(UNREACHABLE_PROBE_INTERVAL_MS), - Self::Quarantined => Some(QUARANTINED_PROBE_INTERVAL_MS), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ProbeOutcome { - Success, - Failure, - AuthFailure, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ServerHealthTracker { - state: ServerHealth, - consecutive_failures: u32, - consecutive_successes: u32, - last_probe_at: Option, - ever_answered: bool, -} - -impl Default for ServerHealthTracker { - fn default() -> Self { - Self::new() - } -} - -impl ServerHealthTracker { - pub const fn new() -> Self { - Self { - state: ServerHealth::Healthy, - consecutive_failures: 0, - consecutive_successes: 0, - last_probe_at: None, - ever_answered: false, - } - } - - pub const fn state(&self) -> ServerHealth { - self.state - } - - pub const fn consecutive_failures(&self) -> u32 { - self.consecutive_failures - } - - pub const fn ever_answered(&self) -> bool { - self.ever_answered - } - - pub const fn record(&mut self, outcome: ProbeOutcome, now: Millis) { - self.last_probe_at = Some(now); - match outcome { - ProbeOutcome::Success => { - self.ever_answered = true; - self.consecutive_failures = 0; - self.consecutive_successes = self.consecutive_successes.saturating_add(1); - if self.consecutive_successes >= SUCCESSES_TO_RECOVER { - self.state = ServerHealth::Healthy; - } - } - ProbeOutcome::Failure => { - self.consecutive_successes = 0; - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - self.state = self - .state - .worse_of(failure_state(self.consecutive_failures)); - } - ProbeOutcome::AuthFailure => { - self.consecutive_successes = 0; - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - self.state = ServerHealth::Quarantined; - } - } - } - - pub const fn may_probe(&self, now: Millis) -> bool { - match (self.state.probe_interval_ms(), self.last_probe_at) { - (None, _) => true, - (Some(_), None) => true, - (Some(interval), Some(last)) => now.saturating_since(last) >= interval, - } - } - - pub const fn last_probe_at(&self) -> Option { - self.last_probe_at - } -} - -const fn failure_state(consecutive_failures: u32) -> ServerHealth { - if consecutive_failures >= QUARANTINED_AT_FAILURES { - ServerHealth::Quarantined - } else if consecutive_failures >= UNREACHABLE_AT_FAILURES { - ServerHealth::Unreachable - } else if consecutive_failures >= DEGRADED_AT_FAILURES { - ServerHealth::Degraded - } else { - ServerHealth::Healthy - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ServerHealthMap { - servers: BTreeMap, -} - -impl ServerHealthMap { - pub fn new() -> Self { - Self::default() - } - - pub fn record(&mut self, location: &Location, outcome: ProbeOutcome, now: Millis) { - self.servers - .entry(location.clone()) - .or_default() - .record(outcome, now); - } - - pub fn health(&self, location: &Location) -> ServerHealth { - self.servers - .get(location) - .map_or(ServerHealth::Healthy, ServerHealthTracker::state) - } - - pub fn ever_answered(&self, location: &Location) -> bool { - self.servers - .get(location) - .is_some_and(ServerHealthTracker::ever_answered) - } - - pub fn may_probe(&self, location: &Location, now: Millis) -> bool { - self.servers - .get(location) - .is_none_or(|tracker| tracker.may_probe(now)) - } - - pub fn tracked(&self) -> impl Iterator { - self.servers.iter() - } - - pub fn forget(&mut self, location: &Location) { - self.servers.remove(location); - } - - pub fn len(&self) -> usize { - self.servers.len() - } - - pub fn is_empty(&self) -> bool { - self.servers.is_empty() - } -} diff --git a/fluxer_recon/src/ids.rs b/fluxer_recon/src/ids.rs deleted file mode 100644 index 155874ff2..000000000 --- a/fluxer_recon/src/ids.rs +++ /dev/null @@ -1,252 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::fmt; - -const REDACTED: &str = "***"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum IdError { - #[error("identifier is empty")] - Empty, - #[error("identifier contains whitespace")] - ContainsWhitespace, - #[error("identifier contains an underscore")] - ContainsUnderscore, -} - -macro_rules! u64_newtype { - ($name:ident) => { - #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct $name(u64); - - impl $name { - pub const fn new(value: u64) -> Self { - Self(value) - } - - pub const fn get(self) -> u64 { - self.0 - } - } - - impl fmt::Display for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } - } - }; -} - -u64_newtype!(GuildId); -u64_newtype!(ChannelId); -u64_newtype!(UserId); -u64_newtype!(TurnId); -u64_newtype!(Epoch); -u64_newtype!(Millis); -u64_newtype!(WallMillis); - -impl TurnId { - pub const FIRST: Self = Self(0); - - pub const fn next(self) -> Self { - Self(self.0.saturating_add(1)) - } -} - -impl Epoch { - pub const FIRST: Self = Self(0); - - pub const fn next(self) -> Self { - Self(self.0.saturating_add(1)) - } -} - -impl Millis { - pub const ZERO: Self = Self(0); - - pub const fn saturating_since(self, earlier: Self) -> u64 { - self.0.saturating_sub(earlier.0) - } - - pub const fn saturating_add_millis(self, delta: u64) -> Self { - Self(self.0.saturating_add(delta)) - } -} - -impl WallMillis { - pub const ZERO: Self = Self(0); - - pub const fn saturating_since(self, earlier: Self) -> u64 { - self.0.saturating_sub(earlier.0) - } - - pub const fn saturating_add_millis(self, delta: u64) -> Self { - Self(self.0.saturating_add(delta)) - } -} - -fn checked_text(value: &str) -> Result, IdError> { - if value.is_empty() { - return Err(IdError::Empty); - } - if value.contains(char::is_whitespace) { - return Err(IdError::ContainsWhitespace); - } - Ok(Box::from(value)) -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ConnectionId(Box); - -impl ConnectionId { - pub fn new(value: &str) -> Result { - let text = checked_text(value)?; - if text.contains('_') { - return Err(IdError::ContainsUnderscore); - } - Ok(Self(text)) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for ConnectionId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct RegionId(Box); - -impl RegionId { - pub fn new(value: &str) -> Result { - checked_text(value).map(Self) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for RegionId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ServerId(Box); - -impl ServerId { - pub fn new(value: &str) -> Result { - checked_text(value).map(Self) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for ServerId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -#[derive(Clone, PartialEq, Eq)] -pub struct ApiSecret(Box); - -impl ApiSecret { - pub fn new(value: &str) -> Self { - Self(Box::from(value)) - } - - pub fn expose(&self) -> &str { - &self.0 - } -} - -impl fmt::Debug for ApiSecret { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(REDACTED) - } -} - -impl fmt::Display for ApiSecret { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(REDACTED) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum Scope { - Guild, - Dm, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum RoomKey { - Guild { - guild_id: GuildId, - channel_id: ChannelId, - }, - Dm { - channel_id: ChannelId, - }, -} - -impl RoomKey { - pub const fn scope(self) -> Scope { - match self { - Self::Guild { .. } => Scope::Guild, - Self::Dm { .. } => Scope::Dm, - } - } - - pub const fn channel_id(self) -> ChannelId { - match self { - Self::Guild { channel_id, .. } => channel_id, - Self::Dm { channel_id } => channel_id, - } - } - - pub const fn guild_id(self) -> Option { - match self { - Self::Guild { guild_id, .. } => Some(guild_id), - Self::Dm { .. } => None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Location { - pub region: RegionId, - pub server: ServerId, -} - -impl Location { - pub const fn new(region: RegionId, server: ServerId) -> Self { - Self { region, server } - } -} - -impl fmt::Display for Location { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}/{}", self.region, self.server) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ConnectionKey { - pub room: RoomKey, - pub connection: ConnectionId, -} - -impl ConnectionKey { - pub const fn new(room: RoomKey, connection: ConnectionId) -> Self { - Self { room, connection } - } -} diff --git a/fluxer_recon/src/ledger.rs b/fluxer_recon/src/ledger.rs deleted file mode 100644 index 4d15a0e02..000000000 --- a/fluxer_recon/src/ledger.rs +++ /dev/null @@ -1,978 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; -use std::mem::size_of; - -use crate::decide::DecideParams; -use crate::evidence::{CliffWindow, PriorConnection}; -use crate::gateway::Nonce; -use crate::ids::{ConnectionId, ConnectionKey, Epoch, Location, Millis, RoomKey, TurnId, UserId}; - -const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; -const CONNECTION_ID_HEAP_BYTES: usize = 40; -pub const MAX_ROSTER_STAMPS: usize = 16; -const ROSTER_STAMP_BYTES: usize = 16; -const ROOM_OVERHEAD_BYTES: usize = 256 + MAX_ROSTER_STAMPS * ROSTER_STAMP_BYTES; - -pub const MAP_ENTRY_OVERHEAD_BYTES: usize = 48; -pub const LOCATION_MAP_BYTES: u64 = 640; -pub const LOCATION_ENTRY_BYTES: u64 = 64; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Fingerprint(u64); - -impl Fingerprint { - pub const fn get(self) -> u64 { - self.0 - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Hasher(u64); - -impl Hasher { - const fn new() -> Self { - Self(FNV_OFFSET) - } - - const fn byte(&mut self, value: u8) { - self.0 ^= value as u64; - self.0 = self.0.wrapping_mul(FNV_PRIME); - } - - const fn u64(&mut self, value: u64) { - let bytes = value.to_le_bytes(); - let mut index = 0; - while index < bytes.len() { - self.byte(bytes[index]); - index += 1; - } - } - - fn text(&mut self, value: &str) { - for byte in value.as_bytes() { - self.byte(*byte); - } - self.byte(0); - } - - const fn finish(self) -> Fingerprint { - Fingerprint(self.0) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct FingerprintInput<'a> { - pub room: RoomKey, - pub connection: &'a ConnectionId, - pub user_id: UserId, - pub gateway_siblings: &'a [ConnectionId], - pub media_locations: &'a [Location], - pub participant_joined_at: Option, - pub census_epoch: Epoch, - pub topology_epoch: Epoch, -} - -pub fn fingerprint(input: &FingerprintInput<'_>) -> Fingerprint { - let mut hasher = Hasher::new(); - - match input.room { - RoomKey::Guild { - guild_id, - channel_id, - } => { - hasher.byte(1); - hasher.u64(guild_id.get()); - hasher.u64(channel_id.get()); - } - RoomKey::Dm { channel_id } => { - hasher.byte(2); - hasher.u64(channel_id.get()); - } - } - - hasher.text(input.connection.as_str()); - hasher.u64(input.user_id.get()); - - hasher.u64(input.gateway_siblings.len() as u64); - for sibling in input.gateway_siblings { - hasher.text(sibling.as_str()); - } - - hasher.u64(input.media_locations.len() as u64); - for location in input.media_locations { - hasher.text(location.region.as_str()); - hasher.text(location.server.as_str()); - } - - match input.participant_joined_at { - None => hasher.byte(0), - Some(joined_at) => { - hasher.byte(1); - hasher.u64(joined_at.get()); - } - } - - hasher.u64(input.census_epoch.get()); - hasher.u64(input.topology_epoch.get()); - hasher.finish() -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct LocationIndex(u16); - -impl LocationIndex { - pub const fn get(self) -> u16 { - self.0 - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct LocationTable { - entries: Vec, -} - -impl LocationTable { - pub const fn new() -> Self { - Self { - entries: Vec::new(), - } - } - - pub fn intern(&mut self, location: &Location) -> Option { - if let Some(position) = self.entries.iter().position(|entry| entry == location) { - return u16::try_from(position).ok().map(LocationIndex); - } - let index = u16::try_from(self.entries.len()).ok()?; - self.entries.push(location.clone()); - Some(LocationIndex(index)) - } - - pub fn get(&self, index: LocationIndex) -> Option<&Location> { - self.entries.get(usize::from(index.0)) - } - - pub fn index_of(&self, location: &Location) -> Option { - self.entries - .iter() - .position(|entry| entry == location) - .and_then(|position| u16::try_from(position).ok()) - .map(LocationIndex) - } - - pub fn len(&self) -> usize { - self.entries.len() - } - - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RepairVerdict { - Repaired, - NoChange, - NotRepairable, -} - -impl RepairVerdict { - pub const fn label(self) -> &'static str { - match self { - Self::Repaired => "repaired", - Self::NoChange => "no_change", - Self::NotRepairable => "not_repairable", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ActionKind { - RemoveGatewayState, - RemoveParticipant, -} - -impl ActionKind { - pub const fn label(self) -> &'static str { - match self { - Self::RemoveGatewayState => "remove_gateway_state", - Self::RemoveParticipant => "remove_participant", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PriorDivergence { - GatewayOnly, - MediaOnly, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum WedgedReason { - RepairExhausted, - ActionIneffective, -} - -impl WedgedReason { - pub const fn label(self) -> &'static str { - match self { - Self::RepairExhausted => "repair_exhausted", - Self::ActionIneffective => "action_ineffective", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConnectionState { - Nascent, - Consistent, - PendingJoin { - expires_at: crate::ids::WallMillis, - nonce: Nonce, - last_confirm: Millis, - }, - GatewayOnly { - since: Millis, - corroborations: u32, - fingerprint: Fingerprint, - last_corroboration: Millis, - }, - MediaOnly { - since: Millis, - corroborations: u32, - fingerprint: Fingerprint, - last_corroboration: Millis, - location: LocationIndex, - participant_joined_at: Millis, - repair_verdict: Option, - }, - Repairing { - since: Millis, - last_attempt: Millis, - }, - ActionTaken { - kind: ActionKind, - at: Millis, - authorizing_turn: TurnId, - prior: PriorDivergence, - since: Millis, - }, - Wedged { - reason: WedgedReason, - at: Millis, - fingerprint: Fingerprint, - }, - Retired { - at: Millis, - }, -} - -impl ConnectionState { - pub const fn label(&self) -> &'static str { - match self { - Self::Nascent => "nascent", - Self::Consistent => "consistent", - Self::PendingJoin { .. } => "pending_join", - Self::GatewayOnly { .. } => "gateway_only", - Self::MediaOnly { .. } => "media_only", - Self::Repairing { .. } => "repairing", - Self::ActionTaken { .. } => "action_taken", - Self::Wedged { .. } => "wedged", - Self::Retired { .. } => "retired", - } - } - - pub const fn stored_fingerprint(&self) -> Option { - match self { - Self::GatewayOnly { fingerprint, .. } - | Self::MediaOnly { fingerprint, .. } - | Self::Wedged { fingerprint, .. } => Some(*fingerprint), - Self::Nascent - | Self::Consistent - | Self::PendingJoin { .. } - | Self::Repairing { .. } - | Self::ActionTaken { .. } - | Self::Retired { .. } => None, - } - } - - pub const fn corroborations(&self) -> u32 { - match self { - Self::GatewayOnly { corroborations, .. } | Self::MediaOnly { corroborations, .. } => { - *corroborations - } - Self::Nascent - | Self::Consistent - | Self::PendingJoin { .. } - | Self::Repairing { .. } - | Self::ActionTaken { .. } - | Self::Wedged { .. } - | Self::Retired { .. } => 0, - } - } - - pub const fn divergence_since(&self) -> Option { - match self { - Self::GatewayOnly { since, .. } - | Self::MediaOnly { since, .. } - | Self::Repairing { since, .. } - | Self::ActionTaken { since, .. } => Some(*since), - Self::Nascent - | Self::Consistent - | Self::PendingJoin { .. } - | Self::Wedged { .. } - | Self::Retired { .. } => None, - } - } - - pub const fn is_divergent(&self) -> bool { - match self { - Self::GatewayOnly { .. } - | Self::MediaOnly { .. } - | Self::Repairing { .. } - | Self::ActionTaken { .. } - | Self::Wedged { .. } => true, - Self::Nascent | Self::Consistent | Self::PendingJoin { .. } | Self::Retired { .. } => { - false - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ConnectionLedger { - pub connection: ConnectionId, - pub user_id: UserId, - pub state: ConnectionState, - pub last_seen: Millis, - pub action_attempts: u8, - pub repair_attempts: u8, - pub last_action_at: Option, - pub last_repair_verdict: Option, - pub last_location: Option, -} - -impl ConnectionLedger { - pub fn new(connection: ConnectionId, user_id: UserId, now: Millis) -> Self { - Self { - connection, - user_id, - state: ConnectionState::Nascent, - last_seen: now, - action_attempts: 0, - repair_attempts: 0, - last_action_at: None, - last_repair_verdict: None, - last_location: None, - } - } - - pub const fn is_divergent(&self) -> bool { - self.state.is_divergent() - } - - pub const fn believed_home(&self) -> Option { - match self.state { - ConnectionState::Retired { .. } => None, - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::GatewayOnly { .. } - | ConnectionState::MediaOnly { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::ActionTaken { .. } - | ConnectionState::Wedged { .. } => self.last_location, - } - } - - pub fn clear_evidence(&mut self) { - self.state = ConnectionState::Nascent; - self.last_repair_verdict = None; - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomLedger { - room: RoomKey, - connections: Vec, - last_seen: Millis, - last_turn: TurnId, - gateway_cliff: CliffWindow, - media_cliff: CliffWindow, - partially_unreadable: bool, - no_candidates: bool, - last_known_locations: Vec, - roster_stamps: Vec<(LocationIndex, Millis)>, -} - -impl RoomLedger { - fn new(room: RoomKey, now: Millis) -> Self { - Self { - room, - connections: Vec::new(), - last_seen: now, - last_turn: TurnId::FIRST, - gateway_cliff: CliffWindow::clear(), - media_cliff: CliffWindow::clear(), - partially_unreadable: false, - no_candidates: false, - last_known_locations: Vec::new(), - roster_stamps: Vec::new(), - } - } - - pub const fn room(&self) -> RoomKey { - self.room - } - - pub fn connections(&self) -> &[ConnectionLedger] { - &self.connections - } - - pub fn connection(&self, connection: &ConnectionId) -> Option<&ConnectionLedger> { - self.connections - .iter() - .find(|entry| &entry.connection == connection) - } - - pub fn connection_mut(&mut self, connection: &ConnectionId) -> Option<&mut ConnectionLedger> { - self.connections - .iter_mut() - .find(|entry| &entry.connection == connection) - } - - pub const fn last_seen(&self) -> Millis { - self.last_seen - } - - pub const fn last_turn(&self) -> TurnId { - self.last_turn - } - - pub const fn note_turn(&mut self, turn: TurnId, now: Millis) { - self.last_turn = turn; - self.last_seen = now; - } - - pub const fn gateway_cliff(&self) -> CliffWindow { - self.gateway_cliff - } - - pub const fn gateway_cliff_mut(&mut self) -> &mut CliffWindow { - &mut self.gateway_cliff - } - - pub const fn media_cliff(&self) -> CliffWindow { - self.media_cliff - } - - pub const fn media_cliff_mut(&mut self) -> &mut CliffWindow { - &mut self.media_cliff - } - - pub const fn partially_unreadable(&self) -> bool { - self.partially_unreadable - } - - pub const fn set_partially_unreadable(&mut self, value: bool) { - self.partially_unreadable = value; - } - - pub const fn no_candidates(&self) -> bool { - self.no_candidates - } - - pub const fn set_no_candidates(&mut self, value: bool) { - self.no_candidates = value; - } - - pub fn last_known_locations(&self) -> &[LocationIndex] { - &self.last_known_locations - } - - pub fn note_location(&mut self, index: LocationIndex) { - if !self.last_known_locations.contains(&index) { - self.last_known_locations.push(index); - } - } - - pub fn note_roster_read(&mut self, index: LocationIndex, at: Millis) { - if let Some(stamp) = self - .roster_stamps - .iter_mut() - .find(|(stamped, _)| *stamped == index) - { - stamp.1 = at; - return; - } - if self.roster_stamps.len() >= MAX_ROSTER_STAMPS { - let oldest = self - .roster_stamps - .iter() - .enumerate() - .min_by_key(|(_, (_, at))| at.get()) - .map(|(position, _)| position); - if let Some(position) = oldest { - self.roster_stamps.remove(position); - } - } - self.roster_stamps.push((index, at)); - } - - pub fn roster_read_at(&self, index: LocationIndex) -> Option { - self.roster_stamps - .iter() - .find(|(stamped, _)| *stamped == index) - .map(|(_, at)| *at) - } - - pub fn retain_roster_stamps(&mut self, live: &[LocationIndex]) { - self.roster_stamps.retain(|(index, _)| live.contains(index)); - } - - pub fn roster_stamps(&self) -> &[(LocationIndex, Millis)] { - &self.roster_stamps - } - - pub fn believed_homes(&self) -> Vec { - let mut homes: Vec = self - .connections - .iter() - .filter_map(ConnectionLedger::believed_home) - .collect(); - homes.sort_by_key(|index| index.get()); - homes.dedup(); - homes - } - - pub fn believed_home_counts(&self) -> Vec<(LocationIndex, u32)> { - let mut counts: Vec<(LocationIndex, u32)> = Vec::new(); - for home in self - .connections - .iter() - .filter_map(ConnectionLedger::believed_home) - { - match counts.iter_mut().find(|(index, _)| *index == home) { - Some((_, count)) => *count = count.saturating_add(1), - None => counts.push((home, 1)), - } - } - counts.sort_by_key(|(index, _)| index.get()); - counts - } - - pub fn prior_connections(&self) -> Vec { - self.connections - .iter() - .map(|entry| PriorConnection { - connection: entry.connection.clone(), - was_consistent: matches!(entry.state, ConnectionState::Consistent), - }) - .collect() - } - - pub fn reset_corroborations(&mut self) { - for entry in &mut self.connections { - if entry.is_divergent() { - entry.clear_evidence(); - } - } - } - - pub fn divergent_connections(&self) -> usize { - self.connections - .iter() - .filter(|entry| entry.is_divergent()) - .count() - } - - fn is_cold(&self) -> bool { - self.connections.iter().all(|entry| { - matches!( - entry.state, - ConnectionState::Consistent - | ConnectionState::Nascent - | ConnectionState::Retired { .. } - ) - }) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct LedgerLimits { - pub max_tracked_rooms: usize, - pub max_tracked_connections: usize, - pub max_connections_per_room: usize, - pub memory_budget_bytes: u64, - pub retired_grace_ms: u64, -} - -impl Default for LedgerLimits { - fn default() -> Self { - Self { - max_tracked_rooms: 8_192, - max_tracked_connections: 65_536, - max_connections_per_room: 512, - memory_budget_bytes: 16_777_216, - retired_grace_ms: 60_000, - } - } -} - -pub const CONNECTION_ENTRY_BYTES: u64 = - (size_of::() + CONNECTION_ID_HEAP_BYTES) as u64; -pub const ROOM_ENTRY_BYTES: u64 = (size_of::() + ROOM_OVERHEAD_BYTES) as u64; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum AdmissionRefusal { - RoomCapacity, - ConnectionCapacity, - RoomConnectionCapacity, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Ledger { - rooms: BTreeMap, - locations: LocationTable, - limits: LedgerLimits, - params: DecideParams, - tracked_connections: usize, - evicted_rooms_total: u64, - evicted_connections_total: u64, -} - -impl Ledger { - pub fn new(limits: LedgerLimits, params: DecideParams) -> Self { - Self { - rooms: BTreeMap::new(), - locations: LocationTable::new(), - limits, - params, - tracked_connections: 0, - evicted_rooms_total: 0, - evicted_connections_total: 0, - } - } - - pub const fn limits(&self) -> LedgerLimits { - self.limits - } - - pub const fn params(&self) -> &DecideParams { - &self.params - } - - pub const fn locations(&self) -> &LocationTable { - &self.locations - } - - pub fn intern_location(&mut self, location: &Location) -> Option { - self.locations.intern(location) - } - - pub fn location(&self, index: LocationIndex) -> Option<&Location> { - self.locations.get(index) - } - - pub fn location_index(&self, location: &Location) -> Option { - self.locations.index_of(location) - } - - pub fn rooms(&self) -> impl Iterator { - self.rooms.iter() - } - - pub fn room(&self, room: &RoomKey) -> Option<&RoomLedger> { - self.rooms.get(room) - } - - pub fn room_mut(&mut self, room: &RoomKey) -> Option<&mut RoomLedger> { - self.rooms.get_mut(room) - } - - pub fn connection(&self, key: &ConnectionKey) -> Option<&ConnectionLedger> { - self.rooms - .get(&key.room) - .and_then(|room| room.connection(&key.connection)) - } - - pub fn connection_mut(&mut self, key: &ConnectionKey) -> Option<&mut ConnectionLedger> { - self.rooms - .get_mut(&key.room) - .and_then(|room| room.connection_mut(&key.connection)) - } - - pub fn ensure_room( - &mut self, - room: RoomKey, - now: Millis, - ) -> Result<&mut RoomLedger, AdmissionRefusal> { - if !self.rooms.contains_key(&room) && self.rooms.len() >= self.limits.max_tracked_rooms { - return Err(AdmissionRefusal::RoomCapacity); - } - Ok(self - .rooms - .entry(room) - .or_insert_with(|| RoomLedger::new(room, now))) - } - - pub fn upsert_connection( - &mut self, - room: RoomKey, - connection: &ConnectionId, - user_id: UserId, - now: Millis, - ) -> Result<&mut ConnectionLedger, AdmissionRefusal> { - let limits = self.limits; - let tracked = self.tracked_connections; - let room_entry = self.ensure_room(room, now)?; - - let existing = room_entry - .connections - .iter() - .position(|entry| &entry.connection == connection); - - let index = match existing { - Some(index) => index, - None => { - if room_entry.connections.len() >= limits.max_connections_per_room { - room_entry.partially_unreadable = true; - return Err(AdmissionRefusal::RoomConnectionCapacity); - } - if tracked >= limits.max_tracked_connections { - return Err(AdmissionRefusal::ConnectionCapacity); - } - room_entry.connections.push(ConnectionLedger::new( - connection.clone(), - user_id, - now, - )); - self.tracked_connections = tracked.saturating_add(1); - self.rooms - .get(&room) - .map(|entry| entry.connections.len().saturating_sub(1)) - .unwrap_or(0) - } - }; - - let entry = self - .rooms - .get_mut(&room) - .and_then(|room_ledger| room_ledger.connections.get_mut(index)) - .expect("the connection was just admitted"); - entry.user_id = user_id; - entry.last_seen = now; - Ok(entry) - } - - pub fn tracked_rooms(&self) -> usize { - self.rooms.len() - } - - pub const fn tracked_connections(&self) -> usize { - self.tracked_connections - } - - pub fn tracked_bytes(&self) -> u64 { - (self.rooms.len() as u64) - .saturating_mul(ROOM_ENTRY_BYTES) - .saturating_add( - (self.tracked_connections as u64).saturating_mul(CONNECTION_ENTRY_BYTES), - ) - } - - pub const fn evicted_rooms_total(&self) -> u64 { - self.evicted_rooms_total - } - - pub const fn evicted_connections_total(&self) -> u64 { - self.evicted_connections_total - } - - pub fn divergent_connections(&self) -> usize { - self.rooms - .values() - .map(RoomLedger::divergent_connections) - .sum() - } - - pub fn divergent_fraction(&self) -> f64 { - if self.tracked_connections == 0 { - return 0.0; - } - self.divergent_connections() as f64 / self.tracked_connections as f64 - } - - pub fn reset_all_corroborations(&mut self) { - for room in self.rooms.values_mut() { - room.reset_corroborations(); - } - } - - pub fn prune_retired(&mut self, now: Millis) -> usize { - let grace = self.limits.retired_grace_ms; - let mut dropped = 0; - - for room in self.rooms.values_mut() { - room.connections.retain(|entry| match entry.state { - ConnectionState::Retired { at } => { - let expired = now.saturating_since(at) >= grace; - if expired { - dropped += 1; - } - !expired - } - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::GatewayOnly { .. } - | ConnectionState::MediaOnly { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::ActionTaken { .. } - | ConnectionState::Wedged { .. } => true, - }); - } - - self.tracked_connections = self.tracked_connections.saturating_sub(dropped); - self.evicted_connections_total = self - .evicted_connections_total - .saturating_add(dropped as u64); - dropped - } - - pub fn drop_empty_rooms(&mut self) -> usize { - let before = self.rooms.len(); - self.rooms.retain(|_, room| !room.connections.is_empty()); - before.saturating_sub(self.rooms.len()) - } - - pub fn evict_to_budget(&mut self) -> usize { - let mut evicted = 0; - - while self.tracked_bytes() > self.limits.memory_budget_bytes - || self.rooms.len() > self.limits.max_tracked_rooms - || self.tracked_connections > self.limits.max_tracked_connections - { - let Some(victim) = self.coldest_room() else { - break; - }; - if self.evict_room(&victim) { - evicted += 1; - } else { - break; - } - } - - evicted - } - - pub fn evict_room(&mut self, room: &RoomKey) -> bool { - match self.rooms.remove(room) { - None => false, - Some(entry) => { - self.tracked_connections = self - .tracked_connections - .saturating_sub(entry.connections.len()); - self.evicted_connections_total = self - .evicted_connections_total - .saturating_add(entry.connections.len() as u64); - self.evicted_rooms_total = self.evicted_rooms_total.saturating_add(1); - true - } - } - } - - pub fn coldest_room(&self) -> Option { - let cold = self - .rooms - .values() - .filter(|room| room.is_cold()) - .min_by_key(|room| (room.last_seen(), room.room())) - .map(RoomLedger::room); - - cold.or_else(|| { - self.rooms - .values() - .min_by_key(|room| (room.last_seen(), room.room())) - .map(RoomLedger::room) - }) - } - - pub fn record_action_issued( - &mut self, - key: &ConnectionKey, - kind: ActionKind, - at: Millis, - authorizing_turn: TurnId, - ) -> bool { - let Some(entry) = self.connection_mut(key) else { - return false; - }; - - let (prior, since) = match &entry.state { - ConnectionState::GatewayOnly { since, .. } => (PriorDivergence::GatewayOnly, *since), - ConnectionState::MediaOnly { since, .. } => (PriorDivergence::MediaOnly, *since), - ConnectionState::Nascent - | ConnectionState::Consistent - | ConnectionState::PendingJoin { .. } - | ConnectionState::Repairing { .. } - | ConnectionState::ActionTaken { .. } - | ConnectionState::Wedged { .. } - | ConnectionState::Retired { .. } => return false, - }; - - entry.action_attempts = entry.action_attempts.saturating_add(1); - entry.last_action_at = Some(at); - entry.state = ConnectionState::ActionTaken { - kind, - at, - authorizing_turn, - prior, - since, - }; - true - } - - pub fn refund_action_attempt(&mut self, key: &ConnectionKey) -> bool { - let Some(entry) = self.connection_mut(key) else { - return false; - }; - if entry.action_attempts == 0 { - return false; - } - entry.action_attempts = entry.action_attempts.saturating_sub(1); - true - } - - pub fn record_repair_issued(&mut self, key: &ConnectionKey, at: Millis) -> bool { - let Some(entry) = self.connection_mut(key) else { - return false; - }; - entry.repair_attempts = entry.repair_attempts.saturating_add(1); - entry.last_action_at = Some(at); - if let ConnectionState::Repairing { last_attempt, .. } = &mut entry.state { - *last_attempt = at; - } - true - } - - pub fn record_repair_verdict(&mut self, key: &ConnectionKey, verdict: RepairVerdict) -> bool { - let Some(entry) = self.connection_mut(key) else { - return false; - }; - entry.last_repair_verdict = Some(verdict); - true - } - - pub fn record_confirm_issued(&mut self, key: &ConnectionKey, at: Millis) -> bool { - let Some(entry) = self.connection_mut(key) else { - return false; - }; - entry.last_action_at = Some(at); - if let ConnectionState::PendingJoin { last_confirm, .. } = &mut entry.state { - *last_confirm = at; - } - true - } -} diff --git a/fluxer_recon/src/lib.rs b/fluxer_recon/src/lib.rs deleted file mode 100644 index c99b69f1a..000000000 --- a/fluxer_recon/src/lib.rs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -pub mod actuate; -pub mod budget; -pub mod census; -pub mod clock; -pub mod config; -pub mod control; -pub mod decide; -pub mod discovery; -pub mod evidence; -pub mod gateway; -pub mod guards; -pub mod health; -pub mod ids; -pub mod ledger; -pub mod livekit; -pub mod metrics; -pub mod names; -pub mod observe; -pub mod runtime; -pub mod schedule; -pub mod service; -pub mod singleton; -pub mod suspicion; -pub mod topology; -pub mod turn; -pub mod webhook; diff --git a/fluxer_recon/src/livekit/auth.rs b/fluxer_recon/src/livekit/auth.rs deleted file mode 100644 index 61dccec42..000000000 --- a/fluxer_recon/src/livekit/auth.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use base64::prelude::*; -use hmac::{Hmac, KeyInit, Mac}; -use serde::{Deserialize, Serialize}; -use sha2::Sha256; - -use crate::ids::ApiSecret; - -type HmacSha256 = Hmac; - -pub const ADMIN_TOKEN_TTL_SECONDS: u64 = 600; - -const HEADER_JSON: &str = r#"{"alg":"HS256","typ":"JWT"}"#; -const HEADER_ALG: &str = "HS256"; -const HEADER_TYP: &str = "JWT"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum TokenError { - #[error("token is not three dot-separated parts")] - Malformed, - #[error("token header is not an HS256 JWT header")] - UnsupportedHeader, - #[error("token carries invalid base64url")] - BadBase64, - #[error("token carries invalid json")] - BadJson, - #[error("token signature does not verify")] - BadSignature, - #[error("token issuer does not match the api key")] - IssuerMismatch, - #[error("token has expired")] - Expired, - #[error("token is not valid yet")] - NotYetValid, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum VideoGrant { - RoomList, - RoomAdmin { room: String }, -} - -impl VideoGrant { - pub fn claim(&self) -> VideoGrantClaim { - match self { - Self::RoomList => VideoGrantClaim { - room_list: Some(true), - room_admin: None, - room: None, - }, - Self::RoomAdmin { room } => VideoGrantClaim { - room_list: None, - room_admin: Some(true), - room: Some(room.clone()), - }, - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct VideoGrantClaim { - #[serde(rename = "roomList", default, skip_serializing_if = "Option::is_none")] - pub room_list: Option, - #[serde(rename = "roomAdmin", default, skip_serializing_if = "Option::is_none")] - pub room_admin: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub room: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AdminClaims { - pub iss: String, - pub exp: u64, - pub nbf: u64, - pub video: VideoGrantClaim, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -struct TokenHeader { - alg: String, - typ: String, -} - -pub fn admin_claims(api_key: &str, grant: &VideoGrant, now_unix_seconds: u64) -> AdminClaims { - AdminClaims { - iss: api_key.to_owned(), - exp: now_unix_seconds.saturating_add(ADMIN_TOKEN_TTL_SECONDS), - nbf: 0, - video: grant.claim(), - } -} - -pub fn mint_admin_token( - api_key: &str, - api_secret: &ApiSecret, - grant: &VideoGrant, - now_unix_seconds: u64, -) -> String { - encode_admin_token(&admin_claims(api_key, grant, now_unix_seconds), api_secret) -} - -pub fn encode_admin_token(claims: &AdminClaims, api_secret: &ApiSecret) -> String { - let header = BASE64_URL_SAFE_NO_PAD.encode(HEADER_JSON.as_bytes()); - let payload = BASE64_URL_SAFE_NO_PAD - .encode(serde_json::to_vec(claims).expect("admin claims serialise to json")); - let signing_input = format!("{header}.{payload}"); - let signature = sign(signing_input.as_bytes(), api_secret); - format!( - "{signing_input}.{}", - BASE64_URL_SAFE_NO_PAD.encode(signature) - ) -} - -pub fn verify_admin_token( - token: &str, - api_key: &str, - api_secret: &ApiSecret, - now_unix_seconds: u64, -) -> Result { - let mut parts = token.split('.'); - let (Some(header_b64), Some(payload_b64), Some(signature_b64), None) = - (parts.next(), parts.next(), parts.next(), parts.next()) - else { - return Err(TokenError::Malformed); - }; - - let header_bytes = decode_part(header_b64)?; - let header: TokenHeader = - serde_json::from_slice(&header_bytes).map_err(|_| TokenError::BadJson)?; - if header.alg != HEADER_ALG || header.typ != HEADER_TYP { - return Err(TokenError::UnsupportedHeader); - } - - let signature = decode_part(signature_b64)?; - let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) - .expect("hmac accepts any key length"); - mac.update(header_b64.as_bytes()); - mac.update(b"."); - mac.update(payload_b64.as_bytes()); - mac.verify_slice(&signature) - .map_err(|_| TokenError::BadSignature)?; - - let payload_bytes = decode_part(payload_b64)?; - let claims: AdminClaims = - serde_json::from_slice(&payload_bytes).map_err(|_| TokenError::BadJson)?; - if claims.iss != api_key { - return Err(TokenError::IssuerMismatch); - } - if now_unix_seconds >= claims.exp { - return Err(TokenError::Expired); - } - if now_unix_seconds < claims.nbf { - return Err(TokenError::NotYetValid); - } - Ok(claims) -} - -fn decode_part(part: &str) -> Result, TokenError> { - BASE64_URL_SAFE_NO_PAD - .decode(part.as_bytes()) - .map_err(|_| TokenError::BadBase64) -} - -fn sign(input: &[u8], api_secret: &ApiSecret) -> Vec { - let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) - .expect("hmac accepts any key length"); - mac.update(input); - mac.finalize().into_bytes().to_vec() -} diff --git a/fluxer_recon/src/livekit/mod.rs b/fluxer_recon/src/livekit/mod.rs deleted file mode 100644 index 8b5c72fe7..000000000 --- a/fluxer_recon/src/livekit/mod.rs +++ /dev/null @@ -1,179 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -pub mod auth; -pub mod twirp; - -use std::future::Future; - -use crate::ids::{ApiSecret, Location, RoomKey}; -use crate::names::ParticipantIdentity; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum LiveKitFault { - Retryable { status: u16 }, - NotFound, - AuthFailed, - Other { status: u16 }, - Transport { timeout: bool }, - ServerMissing, - MalformedEndpoint, -} - -impl LiveKitFault { - pub const fn label(self) -> &'static str { - match self { - Self::Retryable { .. } => "retryable", - Self::NotFound => "not_found", - Self::AuthFailed => "auth_failed", - Self::Other { .. } => "other", - Self::Transport { timeout: true } => "transport_timeout", - Self::Transport { timeout: false } => "transport_refused", - Self::ServerMissing => "server_missing", - Self::MalformedEndpoint => "malformed_endpoint", - } - } - - pub const fn is_retryable(self) -> bool { - match self { - Self::Retryable { .. } | Self::Transport { .. } => true, - Self::NotFound - | Self::AuthFailed - | Self::Other { .. } - | Self::ServerMissing - | Self::MalformedEndpoint => false, - } - } - - pub const fn is_auth_failure(self) -> bool { - matches!(self, Self::AuthFailed) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ReadResult { - Read(T), - Unreadable(LiveKitFault), -} - -impl ReadResult { - pub fn is_readable(&self) -> bool { - matches!(self, Self::Read(_)) - } - - pub fn as_read(&self) -> Option<&T> { - match self { - Self::Read(value) => Some(value), - Self::Unreadable(_) => None, - } - } - - pub fn into_read(self) -> Option { - match self { - Self::Read(value) => Some(value), - Self::Unreadable(_) => None, - } - } - - pub fn fault(&self) -> Option { - match self { - Self::Read(_) => None, - Self::Unreadable(fault) => Some(*fault), - } - } - - pub fn map(self, transform: impl FnOnce(T) -> U) -> ReadResult { - match self { - Self::Read(value) => ReadResult::Read(transform(value)), - Self::Unreadable(fault) => ReadResult::Unreadable(fault), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum RemoveOutcome { - Removed, - AlreadyGone, - Failed(LiveKitFault), -} - -impl RemoveOutcome { - pub const fn is_success(self) -> bool { - match self { - Self::Removed | Self::AlreadyGone => true, - Self::Failed(_) => false, - } - } - - pub const fn fault(self) -> Option { - match self { - Self::Removed | Self::AlreadyGone => None, - Self::Failed(fault) => Some(fault), - } - } - - pub const fn label(self) -> &'static str { - match self { - Self::Removed => "removed", - Self::AlreadyGone => "already_gone", - Self::Failed(_) => "failed", - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum ParticipantState { - #[default] - Joining, - Joined, - Active, - Disconnected, - Unrecognised, -} - -impl ParticipantState { - pub const fn label(self) -> &'static str { - match self { - Self::Joining => "joining", - Self::Joined => "joined", - Self::Active => "active", - Self::Disconnected => "disconnected", - Self::Unrecognised => "unrecognised", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ParticipantRecord { - pub identity: Box, - pub joined_at_unix_seconds: u64, - pub state: ParticipantState, - pub is_publisher: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ServerCredentials { - pub location: Location, - pub endpoint: Box, - pub api_key: Box, - pub api_secret: ApiSecret, -} - -pub trait LiveKitApi { - fn list_rooms( - &self, - location: &Location, - ) -> impl Future>>> + Send; - - fn list_participants( - &self, - location: &Location, - room: RoomKey, - ) -> impl Future>> + Send; - - fn remove_participant( - &self, - location: &Location, - room: RoomKey, - identity: &ParticipantIdentity, - ) -> impl Future + Send; -} diff --git a/fluxer_recon/src/livekit/twirp.rs b/fluxer_recon/src/livekit/twirp.rs deleted file mode 100644 index 76e8a4cbe..000000000 --- a/fluxer_recon/src/livekit/twirp.rs +++ /dev/null @@ -1,482 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::HashMap; -use std::fmt; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use serde::de::{self, Visitor}; -use serde::{Deserialize, Deserializer, Serialize}; - -use crate::ids::{ApiSecret, Location, RoomKey}; -use crate::livekit::auth::{VideoGrant, mint_admin_token}; -use crate::livekit::{ - LiveKitApi, LiveKitFault, ParticipantRecord, ParticipantState, ReadResult, RemoveOutcome, - ServerCredentials, -}; -use crate::names::{ParticipantIdentity, format_participant_identity, format_room_name}; - -pub const TWIRP_PREFIX: &str = "/twirp/livekit.RoomService/"; -pub const METHOD_LIST_ROOMS: &str = "ListRooms"; -pub const METHOD_LIST_PARTICIPANTS: &str = "ListParticipants"; -pub const METHOD_REMOVE_PARTICIPANT: &str = "RemoveParticipant"; - -const MAX_IDLE_CONNECTIONS_PER_HOST: usize = 1; -const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(90); - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TwirpTimeouts { - pub connect: Duration, - pub request: Duration, -} - -impl TwirpTimeouts { - pub const DEFAULT: Self = Self { - connect: Duration::from_secs(3), - request: Duration::from_secs(5), - }; -} - -impl Default for TwirpTimeouts { - fn default() -> Self { - Self::DEFAULT - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum EndpointError { - #[error("endpoint scheme is not ws, wss, http or https")] - UnsupportedScheme, - #[error("endpoint names no host")] - MissingHost, - #[error("endpoint carries a query string or fragment")] - QueryOrFragment, - #[error("endpoint contains whitespace")] - ContainsWhitespace, -} - -pub fn twirp_base(endpoint: &str) -> Result { - if endpoint.contains(char::is_whitespace) { - return Err(EndpointError::ContainsWhitespace); - } - if endpoint.contains('?') || endpoint.contains('#') { - return Err(EndpointError::QueryOrFragment); - } - - let rewritten = if let Some(rest) = endpoint.strip_prefix("wss://") { - format!("https://{rest}") - } else if let Some(rest) = endpoint.strip_prefix("ws://") { - format!("http://{rest}") - } else if endpoint.starts_with("https://") || endpoint.starts_with("http://") { - endpoint.to_owned() - } else { - return Err(EndpointError::UnsupportedScheme); - }; - - let authority_and_path = rewritten - .split_once("://") - .expect("a rewritten endpoint always carries a scheme separator") - .1; - let host = authority_and_path.split('/').next().unwrap_or_default(); - if host.is_empty() { - return Err(EndpointError::MissingHost); - } - - Ok(rewritten.trim_end_matches('/').to_owned()) -} - -pub fn twirp_url(endpoint: &str, method: &str) -> Result { - twirp_base(endpoint).map(|base| format!("{base}{TWIRP_PREFIX}{method}")) -} - -pub const fn classify_status(status: u16) -> LiveKitFault { - match status { - 401 | 403 => LiveKitFault::AuthFailed, - 404 => LiveKitFault::NotFound, - 500..=599 => LiveKitFault::Retryable { status }, - other => LiveKitFault::Other { status: other }, - } -} - -#[derive(Serialize)] -struct ListRoomsRequest { - names: [&'static str; 0], -} - -#[derive(Serialize)] -struct RoomRequest<'a> { - room: &'a str, -} - -#[derive(Serialize)] -struct RemoveParticipantRequest<'a> { - room: &'a str, - identity: &'a str, -} - -#[derive(Debug, Deserialize)] -struct ListRoomsResponse { - #[serde(default)] - rooms: Vec, -} - -#[derive(Debug, Deserialize)] -struct RoomWire { - #[serde(default)] - name: String, -} - -#[derive(Debug, Deserialize)] -struct ListParticipantsResponse { - #[serde(default)] - participants: Vec, -} - -#[derive(Debug, Deserialize)] -struct ParticipantWire { - #[serde(default)] - identity: String, - #[serde( - rename = "joinedAt", - alias = "joined_at", - default, - deserialize_with = "protojson_int64" - )] - joined_at: u64, - #[serde(default)] - state: ParticipantState, - #[serde(rename = "isPublisher", alias = "is_publisher", default)] - is_publisher: bool, -} - -fn protojson_int64<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - struct Int64Visitor; - - impl Visitor<'_> for Int64Visitor { - type Value = u64; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a protojson int64 as a json number or a decimal string") - } - - fn visit_u64(self, value: u64) -> Result { - Ok(value) - } - - fn visit_i64(self, value: i64) -> Result { - Ok(u64::try_from(value).unwrap_or_default()) - } - - fn visit_f64(self, value: f64) -> Result { - Err(E::custom(format!("int64 encoded as a float: {value}"))) - } - - fn visit_str(self, value: &str) -> Result { - value - .parse::() - .map(|seconds| u64::try_from(seconds).unwrap_or_default()) - .map_err(|_| E::custom(format!("int64 string is not decimal: {value}"))) - } - } - - deserializer.deserialize_any(Int64Visitor) -} - -impl<'de> Deserialize<'de> for ParticipantState { - fn deserialize>(deserializer: D) -> Result { - struct StateVisitor; - - impl Visitor<'_> for StateVisitor { - type Value = ParticipantState; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a protobuf enum as its name or its number") - } - - fn visit_u64(self, value: u64) -> Result { - Ok(match value { - 0 => ParticipantState::Joining, - 1 => ParticipantState::Joined, - 2 => ParticipantState::Active, - 3 => ParticipantState::Disconnected, - _ => ParticipantState::Unrecognised, - }) - } - - fn visit_i64(self, value: i64) -> Result { - match u64::try_from(value) { - Ok(value) => self.visit_u64(value), - Err(_) => Ok(ParticipantState::Unrecognised), - } - } - - fn visit_str(self, value: &str) -> Result { - Ok(match value { - "JOINING" => ParticipantState::Joining, - "JOINED" => ParticipantState::Joined, - "ACTIVE" => ParticipantState::Active, - "DISCONNECTED" => ParticipantState::Disconnected, - _ => ParticipantState::Unrecognised, - }) - } - } - - deserializer.deserialize_any(StateVisitor) - } -} - -fn decode_rooms(body: &str) -> Result>, serde_json::Error> { - let response: ListRoomsResponse = serde_json::from_str(body)?; - Ok(response - .rooms - .into_iter() - .filter(|room| !room.name.is_empty()) - .map(|room| Box::from(room.name.as_str())) - .collect()) -} - -fn decode_participants(body: &str) -> Result, serde_json::Error> { - let response: ListParticipantsResponse = serde_json::from_str(body)?; - Ok(response - .participants - .into_iter() - .map(|participant| ParticipantRecord { - identity: Box::from(participant.identity.as_str()), - joined_at_unix_seconds: participant.joined_at, - state: participant.state, - is_publisher: participant.is_publisher, - }) - .collect()) -} - -#[derive(Debug)] -enum HttpOutcome { - Success { status: u16, body: String }, - Status { status: u16 }, - Transport { fault: LiveKitFault }, -} - -fn transport_fault(error: &reqwest::Error) -> LiveKitFault { - if error.is_connect() { - return LiveKitFault::Transport { timeout: false }; - } - if error.is_timeout() { - return LiveKitFault::Transport { timeout: true }; - } - LiveKitFault::Transport { timeout: false } -} - -fn current_unix_seconds() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()) -} - -#[derive(Debug)] -struct ServerClient { - base: String, - api_key: Box, - api_secret: ApiSecret, - http: reqwest::Client, -} - -impl ServerClient { - fn url(&self, method: &str) -> String { - format!("{}{TWIRP_PREFIX}{method}", self.base) - } - - async fn post(&self, method: &str, grant: &VideoGrant, body: &impl Serialize) -> HttpOutcome { - let token = mint_admin_token( - &self.api_key, - &self.api_secret, - grant, - current_unix_seconds(), - ); - let response = match self - .http - .post(self.url(method)) - .bearer_auth(token) - .json(body) - .send() - .await - { - Ok(response) => response, - Err(error) => { - return HttpOutcome::Transport { - fault: transport_fault(&error), - }; - } - }; - - let status = response.status(); - if !status.is_success() { - return HttpOutcome::Status { - status: status.as_u16(), - }; - } - match response.text().await { - Ok(body) => HttpOutcome::Success { - status: status.as_u16(), - body, - }, - Err(error) => HttpOutcome::Transport { - fault: transport_fault(&error), - }, - } - } -} - -#[derive(Debug)] -enum ServerEntry { - Ready(Box), - Unusable(LiveKitFault), -} - -#[derive(Debug)] -pub struct TwirpLiveKit { - servers: HashMap, - timeouts: TwirpTimeouts, -} - -impl TwirpLiveKit { - pub fn new(servers: Vec, timeouts: TwirpTimeouts) -> anyhow::Result { - let mut adapter = Self { - servers: HashMap::new(), - timeouts, - }; - adapter.replace_servers(servers)?; - Ok(adapter) - } - - pub fn replace_servers(&mut self, servers: Vec) -> anyhow::Result<()> { - let mut replacement: HashMap = HashMap::with_capacity(servers.len()); - for credentials in servers { - let entry = match twirp_base(&credentials.endpoint) { - Ok(base) => ServerEntry::Ready(Box::new(ServerClient { - base, - api_key: credentials.api_key, - api_secret: credentials.api_secret, - http: self.build_http_client()?, - })), - Err(error) => { - tracing::warn!( - location = %credentials.location, - %error, - "livekit endpoint is unusable" - ); - ServerEntry::Unusable(LiveKitFault::MalformedEndpoint) - } - }; - replacement.insert(credentials.location, entry); - } - self.servers = replacement; - Ok(()) - } - - fn build_http_client(&self) -> anyhow::Result { - Ok(reqwest::Client::builder() - .connect_timeout(self.timeouts.connect) - .timeout(self.timeouts.request) - .pool_max_idle_per_host(MAX_IDLE_CONNECTIONS_PER_HOST) - .pool_idle_timeout(POOL_IDLE_TIMEOUT) - .build()?) - } - - fn client(&self, location: &Location) -> Result<&ServerClient, LiveKitFault> { - match self.servers.get(location) { - None => Err(LiveKitFault::ServerMissing), - Some(ServerEntry::Unusable(fault)) => Err(*fault), - Some(ServerEntry::Ready(client)) => Ok(client), - } - } -} - -impl LiveKitApi for TwirpLiveKit { - async fn list_rooms(&self, location: &Location) -> ReadResult>> { - let client = match self.client(location) { - Ok(client) => client, - Err(fault) => return ReadResult::Unreadable(fault), - }; - - let outcome = client - .post( - METHOD_LIST_ROOMS, - &VideoGrant::RoomList, - &ListRoomsRequest { names: [] }, - ) - .await; - - match outcome { - HttpOutcome::Transport { fault } => ReadResult::Unreadable(fault), - HttpOutcome::Status { status } => ReadResult::Unreadable(classify_status(status)), - HttpOutcome::Success { status, body } => match decode_rooms(&body) { - Ok(rooms) => ReadResult::Read(rooms), - Err(_) => ReadResult::Unreadable(LiveKitFault::Other { status }), - }, - } - } - - async fn list_participants( - &self, - location: &Location, - room: RoomKey, - ) -> ReadResult> { - let client = match self.client(location) { - Ok(client) => client, - Err(fault) => return ReadResult::Unreadable(fault), - }; - - let room_name = format_room_name(room); - let outcome = client - .post( - METHOD_LIST_PARTICIPANTS, - &VideoGrant::RoomAdmin { - room: room_name.clone(), - }, - &RoomRequest { room: &room_name }, - ) - .await; - - match outcome { - HttpOutcome::Transport { fault } => ReadResult::Unreadable(fault), - HttpOutcome::Status { status } => ReadResult::Unreadable(classify_status(status)), - HttpOutcome::Success { status, body } => match decode_participants(&body) { - Ok(participants) => ReadResult::Read(participants), - Err(_) => ReadResult::Unreadable(LiveKitFault::Other { status }), - }, - } - } - - async fn remove_participant( - &self, - location: &Location, - room: RoomKey, - identity: &ParticipantIdentity, - ) -> RemoveOutcome { - let client = match self.client(location) { - Ok(client) => client, - Err(fault) => return RemoveOutcome::Failed(fault), - }; - - let room_name = format_room_name(room); - let participant_identity = format_participant_identity(identity); - let outcome = client - .post( - METHOD_REMOVE_PARTICIPANT, - &VideoGrant::RoomAdmin { - room: room_name.clone(), - }, - &RemoveParticipantRequest { - room: &room_name, - identity: &participant_identity, - }, - ) - .await; - - match outcome { - HttpOutcome::Transport { fault } => RemoveOutcome::Failed(fault), - HttpOutcome::Success { .. } => RemoveOutcome::Removed, - HttpOutcome::Status { status: 404 } => RemoveOutcome::AlreadyGone, - HttpOutcome::Status { status } => RemoveOutcome::Failed(classify_status(status)), - } - } -} diff --git a/fluxer_recon/src/main.rs b/fluxer_recon/src/main.rs deleted file mode 100644 index e09ab441f..000000000 --- a/fluxer_recon/src/main.rs +++ /dev/null @@ -1,155 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::net::SocketAddr; -use std::sync::Arc; - -use tokio::task::JoinSet; - -use fluxer_recon::clock::{SharedClock, SystemClock}; -use fluxer_recon::config::ReconConfig; -use fluxer_recon::control::{CONTROL_SUBJECT, INSTANCE_SUBJECT, run_control}; -use fluxer_recon::gateway::nats::NatsGateway; -use fluxer_recon::metrics::ReconMetrics; -use fluxer_recon::runtime::{ - InstanceIdentity, Shared, boot_assertions, monotonic_now, run_singleton, runtime_tasks, -}; -use fluxer_recon::service::{ - GuardedClock, ReconEngine, SharedFleet, build_livekit, connect_topology, internal_endpoint, - run_census, run_discovery, run_engine, -}; -use fluxer_recon::webhook::run_webhook; -use fluxer_svc::config::ServiceConfig; -use fluxer_svc::metrics::ServiceMetrics; -use fluxer_svc::shutdown::{DEFAULT_DRAIN_TIMEOUT, drain_with_timeout, wait_for_shutdown}; -use fluxer_svc::transport::NatsTransport; - -enum Exit { - Task(anyhow::Result<()>), - Signal, -} - -fn main() -> anyhow::Result<()> { - fluxer_svc::init_tracing(); - - let service = ServiceConfig::from_env()?; - let recon = ReconConfig::from_env()?; - boot_assertions(&service, &recon)?; - - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(recon.worker_threads) - .max_blocking_threads(1) - .thread_name("recon") - .enable_all() - .build()?; - - runtime.block_on(serve(service, recon)) -} - -async fn serve(service: ServiceConfig, recon: ReconConfig) -> anyhow::Result<()> { - let transport = - NatsTransport::connect(&service.nats_url, service.nats_auth_token.as_deref()).await?; - - let clock: SharedClock = GuardedClock::shared(SystemClock::shared()); - let instance = InstanceIdentity::generate(); - let shared = Shared::new(recon.clone(), instance, monotonic_now()); - - let metrics = Arc::new(ServiceMetrics::with_additional_renderer( - ReconMetrics::renderer(shared.metrics()), - )); - metrics.init(); - - let topology = connect_topology(&service).await?; - let internal = internal_endpoint(&recon); - let fleet = SharedFleet::new(build_livekit(&[], &internal)?); - - tracing::info!( - service = service.service_name, - listen_addr = %service.listen_addr, - instance_id = shared.instance().id, - mode = recon.mode.as_str(), - worker_threads = recon.worker_threads, - tick_ms = recon.tick_ms, - coverage_target_ms = recon.coverage_target_ms, - topology_backend = topology.backend(), - livekit_internal_endpoint = internal.url.as_deref().unwrap_or("none"), - livekit_default_region = internal.default_region_id.as_deref().unwrap_or("none"), - control_subject = CONTROL_SUBJECT, - instance_subject = INSTANCE_SUBJECT, - tasks = ?runtime_tasks(&recon), - "starting recon service" - ); - - let mut tasks: JoinSet> = JoinSet::new(); - - let health_addr = service.listen_addr; - let health_serving = shared.readiness().flag(); - let health_metrics = metrics.clone(); - let health_name = service.service_name.clone(); - tasks.spawn(async move { - fluxer_svc::server::run_http(health_addr, health_serving, health_metrics, health_name).await - }); - - let engine = ReconEngine::new( - shared.clone(), - NatsGateway::new(transport.clone()), - fleet.clone(), - clock.clone(), - ); - tasks.spawn(async move { run_engine(engine).await }); - - let census_shared = shared.clone(); - let census_gateway = NatsGateway::new(transport.clone()); - let census_clock = clock.clone(); - tasks.spawn(async move { run_census(census_shared, census_gateway, census_clock).await }); - - let discovery_shared = shared.clone(); - let discovery_fleet = fleet.clone(); - let discovery_clock = clock.clone(); - tasks.spawn(async move { - run_discovery(discovery_shared, discovery_fleet, topology, discovery_clock).await - }); - - let control_shared = shared.clone(); - let control_transport = transport.clone(); - tasks.spawn(async move { run_control(control_shared, control_transport).await }); - - let singleton_shared = shared.clone(); - let singleton_transport = transport.clone(); - tasks.spawn(async move { run_singleton(singleton_shared, singleton_transport).await }); - - if recon.webhook_enabled { - let webhook_shared = shared.clone(); - let webhook_clock = clock.clone(); - let webhook_addr = SocketAddr::from((service.listen_addr.ip(), recon.webhook_port)); - tasks.spawn(async move { run_webhook(webhook_shared, webhook_clock, webhook_addr).await }); - } - - let exit = tokio::select! { - result = tasks.join_next() => Exit::Task(match result { - Some(Ok(Ok(()))) => Ok(()), - Some(Ok(Err(error))) => Err(error), - Some(Err(error)) => Err(error.into()), - None => Ok(()), - }), - () = wait_for_shutdown() => Exit::Signal, - }; - - shared.readiness().begin_shutdown(); - tasks.abort_all(); - drain_with_timeout( - async { while tasks.join_next().await.is_some() {} }, - DEFAULT_DRAIN_TIMEOUT, - ) - .await; - - match exit { - Exit::Task(result) => { - tracing::warn!("a recon task ended, shutting the service down"); - result - } - Exit::Signal => { - tracing::info!("recon shutdown complete"); - Ok(()) - } - } -} diff --git a/fluxer_recon/src/metrics.rs b/fluxer_recon/src/metrics.rs deleted file mode 100644 index d8e4ad274..000000000 --- a/fluxer_recon/src/metrics.rs +++ /dev/null @@ -1,604 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; -use std::fmt::Write; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; - -use fluxer_svc::metrics::AdditionalMetricsRenderer; - -use crate::decide::{Decision, DecisionAction}; -use crate::evidence::SideKind; -use crate::guards::{AbortReason, UnknownMediaCause}; -use crate::health::{ModeClamp, ReconMode, ServerHealth}; -use crate::ids::{Location, Scope}; -use crate::ledger::{ActionKind, RepairVerdict}; - -const PREFIX: &str = "fluxer_recon"; -const ORDERING: Ordering = Ordering::Relaxed; - -const DURATION_BUCKETS_MS: [u64; 13] = [ - 1, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, -]; - -pub const DECISION_LABELS: [&str; 5] = [ - "hold", - "confirm_connection", - "repair_state", - "remove_gateway_state", - "remove_participant", -]; - -pub const REPAIR_OUTCOMES: [&str; 4] = ["repaired", "no_change", "not_repairable", "failed"]; - -pub const CONFIRM_OUTCOMES: [&str; 4] = - ["confirmed", "already_confirmed", "call_not_found", "failed"]; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum MutationOutcome { - Issued, - Succeeded, - NoChange, - Failed, -} - -impl MutationOutcome { - pub const ALL: [Self; 4] = [Self::Issued, Self::Succeeded, Self::NoChange, Self::Failed]; - - pub const fn label(self) -> &'static str { - match self { - Self::Issued => "issued", - Self::Succeeded => "succeeded", - Self::NoChange => "no_change", - Self::Failed => "failed", - } - } -} - -pub const fn scope_label(scope: Scope) -> &'static str { - match scope { - Scope::Guild => "guild", - Scope::Dm => "dm", - } -} - -#[derive(Debug, Default)] -struct Counter(AtomicU64); - -impl Counter { - fn increment(&self) { - self.0.fetch_add(1, ORDERING); - } - - fn set(&self, value: u64) { - self.0.store(value, ORDERING); - } - - fn get(&self) -> u64 { - self.0.load(ORDERING) - } -} - -#[derive(Debug, Default)] -struct Gauge(AtomicU64); - -impl Gauge { - fn set(&self, value: f64) { - self.0.store(value.to_bits(), ORDERING); - } - - fn get(&self) -> f64 { - f64::from_bits(self.0.load(ORDERING)) - } -} - -#[derive(Debug, Default)] -struct LabelledSeries { - values: Mutex>, -} - -impl LabelledSeries { - fn seeded>(labels: I) -> Self { - let series = Self::default(); - for label in labels { - series.set(label, 0.0); - } - series - } - - fn lock(&self) -> std::sync::MutexGuard<'_, BTreeMap> { - self.values.lock().unwrap_or_else(PoisonError::into_inner) - } - - fn increment(&self, labels: String) { - *self.lock().entry(labels).or_insert(0.0) += 1.0; - } - - fn set(&self, labels: String, value: f64) { - self.lock().insert(labels, value); - } - - fn get(&self, labels: &str) -> Option { - self.lock().get(labels).copied() - } - - fn render(&self, out: &mut String, name: &str, kind: &str) { - let values = self.lock(); - let _ = writeln!(out, "# TYPE {PREFIX}_{name} {kind}"); - for (labels, value) in values.iter() { - let _ = writeln!(out, "{PREFIX}_{name}{{{labels}}} {value}"); - } - } -} - -#[derive(Debug)] -struct DurationHistogram { - buckets: [AtomicU64; DURATION_BUCKETS_MS.len()], - count: AtomicU64, - sum_ms: AtomicU64, -} - -impl Default for DurationHistogram { - fn default() -> Self { - Self { - buckets: [const { AtomicU64::new(0) }; DURATION_BUCKETS_MS.len()], - count: AtomicU64::new(0), - sum_ms: AtomicU64::new(0), - } - } -} - -impl DurationHistogram { - fn observe(&self, ms: u64) { - for (index, upper) in DURATION_BUCKETS_MS.iter().copied().enumerate() { - if ms <= upper { - self.buckets[index].fetch_add(1, ORDERING); - break; - } - } - self.count.fetch_add(1, ORDERING); - self.sum_ms.fetch_add(ms, ORDERING); - } - - fn render(&self, out: &mut String, name: &str) { - let _ = writeln!(out, "# TYPE {PREFIX}_{name} histogram"); - let mut cumulative = 0; - for (index, upper) in DURATION_BUCKETS_MS.iter().copied().enumerate() { - cumulative += self.buckets[index].load(ORDERING); - let _ = writeln!(out, "{PREFIX}_{name}_bucket{{le=\"{upper}\"}} {cumulative}"); - } - let count = self.count.load(ORDERING); - let _ = writeln!(out, "{PREFIX}_{name}_bucket{{le=\"+Inf\"}} {count}"); - let _ = writeln!(out, "{PREFIX}_{name}_sum {}", self.sum_ms.load(ORDERING)); - let _ = writeln!(out, "{PREFIX}_{name}_count {count}"); - } -} - -fn escape_label(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for character in value.chars() { - match character { - '\\' => escaped.push_str("\\\\"), - '"' => escaped.push_str("\\\""), - '\n' => escaped.push_str("\\n"), - other => escaped.push(other), - } - } - escaped -} - -fn one_label(key: &str, value: &str) -> String { - format!("{key}=\"{}\"", escape_label(value)) -} - -fn two_labels(first: (&str, &str), second: (&str, &str)) -> String { - format!( - "{}=\"{}\",{}=\"{}\"", - first.0, - escape_label(first.1), - second.0, - escape_label(second.1) - ) -} - -fn three_labels(first: (&str, &str), second: (&str, &str), third: (&str, &str)) -> String { - format!( - "{}=\"{}\",{}=\"{}\",{}=\"{}\"", - first.0, - escape_label(first.1), - second.0, - escape_label(second.1), - third.0, - escape_label(third.1) - ) -} - -#[derive(Debug)] -pub struct ReconMetrics { - decisions: LabelledSeries, - evictions_aborted: LabelledSeries, - unknown_media_causes: LabelledSeries, - mutations: LabelledSeries, - repairs: LabelledSeries, - confirms: LabelledSeries, - unknown: LabelledSeries, - gateway_rpc: LabelledSeries, - livekit_calls: LabelledSeries, - server_health: LabelledSeries, - webhook_last_event_age_seconds: LabelledSeries, - mode: LabelledSeries, - mode_clamp: LabelledSeries, - ledger_evicted_total: Counter, - scheduler_starved_total: Counter, - tripped_total: Counter, - peer_detected_total: Counter, - rooms_unknown_ratio: Gauge, - oldest_unknown_room_seconds: Gauge, - rooms_no_candidates: Gauge, - wedged_connections: Gauge, - ledger_rooms: Gauge, - ledger_connections: Gauge, - ledger_bytes: Gauge, - coverage_period_seconds: Gauge, - room_staleness_seconds: Gauge, - topology_age_seconds: Gauge, - census_age_seconds: Gauge, - last_decision_age_seconds: Gauge, - turn_duration: DurationHistogram, - mutation_duration: DurationHistogram, -} - -impl Default for ReconMetrics { - fn default() -> Self { - Self::new() - } -} - -impl ReconMetrics { - pub fn new() -> Self { - Self { - decisions: LabelledSeries::seeded( - DECISION_LABELS - .iter() - .map(|decision| one_label("decision", decision)), - ), - evictions_aborted: LabelledSeries::seeded( - AbortReason::ALL - .iter() - .map(|reason| one_label("reason", reason.label())), - ), - unknown_media_causes: LabelledSeries::seeded( - UnknownMediaCause::ALL - .iter() - .map(|cause| one_label("cause", cause.label())), - ), - mutations: LabelledSeries::default(), - repairs: LabelledSeries::seeded( - REPAIR_OUTCOMES - .iter() - .map(|outcome| one_label("outcome", outcome)), - ), - confirms: LabelledSeries::seeded( - CONFIRM_OUTCOMES - .iter() - .map(|outcome| one_label("outcome", outcome)), - ), - unknown: LabelledSeries::default(), - gateway_rpc: LabelledSeries::default(), - livekit_calls: LabelledSeries::default(), - server_health: LabelledSeries::default(), - webhook_last_event_age_seconds: LabelledSeries::default(), - mode: LabelledSeries::seeded( - [ - ReconMode::Halted, - ReconMode::Observing, - ReconMode::Constructive, - ReconMode::Enforcing, - ] - .iter() - .map(|mode| one_label("mode", mode.as_str())), - ), - mode_clamp: LabelledSeries::seeded( - ModeClamp::ALL - .iter() - .map(|clamp| one_label("clamp", clamp.label())), - ), - ledger_evicted_total: Counter::default(), - scheduler_starved_total: Counter::default(), - tripped_total: Counter::default(), - peer_detected_total: Counter::default(), - rooms_unknown_ratio: Gauge::default(), - oldest_unknown_room_seconds: Gauge::default(), - rooms_no_candidates: Gauge::default(), - wedged_connections: Gauge::default(), - ledger_rooms: Gauge::default(), - ledger_connections: Gauge::default(), - ledger_bytes: Gauge::default(), - coverage_period_seconds: Gauge::default(), - room_staleness_seconds: Gauge::default(), - topology_age_seconds: Gauge::default(), - census_age_seconds: Gauge::default(), - last_decision_age_seconds: Gauge::default(), - turn_duration: DurationHistogram::default(), - mutation_duration: DurationHistogram::default(), - } - } - - pub fn renderer(metrics: &Arc) -> AdditionalMetricsRenderer { - let metrics = Arc::clone(metrics); - Arc::new(move |out: &mut String| metrics.render(out)) - } - - pub fn record_decision(&self, decision: &Decision) { - self.decisions - .increment(one_label("decision", decision.action.label())); - if let Some(reason) = decision.blocked_by - && decision.is_destructive() - { - self.evictions_aborted - .increment(one_label("reason", reason.label())); - } - } - - pub fn record_eviction_aborted(&self, reason: AbortReason) { - self.evictions_aborted - .increment(one_label("reason", reason.label())); - } - - pub fn record_unknown_media_cause(&self, cause: UnknownMediaCause) { - self.unknown_media_causes - .increment(one_label("cause", cause.label())); - } - - pub fn unknown_media_cause_count(&self, cause: UnknownMediaCause) -> u64 { - self.unknown_media_causes - .get(&one_label("cause", cause.label())) - .unwrap_or(0.0) as u64 - } - - pub fn record_mutation(&self, kind: ActionKind, scope: Scope, outcome: MutationOutcome) { - self.mutations.increment(three_labels( - ("kind", kind.label()), - ("scope", scope_label(scope)), - ("outcome", outcome.label()), - )); - } - - pub fn record_repair(&self, outcome: &str) { - self.repairs.increment(one_label("outcome", outcome)); - } - - pub fn record_repair_verdict(&self, verdict: RepairVerdict) { - self.record_repair(verdict.label()); - } - - pub fn record_confirm(&self, outcome: &str) { - self.confirms.increment(one_label("outcome", outcome)); - } - - pub fn record_unknown(&self, side: SideKind, reason: &str) { - self.unknown - .increment(two_labels(("side", side.label()), ("reason", reason))); - } - - pub fn record_gateway_rpc(&self, method: &str, outcome: &str) { - self.gateway_rpc - .increment(two_labels(("method", method), ("outcome", outcome))); - } - - pub fn record_livekit_call(&self, method: &str, outcome: &str) { - self.livekit_calls - .increment(two_labels(("method", method), ("outcome", outcome))); - } - - pub fn record_breaker_trip(&self) { - self.tripped_total.increment(); - } - - pub fn record_peer_detected(&self) { - self.peer_detected_total.increment(); - } - - pub fn record_scheduler_starvation(&self) { - self.scheduler_starved_total.increment(); - } - - pub fn observe_turn_duration(&self, ms: u64) { - self.turn_duration.observe(ms); - } - - pub fn observe_mutation_duration(&self, ms: u64) { - self.mutation_duration.observe(ms); - } - - pub fn set_mode(&self, effective: ReconMode) { - for mode in [ - ReconMode::Halted, - ReconMode::Observing, - ReconMode::Constructive, - ReconMode::Enforcing, - ] { - let value = f64::from(u8::from(mode == effective)); - self.mode.set(one_label("mode", mode.as_str()), value); - } - } - - pub fn set_mode_clamp(&self, clamp: ModeClamp, engaged: bool) { - self.mode_clamp.set( - one_label("clamp", clamp.label()), - f64::from(u8::from(engaged)), - ); - } - - pub fn set_server_health(&self, location: &Location, health: ServerHealth) { - self.server_health.set( - two_labels( - ("region", location.region.as_str()), - ("server", location.server.as_str()), - ), - f64::from(health.severity()), - ); - } - - pub fn set_webhook_last_event_age(&self, server: &str, seconds: f64) { - self.webhook_last_event_age_seconds - .set(one_label("server", server), seconds); - } - - pub fn set_ledger(&self, rooms: usize, connections: usize, bytes: u64, evicted_total: u64) { - self.ledger_rooms.set(rooms as f64); - self.ledger_connections.set(connections as f64); - self.ledger_bytes.set(bytes as f64); - self.ledger_evicted_total.set(evicted_total); - } - - pub fn set_rooms_unknown_ratio(&self, ratio: f64) { - self.rooms_unknown_ratio.set(ratio); - } - - pub fn set_oldest_unknown_room_seconds(&self, seconds: f64) { - self.oldest_unknown_room_seconds.set(seconds); - } - - pub fn set_rooms_no_candidates(&self, rooms: usize) { - self.rooms_no_candidates.set(rooms as f64); - } - - pub fn set_wedged_connections(&self, connections: usize) { - self.wedged_connections.set(connections as f64); - } - - pub fn set_coverage_period_seconds(&self, seconds: f64) { - self.coverage_period_seconds.set(seconds); - } - - pub fn set_room_staleness_seconds(&self, seconds: f64) { - self.room_staleness_seconds.set(seconds); - } - - pub fn set_topology_age_seconds(&self, seconds: f64) { - self.topology_age_seconds.set(seconds); - } - - pub fn set_census_age_seconds(&self, seconds: f64) { - self.census_age_seconds.set(seconds); - } - - pub fn set_last_decision_age_seconds(&self, seconds: f64) { - self.last_decision_age_seconds.set(seconds); - } - - pub fn eviction_aborted_count(&self, reason: AbortReason) -> u64 { - self.evictions_aborted - .get(&one_label("reason", reason.label())) - .unwrap_or(0.0) as u64 - } - - pub fn unknown_count(&self, side: SideKind, reason: &str) -> u64 { - self.unknown - .get(&two_labels(("side", side.label()), ("reason", reason))) - .unwrap_or(0.0) as u64 - } - - pub fn gateway_rpc_count(&self, method: &str, outcome: &str) -> u64 { - self.gateway_rpc - .get(&two_labels(("method", method), ("outcome", outcome))) - .unwrap_or(0.0) as u64 - } - - pub fn livekit_call_count(&self, method: &str, outcome: &str) -> u64 { - self.livekit_calls - .get(&two_labels(("method", method), ("outcome", outcome))) - .unwrap_or(0.0) as u64 - } - - pub fn decision_count(&self, action: &DecisionAction) -> u64 { - self.decisions - .get(&one_label("decision", action.label())) - .unwrap_or(0.0) as u64 - } - - pub fn peer_detected_total(&self) -> u64 { - self.peer_detected_total.get() - } - - pub fn tripped_total(&self) -> u64 { - self.tripped_total.get() - } - - pub fn scheduler_starved_total(&self) -> u64 { - self.scheduler_starved_total.get() - } - - pub fn ledger_evicted_total(&self) -> u64 { - self.ledger_evicted_total.get() - } - - pub fn render(&self, out: &mut String) { - self.decisions.render(out, "decisions_total", "counter"); - self.mutations.render(out, "mutations_total", "counter"); - self.evictions_aborted - .render(out, "evictions_aborted_total", "counter"); - self.unknown_media_causes - .render(out, "unknown_media_refusals_total", "counter"); - self.repairs.render(out, "repairs_total", "counter"); - self.confirms.render(out, "confirms_total", "counter"); - self.unknown.render(out, "unknown_total", "counter"); - self.gateway_rpc.render(out, "gateway_rpc_total", "counter"); - self.livekit_calls - .render(out, "livekit_calls_total", "counter"); - - render_counter(out, "ledger_evicted_total", &self.ledger_evicted_total); - render_counter( - out, - "scheduler_starved_total", - &self.scheduler_starved_total, - ); - render_counter(out, "tripped_total", &self.tripped_total); - render_counter(out, "peer_detected_total", &self.peer_detected_total); - - self.mode.render(out, "mode", "gauge"); - self.mode_clamp.render(out, "mode_clamp", "gauge"); - self.server_health.render(out, "server_health", "gauge"); - self.webhook_last_event_age_seconds - .render(out, "webhook_last_event_age_seconds", "gauge"); - - render_gauge(out, "rooms_unknown_ratio", &self.rooms_unknown_ratio); - render_gauge( - out, - "oldest_unknown_room_seconds", - &self.oldest_unknown_room_seconds, - ); - render_gauge(out, "rooms_no_candidates", &self.rooms_no_candidates); - render_gauge(out, "wedged_connections", &self.wedged_connections); - render_gauge(out, "ledger_rooms", &self.ledger_rooms); - render_gauge(out, "ledger_connections", &self.ledger_connections); - render_gauge(out, "ledger_bytes", &self.ledger_bytes); - render_gauge( - out, - "coverage_period_seconds", - &self.coverage_period_seconds, - ); - render_gauge(out, "room_staleness_seconds", &self.room_staleness_seconds); - render_gauge(out, "topology_age_seconds", &self.topology_age_seconds); - render_gauge(out, "census_age_seconds", &self.census_age_seconds); - render_gauge( - out, - "last_decision_age_seconds", - &self.last_decision_age_seconds, - ); - - self.turn_duration.render(out, "turn_duration_ms"); - self.mutation_duration.render(out, "mutation_duration_ms"); - } -} - -fn render_counter(out: &mut String, name: &str, counter: &Counter) { - let _ = writeln!(out, "# TYPE {PREFIX}_{name} counter"); - let _ = writeln!(out, "{PREFIX}_{name} {}", counter.get()); -} - -fn render_gauge(out: &mut String, name: &str, gauge: &Gauge) { - let _ = writeln!(out, "# TYPE {PREFIX}_{name} gauge"); - let _ = writeln!(out, "{PREFIX}_{name} {}", gauge.get()); -} diff --git a/fluxer_recon/src/names.rs b/fluxer_recon/src/names.rs deleted file mode 100644 index 6d9a6c845..000000000 --- a/fluxer_recon/src/names.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::ids::{ChannelId, ConnectionId, GuildId, IdError, RoomKey, UserId}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum NameError { - #[error("room name has {0} underscore-separated parts")] - RoomPartCount(usize), - #[error("room name has an unrecognised shape")] - RoomShape, - #[error("room name has a field that is not a plain decimal u64")] - RoomNumber, - #[error("participant identity has {0} underscore-separated parts")] - IdentityPartCount(usize), - #[error("participant identity has an unrecognised shape")] - IdentityShape, - #[error("participant identity has a field that is not a plain decimal u64")] - IdentityNumber, - #[error("participant identity carries an invalid connection id: {0}")] - IdentityConnection(IdError), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ParticipantIdentity { - pub user_id: UserId, - pub connection_id: ConnectionId, -} - -fn parse_u64_strict(value: &str) -> Option { - if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { - return None; - } - value.parse::().ok() -} - -pub fn format_room_name(room: RoomKey) -> String { - match room { - RoomKey::Guild { - guild_id, - channel_id, - } => format!("guild_{guild_id}_channel_{channel_id}"), - RoomKey::Dm { channel_id } => format!("dm_channel_{channel_id}"), - } -} - -pub fn parse_room_name(name: &str) -> Result { - let parts: Vec<&str> = name.split('_').collect(); - match parts.as_slice() { - ["guild", guild, "channel", channel] => Ok(RoomKey::Guild { - guild_id: GuildId::new(parse_u64_strict(guild).ok_or(NameError::RoomNumber)?), - channel_id: ChannelId::new(parse_u64_strict(channel).ok_or(NameError::RoomNumber)?), - }), - ["dm", "channel", channel] => Ok(RoomKey::Dm { - channel_id: ChannelId::new(parse_u64_strict(channel).ok_or(NameError::RoomNumber)?), - }), - [_, _, _] | [_, _, _, _] => Err(NameError::RoomShape), - other => Err(NameError::RoomPartCount(other.len())), - } -} - -pub fn format_participant_identity(identity: &ParticipantIdentity) -> String { - let ParticipantIdentity { - user_id, - connection_id, - } = identity; - format!("user_{user_id}_{connection_id}") -} - -pub fn parse_participant_identity(identity: &str) -> Result { - let parts: Vec<&str> = identity.split('_').collect(); - match parts.as_slice() { - ["user", user, connection] => Ok(ParticipantIdentity { - user_id: UserId::new(parse_u64_strict(user).ok_or(NameError::IdentityNumber)?), - connection_id: ConnectionId::new(connection).map_err(NameError::IdentityConnection)?, - }), - [_, _, _] => Err(NameError::IdentityShape), - other => Err(NameError::IdentityPartCount(other.len())), - } -} diff --git a/fluxer_recon/src/observe.rs b/fluxer_recon/src/observe.rs deleted file mode 100644 index f88efa805..000000000 --- a/fluxer_recon/src/observe.rs +++ /dev/null @@ -1,380 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::VecDeque; -use std::sync::{Mutex, MutexGuard, PoisonError}; - -use serde::Serialize; - -use crate::decide::{ActionClass, Decision, DecisionSet}; -use crate::evidence::{ConnectionObservation, RoomObservation, SideAuthority}; -use crate::guards::AbortReason; -use crate::health::ReconMode; -use crate::ids::{Millis, RoomKey}; -use crate::ledger::ConnectionLedger; -use crate::metrics::scope_label; - -pub const DEFAULT_JOURNAL_CAPACITY: usize = 512; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Disposition { - Held, - Blocked(AbortReason), - WouldExecute, - Executed, -} - -impl Disposition { - pub const fn label(self) -> &'static str { - match self { - Self::Held => "held", - Self::Blocked(_) => "blocked", - Self::WouldExecute => "would_execute", - Self::Executed => "executed", - } - } - - pub const fn blocked_by(self) -> Option { - match self { - Self::Blocked(reason) => Some(reason), - Self::Held | Self::WouldExecute | Self::Executed => None, - } - } - - pub const fn mutates(self) -> bool { - matches!(self, Self::Executed) - } - - pub const fn is_counterfactual(self) -> bool { - matches!(self, Self::WouldExecute) - } -} - -pub const fn action_class_label(class: ActionClass) -> &'static str { - match class { - ActionClass::None => "none", - ActionClass::Constructive => "constructive", - ActionClass::Destructive => "destructive", - } -} - -pub const fn mode_allows(class: ActionClass, mode: ReconMode) -> bool { - match class { - ActionClass::None => true, - ActionClass::Constructive => mode.allows_constructive(), - ActionClass::Destructive => mode.allows_destructive(), - } -} - -pub fn disposition(decision: &Decision, mode: ReconMode) -> Disposition { - let class = decision.action.class(); - match (class, decision.blocked_by) { - (ActionClass::None, _) => Disposition::Held, - (_, Some(reason)) => Disposition::Blocked(reason), - (class, None) if mode_allows(class, mode) => Disposition::Executed, - (_, None) => Disposition::WouldExecute, - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct EvidenceChain { - pub gateway_presence: &'static str, - pub media_presence: &'static str, - pub gateway_unknown_reason: Option<&'static str>, - pub media_unknown_reason: Option<&'static str>, - pub gateway_authority: Option<&'static str>, - pub media_authority: Option<&'static str>, - pub gateway_siblings: Vec, - pub media_locations: Vec, - pub candidates: Vec, - pub gateway_state_has_connection_id: bool, - pub participant_joined_at_ms: Option, - pub user_has_pending_join: bool, - pub corroborations: u32, - pub divergence_since_ms: Option, - pub fingerprint: Option, - pub action_attempts: u32, - pub repair_attempts: u32, - pub last_repair_verdict: Option<&'static str>, -} - -const fn authority_reason(authority: &SideAuthority) -> Option<&'static str> { - match authority { - SideAuthority::Authoritative => None, - SideAuthority::Unknown(reason) => Some(reason.label()), - } -} - -pub fn evidence_chain( - observation: &RoomObservation, - connection: &ConnectionObservation, - ledger: Option<&ConnectionLedger>, - pending_join_skew_ms: u64, -) -> EvidenceChain { - EvidenceChain { - gateway_presence: connection.presence.gateway.label(), - media_presence: connection.presence.media.label(), - gateway_unknown_reason: connection - .presence - .gateway - .unknown_reason() - .map(|reason| reason.label()), - media_unknown_reason: connection - .presence - .media - .unknown_reason() - .map(|reason| reason.label()), - gateway_authority: authority_reason(&observation.authority.gateway), - media_authority: authority_reason(&observation.authority.media), - gateway_siblings: connection - .gateway_siblings - .iter() - .map(ToString::to_string) - .collect(), - media_locations: connection - .media_locations - .iter() - .map(ToString::to_string) - .collect(), - candidates: observation - .candidates - .locations() - .iter() - .map(ToString::to_string) - .collect(), - gateway_state_has_connection_id: connection.gateway_state_has_connection_id, - participant_joined_at_ms: connection.participant_joined_at.map(Millis::get), - user_has_pending_join: observation.user_has_pending_join( - connection.user_id, - observation.wall_at, - pending_join_skew_ms, - ), - corroborations: ledger.map_or(0, |entry| entry.state.corroborations()), - divergence_since_ms: ledger - .and_then(|entry| entry.state.divergence_since()) - .map(Millis::get), - fingerprint: ledger - .and_then(|entry| entry.state.stored_fingerprint()) - .map(|fingerprint| format!("{:016x}", fingerprint.get())), - action_attempts: ledger.map_or(0, |entry| u32::from(entry.action_attempts)), - repair_attempts: ledger.map_or(0, |entry| u32::from(entry.repair_attempts)), - last_repair_verdict: ledger - .and_then(|entry| entry.last_repair_verdict) - .map(|verdict| verdict.label()), - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct DecisionRecord { - #[serde(skip)] - pub room: RoomKey, - pub turn: u64, - pub at_ms: u64, - pub scope: &'static str, - pub guild_id: Option, - pub channel_id: String, - pub connection_id: String, - pub user_id: String, - pub from: &'static str, - pub to: &'static str, - pub action: &'static str, - pub action_class: &'static str, - pub reason: &'static str, - pub mode: &'static str, - pub disposition: &'static str, - pub blocked_by: Option<&'static str>, - pub would_mutate: bool, - pub evidence: EvidenceChain, -} - -impl DecisionRecord { - pub fn new( - decision: &Decision, - turn: crate::ids::TurnId, - at: Millis, - mode: ReconMode, - evidence: EvidenceChain, - ) -> Self { - let room = decision.connection.room; - let disposition = disposition(decision, mode); - Self { - room, - turn: turn.get(), - at_ms: at.get(), - scope: scope_label(room.scope()), - guild_id: room.guild_id().map(|guild| guild.to_string()), - channel_id: room.channel_id().to_string(), - connection_id: decision.connection.connection.to_string(), - user_id: decision.user_id.to_string(), - from: decision.from, - to: decision.to, - action: decision.action.label(), - action_class: action_class_label(decision.action.class()), - reason: decision.reason.label(), - mode: mode.as_str(), - disposition: disposition.label(), - blocked_by: disposition.blocked_by().map(AbortReason::label), - would_mutate: disposition.is_counterfactual(), - evidence, - } - } - - pub fn is_counterfactual(&self) -> bool { - self.would_mutate - } - - pub fn matches_room(&self, room: RoomKey) -> bool { - self.room == room - } - - pub fn to_json(&self) -> String { - serde_json::to_string(self) - .unwrap_or_else(|error| format!("{{\"serialisation_error\":\"{error}\"}}")) - } -} - -pub fn log_decision(record: &DecisionRecord) { - if record.would_mutate { - tracing::warn!( - room_scope = record.scope, - guild_id = record.guild_id.as_deref(), - channel_id = record.channel_id, - connection_id = record.connection_id, - user_id = record.user_id, - action = record.action, - reason = record.reason, - mode = record.mode, - turn = record.turn, - decision = record.to_json(), - "recon would have mutated" - ); - return; - } - - tracing::info!( - room_scope = record.scope, - guild_id = record.guild_id.as_deref(), - channel_id = record.channel_id, - connection_id = record.connection_id, - user_id = record.user_id, - action = record.action, - reason = record.reason, - mode = record.mode, - disposition = record.disposition, - turn = record.turn, - decision = record.to_json(), - "recon decision" - ); -} - -#[derive(Debug)] -pub struct DecisionJournal { - capacity: usize, - entries: Mutex>, -} - -impl Default for DecisionJournal { - fn default() -> Self { - Self::new(DEFAULT_JOURNAL_CAPACITY) - } -} - -impl DecisionJournal { - pub fn new(capacity: usize) -> Self { - Self { - capacity: capacity.max(1), - entries: Mutex::new(VecDeque::new()), - } - } - - fn lock(&self) -> MutexGuard<'_, VecDeque> { - self.entries.lock().unwrap_or_else(PoisonError::into_inner) - } - - pub fn record(&self, record: DecisionRecord) { - let mut entries = self.lock(); - if entries.len() >= self.capacity { - entries.pop_front(); - } - entries.push_back(record); - } - - pub fn len(&self) -> usize { - self.lock().len() - } - - pub fn is_empty(&self) -> bool { - self.lock().is_empty() - } - - pub fn capacity(&self) -> usize { - self.capacity - } - - pub fn last_at_ms(&self) -> Option { - self.lock().back().map(|record| record.at_ms) - } - - pub fn recent(&self, limit: usize) -> Vec { - let entries = self.lock(); - entries.iter().rev().take(limit).cloned().collect() - } - - pub fn for_room(&self, room: RoomKey, limit: usize) -> Vec { - let entries = self.lock(); - entries - .iter() - .rev() - .filter(|record| record.matches_room(room)) - .take(limit) - .cloned() - .collect() - } - - pub fn counterfactual_count(&self) -> usize { - self.lock() - .iter() - .filter(|record| record.is_counterfactual()) - .count() - } -} - -pub struct ObservedSet<'a> { - pub set: &'a DecisionSet, - pub observation: &'a RoomObservation, - pub mode: ReconMode, - pub pending_join_skew_ms: u64, -} - -pub fn journal_decisions( - journal: &DecisionJournal, - observed: &ObservedSet<'_>, - ledger_of: impl Fn(&Decision) -> Option, -) -> Vec { - let mut recorded = Vec::with_capacity(observed.set.decisions.len()); - for decision in &observed.set.decisions { - let entry = ledger_of(decision); - let chain = match observed - .observation - .observation_for(&decision.connection.connection) - { - Some(connection) => evidence_chain( - observed.observation, - connection, - entry.as_ref(), - observed.pending_join_skew_ms, - ), - None => EvidenceChain::default(), - }; - let record = DecisionRecord::new( - decision, - observed.set.turn, - observed.set.at, - observed.mode, - chain, - ); - log_decision(&record); - journal.record(record.clone()); - recorded.push(record); - } - recorded -} diff --git a/fluxer_recon/src/runtime.rs b/fluxer_recon/src/runtime.rs deleted file mode 100644 index 557b5edce..000000000 --- a/fluxer_recon/src/runtime.rs +++ /dev/null @@ -1,705 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use fluxer_svc::config::ServiceConfig; - -use crate::budget::{BreakerEvent, BreakerLimits, Governor, GovernorLimits}; -use crate::census::{CensusCache, CensusLimits}; -use crate::config::ReconConfig; -use crate::decide::DecideParams; -use crate::discovery::{DirectoryLimits, PassCounters, RoomDirectory, ServerCliffWatch}; -use crate::health::{ModeClamp, PauseBackoff, ReconMode, ServerHealthMap, WarmupGate}; -use crate::ids::{Millis, RoomKey, TurnId}; -use crate::ledger::{ConnectionState, Ledger, LedgerLimits, MAP_ENTRY_OVERHEAD_BYTES}; -use crate::metrics::ReconMetrics; -use crate::observe::DecisionJournal; -use crate::schedule::{ - CoverageInputs, Scheduler, SchedulerConfig, derived_coverage_period_ms, max_auditable_rooms, -}; -use crate::suspicion::SuspicionLane; -use crate::topology::{TopologyCache, TopologyLimits}; -use crate::turn::TurnSequencer; - -pub use crate::singleton::{ - InstanceAnnounce, InstanceIdentity, InstanceProbe, PEER_CLAMP_RELEASE_AFTER_MS, PeerRecord, - PeerVerdict, SINGLETON_PROBE_DEADLINE_MS, SINGLETON_PROBE_INTERVAL_MS, SingletonLayer, - SingletonResponse, peer_verdict, run_singleton, should_answer_probe, -}; - -pub const SERVICE_NAME: &str = "recon"; -pub const RETIRED_GRACE_MS: u64 = 60_000; - -pub const RUNTIME_TASKS: [&str; 6] = [ - "http", - "engine", - "census", - "discovery", - "control", - "singleton", -]; -pub const WEBHOOK_TASK: &str = "webhook"; -pub const OPTIONAL_TASKS: [&str; 1] = [WEBHOOK_TASK]; -pub const HOUSEKEEP_INTERVAL_MS: u64 = 1_000; - -pub fn runtime_tasks(config: &ReconConfig) -> Vec<&'static str> { - let mut names = RUNTIME_TASKS.to_vec(); - if config.webhook_enabled { - names.push(WEBHOOK_TASK); - } - names -} - -pub fn monotonic_now() -> Millis { - Millis::new(fluxer_svc::metrics::now_ms().max(0) as u64) -} - -pub fn wall_now_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |elapsed| { - elapsed.as_millis().min(u128::from(u64::MAX)) as u64 - }) -} - -#[derive(Debug)] -pub struct Readiness { - flag: Arc, - has_been_ready: AtomicBool, - draining: AtomicBool, -} - -impl Default for Readiness { - fn default() -> Self { - Self::new() - } -} - -impl Readiness { - pub fn new() -> Self { - Self { - flag: Arc::new(AtomicBool::new(false)), - has_been_ready: AtomicBool::new(false), - draining: AtomicBool::new(false), - } - } - - pub fn flag(&self) -> Arc { - Arc::clone(&self.flag) - } - - pub fn mark_ready(&self) -> bool { - if self.draining.load(Ordering::SeqCst) { - return false; - } - self.flag.store(true, Ordering::SeqCst); - !self.has_been_ready.swap(true, Ordering::SeqCst) - } - - pub fn is_ready(&self) -> bool { - self.flag.load(Ordering::SeqCst) - } - - pub fn has_been_ready(&self) -> bool { - self.has_been_ready.load(Ordering::SeqCst) - } - - pub fn begin_shutdown(&self) { - self.draining.store(true, Ordering::SeqCst); - self.flag.store(false, Ordering::SeqCst); - } - - pub fn is_draining(&self) -> bool { - self.draining.load(Ordering::SeqCst) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ModeTransition { - pub configured: ReconMode, - pub previous_override: Option, - pub requested: Option, - pub effective_before: ReconMode, - pub effective_after: ReconMode, -} - -impl ModeTransition { - pub fn changed(&self) -> bool { - self.effective_before != self.effective_after - } - - pub fn escalated(&self) -> bool { - self.effective_after > self.effective_before - } -} - -pub fn governor_limits(config: &ReconConfig) -> GovernorLimits { - GovernorLimits { - gateway_read_rps: config.gateway_read_rps, - gateway_mutate_rpm: config.gateway_mutate_rpm, - livekit_read_rps: config.livekit_read_rps, - livekit_read_rps_per_server: config.livekit_read_rps_per_server, - livekit_mutate_rpm: config.livekit_mutate_rpm, - max_inflight_room_turns: config.max_inflight_room_turns, - max_inflight_gateway: config.max_inflight_gateway, - max_inflight_livekit: config.max_inflight_livekit, - max_inflight_per_server: config.max_inflight_per_server, - max_mutations_per_room_per_min: config.max_mutations_per_room_per_min, - max_mutations_per_guild_per_min: config.max_mutations_per_guild_per_min, - max_mutations_per_min: config.max_mutations_per_min, - max_action_attempts: config.max_action_attempts, - preflight_max_age_ms: config.preflight_max_age_ms, - breaker: BreakerLimits { - max_divergent_fraction: config.max_divergent_fraction, - auto_reset_ms: config.breaker_auto_reset_ms, - max_auto_resets: config.max_breaker_auto_resets, - }, - } -} - -pub fn ledger_limits(config: &ReconConfig) -> LedgerLimits { - LedgerLimits { - max_tracked_rooms: config.max_tracked_rooms, - max_tracked_connections: config.max_tracked_connections, - max_connections_per_room: config.max_connections_per_room, - memory_budget_bytes: config.memory_budget_bytes, - retired_grace_ms: RETIRED_GRACE_MS, - } -} - -#[derive(Debug)] -pub struct RuntimeState { - pub governor: Governor, - pub ledger: Ledger, - pub server_health: ServerHealthMap, - pub turn: TurnId, - pub sequencer: TurnSequencer, - pub scheduler: Scheduler, - pub directory: RoomDirectory, - pub census: CensusCache, - pub topology: TopologyCache, - pub suspicion: SuspicionLane, - pub cliffs: ServerCliffWatch, - pub counters: PassCounters, - pub pass_started_at: Millis, - pub pass_pending: BTreeSet, - pub topology_loaded_at: Option, - pub topology_servers: usize, - pub census_at: Option, - pub coverage_period_ms: u64, - pub peer: Option, - pub unknown_since: BTreeMap, - pub last_housekeep_at: Option, - pub topology_refresh_requested: bool, - pub mutations_pause: PauseBackoff, -} - -impl RuntimeState { - pub fn new(config: &ReconConfig, now: Millis) -> Self { - let warmup = WarmupGate::new(now, config.warmup_seconds.saturating_mul(1_000)); - let mut governor = Governor::new(config.mode, governor_limits(config), warmup, now); - let clamps = governor.clamps_mut(); - clamps.set(ModeClamp::Warmup, true); - clamps.set(ModeClamp::ColdStart, true); - clamps.set(ModeClamp::TopologyStale, true); - Self { - governor, - ledger: Ledger::new(ledger_limits(config), DecideParams::from_config(config)), - server_health: ServerHealthMap::new(), - turn: TurnId::FIRST, - sequencer: TurnSequencer::new(), - scheduler: Scheduler::new(SchedulerConfig::from_config(config)), - directory: RoomDirectory::new(DirectoryLimits::from_config(config)), - census: CensusCache::new(CensusLimits::from_config(config)), - topology: TopologyCache::new(TopologyLimits::from_config(config)), - suspicion: SuspicionLane::from_config(config), - cliffs: ServerCliffWatch::from_config(config), - counters: PassCounters::new(), - pass_started_at: now, - pass_pending: BTreeSet::new(), - topology_loaded_at: None, - topology_servers: 0, - census_at: None, - coverage_period_ms: config.coverage_target_ms, - peer: None, - unknown_since: BTreeMap::new(), - last_housekeep_at: None, - topology_refresh_requested: false, - mutations_pause: PauseBackoff::new(), - } - } - - pub fn forget_room(&mut self, room: &RoomKey) { - self.directory.forget(room); - self.scheduler.forget(room); - self.sequencer.forget(room); - self.suspicion.forget(room); - self.pass_pending.remove(room); - self.unknown_since.remove(room); - self.governor.forget_room(room); - } - - pub fn tracked_bytes(&self) -> u64 { - let room_key_entry = (size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; - self.ledger - .tracked_bytes() - .saturating_add(self.directory.tracked_bytes()) - .saturating_add(self.scheduler.tracked_bytes()) - .saturating_add(self.sequencer.tracked_bytes()) - .saturating_add(self.suspicion.tracked_bytes()) - .saturating_add(self.governor.mutations().tracked_bytes()) - .saturating_add(self.census.tracked_bytes()) - .saturating_add((self.pass_pending.len() as u64).saturating_mul(room_key_entry)) - .saturating_add( - (self.unknown_since.len() as u64) - .saturating_mul(room_key_entry.saturating_add(size_of::() as u64)), - ) - } - - pub const fn take_topology_refresh_request(&mut self) -> bool { - let requested = self.topology_refresh_requested; - self.topology_refresh_requested = false; - requested - } - - pub fn next_turn(&mut self) -> TurnId { - self.turn = self.turn.next(); - self.turn - } - - pub fn topology_age_ms(&self, now: Millis) -> Option { - self.topology_loaded_at - .map(|loaded| now.saturating_since(loaded)) - } - - pub fn census_age_ms(&self, now: Millis) -> Option { - self.census_at.map(|at| now.saturating_since(at)) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TickInputs { - pub now: Millis, - pub nats_connected: bool, - pub last_decision_at: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct HousekeepReport { - pub effective_mode: ReconMode, - pub breaker: BreakerEvent, - pub divergent_fraction: f64, - pub pruned_connections: usize, - pub dropped_rooms: usize, - pub evicted_rooms: usize, - pub unknown_rooms: usize, - pub wedged_connections: usize, -} - -pub const ROOM_RETENTION_MS: u64 = 600_000; - -fn last_seen_by_any_source(provenance: &crate::discovery::RoomProvenance) -> Option { - provenance - .sources() - .into_iter() - .filter_map(|source| provenance.last_seen_by(source)) - .max() -} - -fn prune_room_set(state: &mut RuntimeState, now: Millis) { - let ledger = &state.ledger; - let scheduler = &mut state.scheduler; - let sequencer = &mut state.sequencer; - let suspicion = &mut state.suspicion; - let pending = &mut state.pass_pending; - - let mut forgotten: Vec = Vec::new(); - state.directory.retain(|room, provenance| { - if ledger.room(room).is_some() { - return true; - } - let cold = last_seen_by_any_source(provenance) - .is_none_or(|seen| now.saturating_since(seen) > ROOM_RETENTION_MS); - if cold { - forgotten.push(*room); - } - !cold - }); - - for room in &forgotten { - scheduler.forget(room); - sequencer.forget(room); - suspicion.forget(room); - pending.remove(room); - } - - pending.retain(|room| scheduler.is_tracked(room)); - - for room in &forgotten { - state.governor.forget_room(room); - state.unknown_since.remove(room); - } -} - -pub const MAX_EVICTIONS_PER_HOUSEKEEP: usize = 256; - -fn evict_to_budget(state: &mut RuntimeState) -> usize { - let limits = state.ledger.limits(); - let mut evicted = 0; - - while evicted < MAX_EVICTIONS_PER_HOUSEKEEP - && (state.tracked_bytes() > limits.memory_budget_bytes - || state.ledger.tracked_rooms() > limits.max_tracked_rooms - || state.ledger.tracked_connections() > limits.max_tracked_connections) - { - let Some(victim) = state.ledger.coldest_room() else { - break; - }; - if !state.ledger.evict_room(&victim) { - break; - } - state.forget_room(&victim); - evicted += 1; - } - - evicted -} - -fn room_is_unknown(room: &crate::ledger::RoomLedger, config: &ReconConfig, now: Millis) -> bool { - room.no_candidates() - || room.partially_unreadable() - || room.gateway_cliff().is_held(now, config.room_cliff_hold_ms) - || room.media_cliff().is_held(now, config.server_cliff_hold_ms) -} - -pub fn housekeep( - state: &mut RuntimeState, - config: &ReconConfig, - metrics: &ReconMetrics, - inputs: TickInputs, -) -> HousekeepReport { - let now = inputs.now; - - state.governor.clamps_mut().release_expired(now); - state.mutations_pause.decay(now); - - let warmup_pending = !state.governor.warmup().elapsed_complete(now); - let coverage_pending = !state.governor.warmup().coverage_pass_complete(); - let topology_stale = state - .topology_age_ms(now) - .is_none_or(|age| age > config.topology_max_age_ms); - let peer_present = state - .peer - .as_ref() - .is_some_and(|peer| now.saturating_since(peer.last_seen_at) < PEER_CLAMP_RELEASE_AFTER_MS); - - { - let clamps = state.governor.clamps_mut(); - clamps.set(ModeClamp::Warmup, warmup_pending); - clamps.set(ModeClamp::ColdStart, coverage_pending); - clamps.set(ModeClamp::NatsReconnect, !inputs.nats_connected); - clamps.set(ModeClamp::TopologyStale, topology_stale); - clamps.set(ModeClamp::SingletonConflict, peer_present); - } - - let divergent_fraction = state.ledger.divergent_fraction(); - let breaker = state.governor.observe_divergence(divergent_fraction, now); - if matches!(breaker, BreakerEvent::Tripped) { - metrics.record_breaker_trip(); - state.ledger.reset_all_corroborations(); - } - - let pruned_connections = state.ledger.prune_retired(now); - let dropped_rooms = state.ledger.drop_empty_rooms(); - state.governor.prune_mutations(now); - let evicted_rooms = evict_to_budget(state); - - state.suspicion.expire(now); - state.scheduler.expire_hot(now); - prune_room_set(state, now); - - let live_servers = state.topology.lens(now).live_locations(); - state.governor.retain_servers(&live_servers); - - let mut no_candidate_rooms = 0; - let mut wedged_connections = 0; - let mut oldest_staleness_ms = 0; - let mut unknown_keys: Vec = Vec::new(); - - for (key, room) in state.ledger.rooms() { - oldest_staleness_ms = oldest_staleness_ms.max(now.saturating_since(room.last_seen())); - if room.no_candidates() { - no_candidate_rooms += 1; - } - if room_is_unknown(room, config, now) { - unknown_keys.push(*key); - } - wedged_connections += room - .connections() - .iter() - .filter(|entry| matches!(entry.state, ConnectionState::Wedged { .. })) - .count(); - } - - let unknown_rooms = unknown_keys.len(); - state - .unknown_since - .retain(|key, _| unknown_keys.binary_search(key).is_ok()); - for key in unknown_keys { - state.unknown_since.entry(key).or_insert(now); - } - - let oldest_unknown_ms = state - .unknown_since - .values() - .map(|since| now.saturating_since(*since)) - .max() - .unwrap_or(0); - - let tracked_rooms = state.ledger.tracked_rooms(); - let effective_mode = state.governor.effective_mode(); - - metrics.set_mode(effective_mode); - for clamp in ModeClamp::ALL { - metrics.set_mode_clamp(clamp, state.governor.clamps().is_engaged(clamp)); - } - metrics.set_ledger( - tracked_rooms, - state.ledger.tracked_connections(), - state.tracked_bytes(), - state - .ledger - .evicted_rooms_total() - .saturating_add(state.ledger.evicted_connections_total()), - ); - metrics.set_wedged_connections(wedged_connections); - metrics.set_rooms_no_candidates(no_candidate_rooms); - metrics.set_rooms_unknown_ratio(if tracked_rooms == 0 { - 0.0 - } else { - unknown_rooms as f64 / tracked_rooms as f64 - }); - metrics.set_oldest_unknown_room_seconds(oldest_unknown_ms as f64 / 1_000.0); - metrics.set_room_staleness_seconds(oldest_staleness_ms as f64 / 1_000.0); - metrics.set_coverage_period_seconds(state.coverage_period_ms as f64 / 1_000.0); - metrics.set_topology_age_seconds( - state - .topology_age_ms(now) - .map_or(f64::INFINITY, |age| age as f64 / 1_000.0), - ); - metrics.set_census_age_seconds( - state - .census_age_ms(now) - .map_or(f64::INFINITY, |age| age as f64 / 1_000.0), - ); - metrics.set_last_decision_age_seconds(inputs.last_decision_at.map_or(f64::INFINITY, |at| { - now.saturating_since(at) as f64 / 1_000.0 - })); - for (location, tracker) in state.server_health.tracked() { - metrics.set_server_health(location, tracker.state()); - } - - state.last_housekeep_at = Some(now); - - HousekeepReport { - effective_mode, - breaker, - divergent_fraction, - pruned_connections, - dropped_rooms, - evicted_rooms, - unknown_rooms, - wedged_connections, - } -} - -pub struct SharedInner { - config: ReconConfig, - instance: InstanceIdentity, - metrics: Arc, - journal: DecisionJournal, - readiness: Readiness, - boot_at: Millis, - state: Mutex, -} - -#[derive(Clone)] -pub struct Shared(Arc); - -impl Shared { - pub fn new(config: ReconConfig, instance: InstanceIdentity, now: Millis) -> Self { - let state = RuntimeState::new(&config, now); - Self(Arc::new(SharedInner { - config, - instance, - metrics: Arc::new(ReconMetrics::new()), - journal: DecisionJournal::default(), - readiness: Readiness::new(), - boot_at: now, - state: Mutex::new(state), - })) - } - - pub fn config(&self) -> &ReconConfig { - &self.0.config - } - - pub fn instance(&self) -> &InstanceIdentity { - &self.0.instance - } - - pub fn metrics(&self) -> &Arc { - &self.0.metrics - } - - pub fn journal(&self) -> &DecisionJournal { - &self.0.journal - } - - pub fn readiness(&self) -> &Readiness { - &self.0.readiness - } - - pub fn boot_at(&self) -> Millis { - self.0.boot_at - } - - fn lock(&self) -> MutexGuard<'_, RuntimeState> { - self.0.state.lock().unwrap_or_else(PoisonError::into_inner) - } - - pub fn with_state(&self, action: impl FnOnce(&RuntimeState) -> R) -> R { - action(&self.lock()) - } - - pub fn with_state_mut(&self, action: impl FnOnce(&mut RuntimeState) -> R) -> R { - action(&mut self.lock()) - } - - pub fn effective_mode(&self) -> ReconMode { - self.with_state(|state| state.governor.effective_mode()) - } - - pub fn tick(&self, inputs: TickInputs) -> HousekeepReport { - let config = self.0.config.clone(); - let metrics = Arc::clone(&self.0.metrics); - self.with_state_mut(|state| housekeep(state, &config, &metrics, inputs)) - } - - pub fn tick_if_due(&self, inputs: TickInputs, interval_ms: u64) -> Option { - let due = self.with_state(|state| { - state - .last_housekeep_at - .is_none_or(|last| inputs.now.saturating_since(last) >= interval_ms) - }); - due.then(|| self.tick(inputs)) - } - - pub fn set_mode_override(&self, requested: Option) -> ModeTransition { - self.with_state_mut(|state| { - let configured = state.governor.configured_mode(); - let previous_override = state.governor.runtime_override(); - let effective_before = state.governor.effective_mode(); - state.governor.set_runtime_override(requested); - let effective_after = state.governor.effective_mode(); - ModeTransition { - configured, - previous_override, - requested, - effective_before, - effective_after, - } - }) - } - - pub fn note_peer(&self, announce: &InstanceAnnounce, now: Millis) { - self.0.metrics.record_peer_detected(); - self.with_state_mut(|state| { - let detections = state - .peer - .as_ref() - .filter(|peer| peer.instance_id == announce.instance_id) - .map_or(0, |peer| peer.detections); - let first_seen_at = state - .peer - .as_ref() - .filter(|peer| peer.instance_id == announce.instance_id) - .map_or(now, |peer| peer.first_seen_at); - state.peer = Some(PeerRecord { - instance_id: announce.instance_id.clone(), - started_at: announce.started_at, - first_seen_at, - last_seen_at: now, - detections: detections.saturating_add(1), - }); - state - .governor - .clamps_mut() - .set(ModeClamp::SingletonConflict, true); - }); - } -} - -pub fn assert_single_replica( - service_name: &str, - shard_count: u32, - shard_id: u32, -) -> anyhow::Result<()> { - if service_name != SERVICE_NAME { - anyhow::bail!( - "FLUXER_SVC_NAME must be {SERVICE_NAME} so the control subject and metric namespace agree, got {service_name}" - ); - } - if shard_count != 1 { - anyhow::bail!( - "FLUXER_SVC_SHARD_COUNT must be 1 for a single-replica service, got {shard_count}" - ); - } - if shard_id != 0 { - anyhow::bail!("FLUXER_SVC_SHARD_ID must be 0 for a single-replica service, got {shard_id}"); - } - Ok(()) -} - -pub fn assert_coverage_budget(config: &ReconConfig) -> anyhow::Result<()> { - if config.coverage_target_ms > config.coverage_period_hard_cap_ms { - anyhow::bail!( - "FLUXER_RECON_COVERAGE_TARGET_MS ({}) exceeds FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({})", - config.coverage_target_ms, - config.coverage_period_hard_cap_ms - ); - } - if config.tick_ms == 0 { - anyhow::bail!("FLUXER_RECON_TICK_MS must be greater than zero"); - } - if config.worker_threads == 0 { - anyhow::bail!("FLUXER_RECON_WORKER_THREADS must be greater than zero"); - } - - let auditable = max_auditable_rooms(config); - let derived = - derived_coverage_period_ms(&CoverageInputs::for_rooms(config.expected_rooms, config)); - if derived > config.coverage_period_hard_cap_ms { - anyhow::bail!( - "FLUXER_RECON_EXPECTED_ROOMS ({}) needs a coverage period of {derived} ms at the configured read rates, above FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({}); this process can audit {auditable} rooms within that cap", - config.expected_rooms, - config.coverage_period_hard_cap_ms - ); - } - - tracing::info!( - expected_rooms = config.expected_rooms, - derived_coverage_period_ms = derived, - max_auditable_rooms = auditable, - "the configured read rates admit this many rooms inside the coverage hard cap" - ); - Ok(()) -} - -pub fn boot_assertions(service: &ServiceConfig, config: &ReconConfig) -> anyhow::Result<()> { - assert_single_replica(&service.service_name, service.shard_count, service.shard_id)?; - assert_coverage_budget(config) -} diff --git a/fluxer_recon/src/schedule.rs b/fluxer_recon/src/schedule.rs deleted file mode 100644 index e8e44c6fd..000000000 --- a/fluxer_recon/src/schedule.rs +++ /dev/null @@ -1,562 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::cmp::Ordering; -use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; - -use crate::config::ReconConfig; -use crate::ids::{Millis, RoomKey}; -use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; -use crate::turn::Digest; - -pub const SCHEDULE_ENTRY_BYTES: u64 = (size_of::() - + size_of::() - + size_of::() - + (2 * MAP_ENTRY_OVERHEAD_BYTES)) as u64; - -pub const GATEWAY_READS_PER_ROOM_TURN: u32 = 2; -pub const LIVEKIT_READS_PER_ROOM_TURN: u32 = 1; - -const fn room_hash(room: RoomKey) -> u64 { - match room { - RoomKey::Guild { - guild_id, - channel_id, - } => Digest::new() - .text("guild") - .number(guild_id.get()) - .number(channel_id.get()) - .finish(), - RoomKey::Dm { channel_id } => Digest::new().text("dm").number(channel_id.get()).finish(), - } -} - -const fn tiebreak(room: RoomKey, stamp: u64) -> u64 { - Digest::new().number(room_hash(room)).number(stamp).finish() -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum Tier { - Hot, - Background, -} - -impl Tier { - pub const fn label(self) -> &'static str { - match self { - Self::Hot => "hot", - Self::Background => "background", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Promotion { - Promoted, - Extended, - Refused, -} - -impl Promotion { - pub const fn label(self) -> &'static str { - match self { - Self::Promoted => "promoted", - Self::Extended => "extended", - Self::Refused => "refused", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DeferReason { - Budget, - Unreadable, -} - -impl DeferReason { - pub const fn label(self) -> &'static str { - match self { - Self::Budget => "budget", - Self::Unreadable => "unreadable", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct SchedulerConfig { - pub hot_period_ms: u64, - pub background_period_ms: u64, - pub max_hot_rooms: usize, - pub suspicion_hold_ms: u64, - pub turns_per_tick: usize, -} - -impl SchedulerConfig { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - hot_period_ms: config.hot_period_ms, - background_period_ms: config.coverage_target_ms, - max_hot_rooms: config.max_hot_rooms, - suspicion_hold_ms: config.suspicion_hold_ms, - turns_per_tick: config.turns_per_tick, - } - } - - pub const fn period_ms(&self, tier: Tier) -> u64 { - match tier { - Tier::Hot => self.hot_period_ms, - Tier::Background => self.background_period_ms, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Due { - at: Millis, - tiebreak: u64, - stamp: u64, - room: RoomKey, -} - -impl Ord for Due { - fn cmp(&self, other: &Self) -> Ordering { - other - .at - .cmp(&self.at) - .then_with(|| other.tiebreak.cmp(&self.tiebreak)) - .then_with(|| other.room.cmp(&self.room)) - .then_with(|| other.stamp.cmp(&self.stamp)) - } -} - -impl PartialOrd for Due { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct RoomSchedule { - stamp: u64, - tier: Tier, - hot_until: Option, - due_at: Millis, - in_flight: bool, - served: u64, -} - -#[derive(Clone, Debug)] -pub struct Scheduler { - config: SchedulerConfig, - heap: BinaryHeap, - rooms: BTreeMap, - hot: BTreeSet, - stamps: u64, - in_flight: usize, - served_total: u64, - starved_total: u64, - deferred_total: u64, - hot_refused_total: u64, - demoted_total: u64, -} - -impl Scheduler { - pub fn tracked_bytes(&self) -> u64 { - (self.rooms.len() as u64) - .saturating_mul(SCHEDULE_ENTRY_BYTES) - .saturating_add( - (self.hot.len() as u64) - .saturating_mul((size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64), - ) - } - - pub fn new(config: SchedulerConfig) -> Self { - Self { - config, - heap: BinaryHeap::new(), - rooms: BTreeMap::new(), - hot: BTreeSet::new(), - stamps: 0, - in_flight: 0, - served_total: 0, - starved_total: 0, - deferred_total: 0, - hot_refused_total: 0, - demoted_total: 0, - } - } - - pub const fn config(&self) -> SchedulerConfig { - self.config - } - - pub const fn turns_per_tick(&self) -> usize { - self.config.turns_per_tick - } - - pub fn tracked(&self) -> usize { - self.rooms.len() - } - - pub fn is_empty(&self) -> bool { - self.rooms.is_empty() - } - - pub fn hot_rooms(&self) -> usize { - self.hot.len() - } - - pub const fn in_flight(&self) -> usize { - self.in_flight - } - - pub const fn served_total(&self) -> u64 { - self.served_total - } - - pub const fn starved_total(&self) -> u64 { - self.starved_total - } - - pub const fn deferred_total(&self) -> u64 { - self.deferred_total - } - - pub const fn hot_refused_total(&self) -> u64 { - self.hot_refused_total - } - - pub const fn demoted_total(&self) -> u64 { - self.demoted_total - } - - pub fn tier(&self, room: &RoomKey) -> Option { - self.rooms.get(room).map(|entry| entry.tier) - } - - pub fn due_at(&self, room: &RoomKey) -> Option { - self.rooms.get(room).map(|entry| entry.due_at) - } - - pub fn served(&self, room: &RoomKey) -> Option { - self.rooms.get(room).map(|entry| entry.served) - } - - pub fn is_tracked(&self, room: &RoomKey) -> bool { - self.rooms.contains_key(room) - } - - pub fn track(&mut self, room: RoomKey, now: Millis) -> bool { - if self.rooms.contains_key(&room) { - return false; - } - let entry = RoomSchedule { - stamp: self.next_stamp(), - tier: Tier::Background, - hot_until: None, - due_at: now, - in_flight: false, - served: 0, - }; - self.rooms.insert(room, entry); - self.push(room, entry); - true - } - - pub fn forget(&mut self, room: &RoomKey) -> bool { - self.hot.remove(room); - match self.rooms.remove(room) { - None => false, - Some(entry) => { - if entry.in_flight { - self.in_flight = self.in_flight.saturating_sub(1); - } - true - } - } - } - - pub fn retain(&mut self, mut keep: F) - where - F: FnMut(&RoomKey) -> bool, - { - let dropped: Vec = self - .rooms - .keys() - .copied() - .filter(|room| !keep(room)) - .collect(); - for room in dropped { - self.forget(&room); - } - } - - pub fn take_due(&mut self, now: Millis) -> Vec { - self.take_due_up_to(now, self.config.turns_per_tick) - } - - pub fn take_due_up_to(&mut self, now: Millis, limit: usize) -> Vec { - let mut taken: Vec = Vec::new(); - while taken.len() < limit { - let Some(candidate) = self.heap.peek().copied() else { - break; - }; - if candidate.at > now { - break; - } - self.heap.pop(); - let Some(entry) = self.rooms.get_mut(&candidate.room) else { - continue; - }; - if entry.stamp != candidate.stamp || entry.in_flight { - continue; - } - entry.in_flight = true; - self.in_flight = self.in_flight.saturating_add(1); - taken.push(candidate.room); - } - taken - } - - pub fn completed(&mut self, room: RoomKey, now: Millis) -> bool { - let Some(mut entry) = self.rooms.get(&room).copied() else { - return false; - }; - self.release(&mut entry); - entry.served = entry.served.saturating_add(1); - self.served_total = self.served_total.saturating_add(1); - entry.stamp = self.next_stamp(); - entry.due_at = now.saturating_add_millis(self.config.period_ms(entry.tier)); - self.rooms.insert(room, entry); - self.push(room, entry); - true - } - - pub fn defer(&mut self, room: RoomKey, now: Millis, reason: DeferReason) -> bool { - let Some(mut entry) = self.rooms.get(&room).copied() else { - return false; - }; - self.release(&mut entry); - self.deferred_total = self.deferred_total.saturating_add(1); - if matches!(reason, DeferReason::Budget) { - self.starved_total = self.starved_total.saturating_add(1); - } - entry.stamp = self.next_stamp(); - entry.due_at = now; - self.rooms.insert(room, entry); - self.push(room, entry); - true - } - - pub fn promote(&mut self, room: RoomKey, now: Millis) -> Promotion { - self.track(room, now); - let Some(mut entry) = self.rooms.get(&room).copied() else { - return Promotion::Refused; - }; - let hot_until = now.saturating_add_millis(self.config.suspicion_hold_ms); - - if matches!(entry.tier, Tier::Hot) { - entry.hot_until = Some(hot_until); - self.rooms.insert(room, entry); - self.hot.insert(room); - return Promotion::Extended; - } - if self.hot.len() >= self.config.max_hot_rooms { - self.hot_refused_total = self.hot_refused_total.saturating_add(1); - return Promotion::Refused; - } - - entry.tier = Tier::Hot; - entry.hot_until = Some(hot_until); - entry.stamp = self.next_stamp(); - entry.due_at = now; - self.rooms.insert(room, entry); - self.hot.insert(room); - if !entry.in_flight { - self.push(room, entry); - } - Promotion::Promoted - } - - pub fn expire_hot(&mut self, now: Millis) -> usize { - let expired: Vec = self - .hot - .iter() - .copied() - .filter(|room| { - self.rooms - .get(room) - .is_none_or(|entry| entry.hot_until.is_none_or(|until| until <= now)) - }) - .collect(); - - for room in &expired { - self.hot.remove(room); - let Some(mut entry) = self.rooms.get(room).copied() else { - continue; - }; - entry.tier = Tier::Background; - entry.hot_until = None; - entry.stamp = self.next_stamp(); - entry.due_at = now.saturating_add_millis(self.config.background_period_ms); - self.rooms.insert(*room, entry); - if !entry.in_flight { - self.push(*room, entry); - } - self.demoted_total = self.demoted_total.saturating_add(1); - } - expired.len() - } - - fn release(&mut self, entry: &mut RoomSchedule) { - if entry.in_flight { - self.in_flight = self.in_flight.saturating_sub(1); - } - entry.in_flight = false; - } - - fn next_stamp(&mut self) -> u64 { - self.stamps = self.stamps.saturating_add(1); - self.stamps - } - - fn push(&mut self, room: RoomKey, entry: RoomSchedule) { - self.heap.push(Due { - at: entry.due_at, - tiebreak: tiebreak(room, entry.stamp), - stamp: entry.stamp, - room, - }); - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Cadence { - interval_ms: u64, - next_at: Option, - fired_total: u64, - forced: bool, -} - -impl Cadence { - pub const fn new(interval_ms: u64) -> Self { - Self { - interval_ms: if interval_ms == 0 { 1 } else { interval_ms }, - next_at: None, - fired_total: 0, - forced: false, - } - } - - pub const fn interval_ms(self) -> u64 { - self.interval_ms - } - - pub const fn next_due_at(self) -> Option { - self.next_at - } - - pub const fn fired_total(self) -> u64 { - self.fired_total - } - - pub const fn force(&mut self) { - self.forced = true; - } - - pub fn due(&mut self, now: Millis) -> bool { - let ready = self.forced || self.next_at.is_none_or(|at| now >= at); - if !ready { - return false; - } - self.forced = false; - self.next_at = Some(now.saturating_add_millis(self.interval_ms)); - self.fired_total = self.fired_total.saturating_add(1); - true - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct CoverageInputs { - pub rooms: usize, - pub gateway_reads_per_turn: u32, - pub livekit_reads_per_turn: u32, - pub gateway_read_rps: u32, - pub livekit_read_rps: u32, -} - -impl CoverageInputs { - pub const fn for_rooms(rooms: usize, config: &ReconConfig) -> Self { - Self { - rooms, - gateway_reads_per_turn: GATEWAY_READS_PER_ROOM_TURN, - livekit_reads_per_turn: LIVEKIT_READS_PER_ROOM_TURN, - gateway_read_rps: config.gateway_read_rps, - livekit_read_rps: config.livekit_read_rps, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum CoverageVerdict { - WithinTarget, - AboveTarget, -} - -impl CoverageVerdict { - pub const fn label(self) -> &'static str { - match self { - Self::WithinTarget => "within_target", - Self::AboveTarget => "above_target", - } - } -} - -fn lane_period_ms(rooms: usize, reads_per_turn: u32, rps: u32) -> u64 { - let calls = u64::try_from(rooms) - .unwrap_or(u64::MAX) - .saturating_mul(u64::from(reads_per_turn)); - let rate = u64::from(rps.max(1)); - calls.saturating_mul(1_000).div_ceil(rate) -} - -pub fn derived_coverage_period_ms(inputs: &CoverageInputs) -> u64 { - let gateway = lane_period_ms( - inputs.rooms, - inputs.gateway_reads_per_turn, - inputs.gateway_read_rps, - ); - let livekit = lane_period_ms( - inputs.rooms, - inputs.livekit_reads_per_turn, - inputs.livekit_read_rps, - ); - gateway.max(livekit) -} - -pub fn max_auditable_rooms(config: &ReconConfig) -> usize { - let gateway = u64::from(config.gateway_read_rps) - .saturating_mul(config.coverage_period_hard_cap_ms) - .checked_div(1_000 * u64::from(GATEWAY_READS_PER_ROOM_TURN.max(1))) - .unwrap_or(0); - let livekit = u64::from(config.livekit_read_rps) - .saturating_mul(config.coverage_period_hard_cap_ms) - .checked_div(1_000 * u64::from(LIVEKIT_READS_PER_ROOM_TURN.max(1))) - .unwrap_or(0); - usize::try_from(gateway.min(livekit)).unwrap_or(usize::MAX) -} - -pub fn check_coverage_period( - period_ms: u64, - target_ms: u64, - hard_cap_ms: u64, -) -> anyhow::Result { - if period_ms > hard_cap_ms { - anyhow::bail!( - "the derived coverage period is {period_ms} ms, above FLUXER_RECON_COVERAGE_PERIOD_HARD_CAP_MS ({hard_cap_ms} ms); the room count this process can audit at the configured read rates has been exceeded" - ); - } - if period_ms > target_ms { - return Ok(CoverageVerdict::AboveTarget); - } - Ok(CoverageVerdict::WithinTarget) -} diff --git a/fluxer_recon/src/service.rs b/fluxer_recon/src/service.rs deleted file mode 100644 index 7206925f0..000000000 --- a/fluxer_recon/src/service.rs +++ /dev/null @@ -1,1315 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::sync::Arc; -use std::time::Duration; - -use tokio::sync::RwLock; - -use fluxer_svc::config::{DatabaseBackend, ServiceConfig}; -use fluxer_svc::transport::Transport; - -use crate::actuate::{ActuationPolicy, Ports, TurnFacts, apply_decision}; -use crate::budget::{BreakerEvent, BudgetClass}; -use crate::census::{CensusOutcome, read_census}; -use crate::clock::{Clock, SharedClock, WallAnchor}; -use crate::config::ReconConfig; -use crate::decide::{Decision, decide}; -use crate::discovery::{ - ExpectationSource, LocationReadout, RoomContext, RoomReadPlan, RoomTurnReads, RoomView, - ServerRoomList, list_fleet_rooms, observe_room, read_room, record_health, -}; -use crate::evidence::{ - CandidateSet, GatewayRead, GatewayVoiceState, MediaSighting, SideKind, TopologyFreshness, - candidate_set, room_cliff_input, room_cliff_trips, -}; -use crate::gateway::GatewayApi; -use crate::gateway::nats::NatsGateway; -use crate::health::ModeClamp; -use crate::ids::{Location, Millis, RoomKey, TurnId, WallMillis}; -use crate::ledger::{LocationIndex, RoomLedger}; -use crate::livekit::twirp::{TwirpLiveKit, TwirpTimeouts}; -use crate::livekit::{ - LiveKitApi, LiveKitFault, ParticipantRecord, ReadResult, RemoveOutcome, ServerCredentials, -}; -use crate::metrics::ReconMetrics; -use crate::names::ParticipantIdentity; -use crate::observe::{DecisionJournal, ObservedSet, journal_decisions}; -use crate::runtime::{HOUSEKEEP_INTERVAL_MS, RuntimeState, Shared, TickInputs}; -use crate::schedule::{ - Cadence, CoverageInputs, CoverageVerdict, DeferReason, GATEWAY_READS_PER_ROOM_TURN, - check_coverage_period, derived_coverage_period_ms, max_auditable_rooms, -}; -use crate::suspicion::{SuspicionHint, SuspicionSource}; -use crate::topology::{InternalEndpoint, RefreshOutcome, TopologyStore, VoiceServer}; -use crate::turn::Fresh; -use crate::turn::{TurnScope, TurnToken}; - -pub const CADENCE_POLL_MS: u64 = 250; -pub const FORCED_TOPOLOGY_REFRESH_FLOOR_MS: u64 = 60_000; -pub const DISCOVERY_LIST_ROOMS_COST: u32 = 1; - -pub trait Uplink: Send + Sync { - fn is_connected(&self) -> bool; -} - -impl Uplink for NatsGateway { - fn is_connected(&self) -> bool { - self.transport().is_connected() - } -} - -pub trait Fleet { - fn install(&mut self, servers: Vec) -> anyhow::Result<()>; -} - -impl Fleet for TwirpLiveKit { - fn install(&mut self, servers: Vec) -> anyhow::Result<()> { - self.replace_servers(servers) - } -} - -#[derive(Debug)] -pub struct SharedFleet { - inner: Arc>, -} - -impl Clone for SharedFleet { - fn clone(&self) -> Self { - Self { - inner: Arc::clone(&self.inner), - } - } -} - -impl SharedFleet { - pub fn new(livekit: L) -> Self { - Self { - inner: Arc::new(RwLock::new(livekit)), - } - } -} - -impl SharedFleet { - pub async fn install(&self, servers: Vec) -> anyhow::Result<()> { - self.inner.write().await.install(servers) - } -} - -impl LiveKitApi for SharedFleet { - async fn list_rooms(&self, location: &Location) -> ReadResult>> { - self.inner.read().await.list_rooms(location).await - } - - async fn list_participants( - &self, - location: &Location, - room: RoomKey, - ) -> ReadResult> { - self.inner - .read() - .await - .list_participants(location, room) - .await - } - - async fn remove_participant( - &self, - location: &Location, - room: RoomKey, - identity: &ParticipantIdentity, - ) -> RemoveOutcome { - self.inner - .read() - .await - .remove_participant(location, room, identity) - .await - } -} - -pub struct GuardedClock { - inner: SharedClock, - anchor: WallAnchor, -} - -impl std::fmt::Debug for GuardedClock { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("GuardedClock") - .field("anchor", &self.anchor) - .finish() - } -} - -impl GuardedClock { - pub fn anchored(inner: SharedClock) -> Self { - let anchor = WallAnchor::new(inner.now(), inner.wall_now()); - Self { inner, anchor } - } - - pub fn shared(inner: SharedClock) -> SharedClock { - Arc::new(Self::anchored(inner)) - } -} - -impl Clock for GuardedClock { - fn now(&self) -> Millis { - self.inner.now() - } - - fn wall_now(&self) -> WallMillis { - self.anchor.guard(self.inner.now(), self.inner.wall_now()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum TurnOutcome { - Completed, - Unreadable, - Starved, - Broken, -} - -impl TurnOutcome { - pub const fn label(self) -> &'static str { - match self { - Self::Completed => "completed", - Self::Unreadable => "unreadable", - Self::Starved => "starved", - Self::Broken => "broken", - } - } -} - -struct RoomTurnPlan { - token: TurnToken, - locations: Vec, - gateway_reads: u32, -} - -struct RoomTurnPermits { - locations: Vec, - livekit_held: bool, -} - -struct TurnReading { - decisions: Vec, - media: Vec, - turn: TurnId, - outcome: TurnOutcome, - divergent: bool, -} - -pub struct ReconEngine { - shared: Shared, - gateway: G, - livekit: L, - clock: SharedClock, - policy: ActuationPolicy, -} - -impl ReconEngine -where - G: GatewayApi + Uplink, - L: LiveKitApi, -{ - pub fn new(shared: Shared, gateway: G, livekit: L, clock: SharedClock) -> Self { - let policy = ActuationPolicy::from_config(shared.config()); - Self { - shared, - gateway, - livekit, - clock, - policy, - } - } - - pub fn shared(&self) -> &Shared { - &self.shared - } - - fn config(&self) -> &ReconConfig { - self.shared.config() - } - - fn metrics(&self) -> Arc { - Arc::clone(self.shared.metrics()) - } - - pub async fn step(&self) { - let now = self.clock.now(); - - if let Some(report) = self.shared.tick_if_due( - TickInputs { - now, - nats_connected: self.gateway.is_connected(), - last_decision_at: self.shared.journal().last_at_ms().map(Millis::new), - }, - HOUSEKEEP_INTERVAL_MS, - ) { - if matches!(report.breaker, BreakerEvent::Tripped) { - tracing::error!( - divergent_fraction = report.divergent_fraction, - "divergent fraction over budget, breaker tripped and every corroboration reset" - ); - } - if self.shared.readiness().mark_ready() { - tracing::info!( - mode = report.effective_mode.as_str(), - tick_ms = self.config().tick_ms, - "recon engine ready" - ); - } - } - - let due = self - .shared - .with_state_mut(|state| admit_and_take_due(state, now)); - - for room in due { - let started = self.clock.now(); - let outcome = self.room_turn(room).await; - let settled = self.clock.now(); - self.metrics() - .observe_turn_duration(settled.saturating_since(started)); - self.shared - .with_state_mut(|state| settle_turn(state, room, outcome, settled)); - if matches!(outcome, TurnOutcome::Starved) { - self.metrics().record_scheduler_starvation(); - } - } - - self.close_pass_if_complete(); - } - - fn close_pass_if_complete(&self) { - let now = self.clock.now(); - let target_ms = self.config().coverage_target_ms; - let hard_cap_ms = self.config().coverage_period_hard_cap_ms; - let config = self.config().clone(); - - let closed = self.shared.with_state_mut(|state| { - state - .pass_pending - .retain(|room| state.scheduler.is_tracked(room)); - if !state.pass_pending.is_empty() { - return None; - } - if now.saturating_since(state.pass_started_at) < target_ms { - return None; - } - if state.scheduler.served_total() == 0 { - return None; - } - - state.counters.note_expectations_accounted(); - let cliffed = state.cliffs.observe_pass(&state.counters, now); - state.counters.clear(); - for location in &cliffed { - let rooms: Vec = state - .directory - .room_keys() - .into_iter() - .filter(|room| state.directory.locations_for(room).contains(location)) - .collect(); - for room in rooms { - if let Some(entry) = state.ledger.room_mut(&room) { - entry.reset_corroborations(); - } - } - } - - let period_ms = now.saturating_since(state.pass_started_at); - state.coverage_period_ms = period_ms; - state.pass_started_at = now; - state.pass_pending = state - .directory - .room_keys() - .into_iter() - .filter(|room| state.scheduler.is_tracked(room)) - .collect(); - state.governor.warmup_mut().note_coverage_pass(); - - let rooms = state.directory.len(); - Some((period_ms, rooms, cliffed)) - }); - - let Some((period_ms, rooms, cliffed)) = closed else { - return; - }; - - let derived = derived_coverage_period_ms(&CoverageInputs::for_rooms(rooms, &config)); - let verdict = check_coverage_period(derived, target_ms, hard_cap_ms); - let overrun = verdict.is_err(); - self.shared.with_state_mut(|state| { - state - .governor - .clamps_mut() - .set(ModeClamp::CoverageOverrun, overrun); - }); - match verdict { - Err(error) => { - tracing::error!( - error = %error, - rooms, - derived_coverage_period_ms = derived, - max_auditable_rooms = max_auditable_rooms(&config), - "the room count is beyond what this process can audit at the configured read \ - rates; the destructive lane is clamped off until the read rates or the room \ - count change" - ); - } - Ok(CoverageVerdict::AboveTarget) if derived > target_ms.saturating_mul(2) => { - tracing::warn!( - rooms, - derived_coverage_period_ms = derived, - coverage_target_ms = target_ms, - "the derived coverage period is more than twice the target" - ); - } - Ok(_) => {} - } - - tracing::debug!( - period_ms, - rooms, - cliffed = cliffed.len(), - "coverage pass complete" - ); - } - - async fn room_turn(&self, room: RoomKey) -> TurnOutcome { - let opened_at = self.clock.now(); - let Some(plan) = self - .shared - .with_state_mut(|state| plan_room_turn(state, room, opened_at)) - else { - return TurnOutcome::Starved; - }; - - let permits = RoomTurnPermits { - locations: plan.locations.clone(), - livekit_held: !plan.locations.is_empty(), - }; - let outcome = self.take_room_turn(room, plan).await; - self.shared - .with_state_mut(|state| release_room_turn(state, &permits)); - outcome - } - - async fn take_room_turn(&self, room: RoomKey, plan: RoomTurnPlan) -> TurnOutcome { - let RoomTurnPlan { - mut token, - locations, - gateway_reads, - } = plan; - - let reads = read_room( - &self.gateway, - &self.livekit, - &RoomReadPlan { - room, - locations: &locations, - }, - self.clock.as_ref(), - |_| true, - &mut token, - ) - .await; - - let fresh = match reads { - Err(error) => { - tracing::error!( - ?error, - channel_id = room.channel_id().get(), - "a room read broke the turn accounting, abandoning the turn" - ); - self.shared.with_state_mut(|state| { - state.sequencer.abandon(token); - }); - return TurnOutcome::Broken; - } - Ok(fresh) => fresh, - }; - - let at = self.clock.now(); - let wall_at = self.clock.wall_now(); - let metrics = self.metrics(); - let reading = self.shared.with_state_mut(|state| { - observe_and_decide( - state, - &ObserveInputs { - config: self.config(), - metrics: &metrics, - journal: self.shared.journal(), - room, - at, - wall_at, - gateway_reads, - }, - token, - fresh, - ) - }); - - let Some(reading) = reading else { - return TurnOutcome::Broken; - }; - - if reading.divergent { - let hint = SuspicionHint::new(room, SuspicionSource::Divergence, at); - self.shared.with_state_mut(|state| { - state.suspicion.note(hint); - }); - } - - let facts = TurnFacts { - room, - authorizing_turn: reading.turn, - media: reading.media, - }; - let ports = Ports { - gateway: &self.gateway, - livekit: &self.livekit, - clock: self.clock.as_ref(), - shared: &self.shared, - policy: self.policy, - }; - - for decision in &reading.decisions { - let actuation = apply_decision(&ports, &facts, decision).await; - if actuation.called_out() { - tracing::info!( - channel_id = room.channel_id().get(), - guild_id = room.guild_id().map(|guild| guild.get()), - connection_id = decision.connection.connection.as_str(), - user_id = decision.user_id.get(), - action = decision.action.label(), - outcome = ?actuation, - "recon issued a reconciliation call" - ); - } - } - - reading.outcome - } -} - -fn admit_and_take_due(state: &mut RuntimeState, now: Millis) -> Vec { - for room in state.directory.room_keys() { - if state.scheduler.track(room, now) { - state.pass_pending.insert(room); - } - } - - for room in state.suspicion.drain_raised() { - state.scheduler.promote(room, now); - } - - let free = state - .governor - .limits() - .max_inflight_room_turns - .min(state.scheduler.turns_per_tick()); - state.scheduler.take_due_up_to(now, free) -} - -fn settle_turn(state: &mut RuntimeState, room: RoomKey, outcome: TurnOutcome, now: Millis) { - match outcome { - TurnOutcome::Completed | TurnOutcome::Unreadable => { - state.scheduler.completed(room, now); - state.pass_pending.remove(&room); - } - TurnOutcome::Starved => { - state.scheduler.defer(room, now, DeferReason::Budget); - } - TurnOutcome::Broken => { - state.scheduler.defer(room, now, DeferReason::Unreadable); - } - } -} - -fn ledger_last_known(state: &RuntimeState, room: RoomKey) -> Vec { - let indexes = state - .ledger - .room(&room) - .map(|entry| entry.last_known_locations().to_vec()) - .unwrap_or_default(); - indexes - .into_iter() - .filter_map(|index| state.ledger.location(index).cloned()) - .collect() -} - -fn believed_homes(state: &RuntimeState, room: RoomKey) -> Vec { - let indexes = state - .ledger - .room(&room) - .map(RoomLedger::believed_homes) - .unwrap_or_default(); - indexes - .into_iter() - .filter_map(|index| state.ledger.location(index).cloned()) - .collect() -} - -fn planned_candidates(state: &RuntimeState, room: RoomKey, now: Millis) -> CandidateSet { - let sources = state - .directory - .sources_for(&room, Vec::new(), ledger_last_known(state, room)); - candidate_set(&sources, &state.topology.lens(now)) -} - -fn record_read_calls(metrics: &ReconMetrics, reads: &RoomTurnReads) { - let gateway_outcome = match &reads.gateway { - GatewayRead::Ok { .. } => "ok", - GatewayRead::Failed(fault) => fault.label(), - }; - if reads.gateway_calls > 1 { - metrics.record_gateway_rpc("voice_states_for_channel", "ok"); - metrics.record_gateway_rpc("pending_joins_for_channel", gateway_outcome); - } else { - metrics.record_gateway_rpc("voice_states_for_channel", gateway_outcome); - } - - for readout in &reads.readouts { - let outcome = readout.fault().map_or("ok", LiveKitFault::label); - metrics.record_livekit_call("list_participants", outcome); - } -} - -fn release_room_turn(state: &mut RuntimeState, permits: &RoomTurnPermits) { - for location in &permits.locations { - state.governor.server_inflight(location).release(); - } - if permits.livekit_held { - state.governor.livekit_inflight().release(); - } - state.governor.gateway_inflight().release(); - state.governor.room_turns().release(); -} - -fn abandon_room_turn(state: &mut RuntimeState, permits: &RoomTurnPermits) { - for location in &permits.locations { - state.governor.refund_server_read(location, 1); - } - release_room_turn(state, permits); -} - -fn admit_locations( - state: &mut RuntimeState, - candidates: Vec, - now: Millis, -) -> Option> { - let mut admitted: Vec = Vec::new(); - for location in candidates { - if !state.governor.server_inflight(&location).try_acquire() { - release_locations(state, &admitted); - return None; - } - if !state.governor.try_acquire_server_read(&location, 1, now) { - state.governor.server_inflight(&location).release(); - release_locations(state, &admitted); - return None; - } - admitted.push(location); - } - Some(admitted) -} - -fn release_locations(state: &mut RuntimeState, locations: &[Location]) { - for location in locations { - state.governor.refund_server_read(location, 1); - state.governor.server_inflight(location).release(); - } -} - -fn plan_room_turn(state: &mut RuntimeState, room: RoomKey, now: Millis) -> Option { - let candidates = planned_candidates(state, room, now); - let offered: Vec = candidates - .locations() - .iter() - .filter(|location| state.server_health.may_probe(location, now)) - .cloned() - .collect(); - - let gateway_reads = GATEWAY_READS_PER_ROOM_TURN; - - if !state.governor.room_turns().try_acquire() { - return None; - } - if !state.governor.gateway_inflight().try_acquire() { - state.governor.room_turns().release(); - return None; - } - - let Some(locations) = admit_locations(state, offered, now) else { - state.governor.gateway_inflight().release(); - state.governor.room_turns().release(); - return None; - }; - let livekit_reads = u32::try_from(locations.len()).unwrap_or(u32::MAX); - let livekit_held = livekit_reads > 0 && state.governor.livekit_inflight().try_acquire(); - let permits = RoomTurnPermits { - locations: locations.clone(), - livekit_held, - }; - - if livekit_reads > 0 && !livekit_held { - abandon_room_turn(state, &permits); - return None; - } - if state.governor.tokens(BudgetClass::GatewayRead, now) < f64::from(gateway_reads) { - abandon_room_turn(state, &permits); - return None; - } - if state.governor.tokens(BudgetClass::LiveKitRead, now) < f64::from(livekit_reads) { - abandon_room_turn(state, &permits); - return None; - } - if !state - .governor - .try_acquire_many(BudgetClass::GatewayRead, gateway_reads, now) - { - abandon_room_turn(state, &permits); - return None; - } - if livekit_reads > 0 - && !state - .governor - .try_acquire_many(BudgetClass::LiveKitRead, livekit_reads, now) - { - state - .governor - .refund(BudgetClass::GatewayRead, gateway_reads); - abandon_room_turn(state, &permits); - return None; - } - - Some(RoomTurnPlan { - token: state.sequencer.open(TurnScope::room_turn(room), now), - locations, - gateway_reads, - }) -} - -fn gateway_hints(states: &[GatewayVoiceState]) -> Vec { - let mut hints: Vec = states - .iter() - .filter_map(|state| state.hint.clone()) - .collect(); - hints.sort(); - hints.dedup(); - hints -} - -fn media_sightings(reads: &RoomTurnReads) -> Vec { - let mut sightings: Vec = Vec::new(); - for readout in &reads.readouts { - let Some(roster) = readout.roster() else { - continue; - }; - for participant in roster.participants() { - sightings.push(MediaSighting { - connection: participant.connection.clone(), - user_id: participant.user_id, - }); - } - } - sightings.sort_by(|left, right| left.connection.cmp(&right.connection)); - sightings.dedup_by(|left, right| left.connection == right.connection); - sightings -} - -fn note_roster_reads( - state: &mut RuntimeState, - room: RoomKey, - readouts: &[LocationReadout], - at: Millis, -) { - let indices: Vec = readouts - .iter() - .filter(|readout| readout.roster().is_some()) - .filter_map(|readout| state.ledger.intern_location(readout.location())) - .collect(); - if indices.is_empty() { - return; - } - if state.ledger.ensure_room(room, at).is_err() { - return; - } - let Some(entry) = state.ledger.room_mut(&room) else { - return; - }; - for index in indices { - entry.note_roster_read(index, at); - } -} - -fn note_expected_from_ledger(state: &mut RuntimeState, room: RoomKey) { - let Some(entry) = state.ledger.room(&room) else { - return; - }; - let counts: Vec<(Location, u32)> = entry - .believed_home_counts() - .into_iter() - .filter_map(|(index, count)| { - state - .ledger - .location(index) - .cloned() - .map(|location| (location, count)) - }) - .collect(); - for (location, count) in counts { - state - .counters - .note_room_expected(&location, room, ExpectationSource::Ledger, count); - } -} - -struct ObserveInputs<'a> { - config: &'a ReconConfig, - metrics: &'a ReconMetrics, - journal: &'a DecisionJournal, - room: RoomKey, - at: Millis, - wall_at: WallMillis, - gateway_reads: u32, -} - -fn observe_and_decide( - state: &mut RuntimeState, - inputs: &ObserveInputs<'_>, - token: TurnToken, - fresh: Fresh, -) -> Option { - let ObserveInputs { - config, - metrics, - journal, - room, - at, - wall_at, - gateway_reads, - } = *inputs; - - let turn = token.turn(); - state.turn = turn; - - record_read_calls(metrics, fresh.peek()); - - let unused_gateway_reads = gateway_reads.saturating_sub(fresh.peek().gateway_calls); - if unused_gateway_reads > 0 { - state - .governor - .refund(BudgetClass::GatewayRead, unused_gateway_reads); - } - - record_health(&fresh.peek().readouts, &mut state.server_health, at); - note_roster_reads(state, room, &fresh.peek().readouts, at); - if fresh - .peek() - .readouts - .iter() - .any(|readout| readout.fault().is_some_and(LiveKitFault::is_auth_failure)) - { - state.topology_refresh_requested = true; - } - - let hints = gateway_hints(fresh.peek().gateway_states()); - for hint in &hints { - state.directory.note_pinned(room, hint.clone(), at); - } - - let gateway_connections = fresh.peek().gateway_connections(); - let media_connections = fresh.peek().media_connections(); - let readable = matches!(fresh.peek().gateway, GatewayRead::Ok { .. }); - - let prior = state - .ledger - .room(&room) - .map(RoomLedger::prior_connections) - .unwrap_or_default(); - let cliff_input = room_cliff_input(&prior, &gateway_connections, &media_connections); - let hold_ms = config.room_cliff_hold_ms; - if state.ledger.ensure_room(room, at).is_ok() - && let Some(entry) = state.ledger.room_mut(&room) - { - if readable && room_cliff_trips(cliff_input) { - entry.gateway_cliff_mut().trip(at); - entry.reset_corroborations(); - } else { - entry.gateway_cliff_mut().expire(at, hold_ms); - } - } - let room_cliffed = state - .ledger - .room(&room) - .is_some_and(|entry| entry.gateway_cliff().is_held(at, hold_ms)); - - let media_hold_ms = config.server_cliff_hold_ms; - let readouts = &fresh.peek().readouts; - let media_fully_readable = - !readouts.is_empty() && readouts.iter().all(LocationReadout::is_readable); - let media_cliff_input = room_cliff_input(&prior, &media_connections, &gateway_connections); - if let Some(entry) = state.ledger.room_mut(&room) { - if media_fully_readable && room_cliff_trips(media_cliff_input) { - entry.media_cliff_mut().trip(at); - entry.reset_corroborations(); - } else { - entry.media_cliff_mut().expire(at, media_hold_ms); - } - } - let media_cliffed = state - .ledger - .room(&room) - .is_some_and(|entry| entry.media_cliff().is_held(at, media_hold_ms)); - - let sources = state - .directory - .sources_for(&room, hints, ledger_last_known(state, room)); - let candidates = candidate_set(&sources, &state.topology.lens(at)); - let census = state.census.cross_check(&room, at); - let topology = TopologyFreshness { - age_ms: state.topology.age_ms(at).unwrap_or(u64::MAX), - max_age_ms: config.topology_max_age_ms, - }; - let census_epoch = state.census.epoch(); - let topology_epoch = state.topology.epoch(); - let media = media_sightings(fresh.peek()); - let homes = believed_homes(state, room); - - let observed = { - let mut holed = state.suspicion.holed_servers(); - if media_cliffed { - holed.extend(candidates.locations().iter().cloned()); - holed.sort(); - holed.dedup(); - } - let context = RoomContext { - room, - turn, - at, - wall_at, - candidates: &candidates, - health: &state.server_health, - cliffs: &state.cliffs, - holed: &holed, - believed_homes: &homes, - room_cliffed, - census, - topology, - census_epoch, - topology_epoch, - max_connections_per_room: config.max_connections_per_room, - }; - let view = fresh.map(|reads| observe_room(reads, &context)); - state.counters.note_view(view.peek()); - state.sequencer.seal(token, at, view) - }; - - let view: RoomView = match observed { - Err(error) => { - tracing::error!( - ?error, - channel_id = room.channel_id().get(), - "sealing a room turn failed, so no decision was taken from it" - ); - return None; - } - Ok(observed) => observed.into_parts().1, - }; - - note_expected_from_ledger(state, room); - - let gateway_ok = view.authority().gateway.is_authoritative(); - let media_ok = view.authority().media.is_authoritative(); - let budget = state.governor.view(at); - let outcome = decide(&mut state.ledger, view.observation(), at, &budget); - let mode = state.governor.effective_mode(); - - for decision in &outcome.decisions { - metrics.record_decision(decision); - } - for (side, reason) in [ - (SideKind::Gateway, view.authority().gateway.unknown_reason()), - (SideKind::Media, view.authority().media.unknown_reason()), - ] { - if let Some(reason) = reason { - metrics.record_unknown(side, reason.label()); - } - } - - let ledger_snapshot = &state.ledger; - journal_decisions( - journal, - &ObservedSet { - set: &outcome, - observation: view.observation(), - mode, - pending_join_skew_ms: config.pending_join_skew_ms, - }, - |decision| ledger_snapshot.connection(&decision.connection).cloned(), - ); - - let divergent = state - .ledger - .room(&room) - .is_some_and(|entry| entry.divergent_connections() > 0); - - if state.ledger.room(&room).is_some() { - state.directory.note_ledger(room, at); - } - - Some(TurnReading { - decisions: outcome.decisions, - media, - turn, - outcome: if gateway_ok && media_ok { - TurnOutcome::Completed - } else { - TurnOutcome::Unreadable - }, - divergent, - }) -} - -pub async fn run_engine(engine: ReconEngine) -> anyhow::Result<()> -where - G: GatewayApi + Uplink, - L: LiveKitApi, -{ - let tick = Duration::from_millis(engine.config().tick_ms.max(1)); - let mut ticker = tokio::time::interval(tick); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - loop { - ticker.tick().await; - engine.step().await; - } -} - -pub async fn run_census(shared: Shared, gateway: G, clock: SharedClock) -> anyhow::Result<()> -where - G: GatewayApi, -{ - let mut cadence = Cadence::new(shared.config().census_interval_ms.max(1)); - let mut ticker = tokio::time::interval(Duration::from_millis(CADENCE_POLL_MS)); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - loop { - ticker.tick().await; - let now = clock.now(); - if !cadence.due(now) { - continue; - } - - take_census(&shared, &gateway, clock.as_ref()).await; - } -} - -pub async fn take_census(shared: &Shared, gateway: &G, clock: &dyn Clock) -where - G: GatewayApi, -{ - if !shared.with_state_mut(|state| state.governor.gateway_inflight().try_acquire()) { - return; - } - let read = read_census(gateway).await; - shared.with_state_mut(|state| state.governor.gateway_inflight().release()); - let label = read.label(); - shared - .metrics() - .record_gateway_rpc("active_voice_rooms", label); - let at = clock.now(); - let outcome = shared.with_state_mut(|state| { - let outcome = state.census.accept(&read, at); - let baseline = state.census.applied_total() <= 1; - if let Some(snapshot) = state.census.snapshot() { - state.census_at = Some(snapshot.taken_at()); - let seeded = state.directory.note_census(snapshot, at); - if !seeded.is_empty() { - tracing::debug!( - seeded = seeded.len(), - baseline, - "the census seeded rooms into the directory" - ); - } - if !baseline { - for room in seeded { - state.suspicion.note_room(room, SuspicionSource::Census, at); - } - } - } - outcome - }); - - match outcome { - CensusOutcome::Failed => { - tracing::warn!( - read = label, - "the census was unreadable, keeping the last good snapshot and skipping the \ - cross-check rather than stalling the service" - ); - } - CensusOutcome::Applied { rooms, .. } => { - tracing::debug!(rooms, "census applied"); - } - CensusOutcome::Unchanged { .. } => {} - } -} - -pub async fn run_discovery( - shared: Shared, - fleet: SharedFleet, - store: Arc, - clock: SharedClock, -) -> anyhow::Result<()> -where - L: LiveKitApi + Fleet + Send + Sync, -{ - let mut topology_cadence = Cadence::new(shared.config().topology_refresh_ms.max(1)); - let mut rooms_cadence = Cadence::new(shared.config().discovery_interval_ms.max(1)); - let mut forced = Cadence::new(FORCED_TOPOLOGY_REFRESH_FLOOR_MS); - let mut ticker = tokio::time::interval(Duration::from_millis(CADENCE_POLL_MS)); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - loop { - ticker.tick().await; - let now = clock.now(); - - if shared.with_state_mut(RuntimeState::take_topology_refresh_request) && forced.due(now) { - topology_cadence.force(); - } - - if topology_cadence.due(now) { - refresh_topology(&shared, &fleet, store.as_ref(), clock.as_ref()).await; - } - - if rooms_cadence.due(now) { - discover_rooms(&shared, &fleet, clock.as_ref()).await; - } - } -} - -pub fn internal_endpoint(config: &ReconConfig) -> InternalEndpoint { - InternalEndpoint { - url: config.livekit_internal_url.clone(), - default_region_id: config.livekit_default_region_id.clone(), - } -} - -fn credentials_of(servers: &[VoiceServer], internal: &InternalEndpoint) -> Vec { - servers - .iter() - .map(|server| ServerCredentials { - location: server.location().clone(), - endpoint: Box::from(internal.resolve(server)), - api_key: Box::from(server.api_key()), - api_secret: server.api_secret().clone(), - }) - .collect() -} - -pub async fn refresh_topology( - shared: &Shared, - fleet: &SharedFleet, - store: &dyn TopologyStore, - clock: &dyn Clock, -) where - L: LiveKitApi + Fleet + Send + Sync, -{ - let loaded = store.load().await; - let at = clock.now(); - let internal = internal_endpoint(shared.config()); - - let (outcome, credentials) = shared.with_state_mut(|state| match loaded { - Err(error) => { - tracing::warn!( - backend = store.backend(), - error = %error, - "the topology load failed, keeping the last good snapshot and letting it age out" - ); - (state.topology.note_failure(at), Vec::new()) - } - Ok(servers) => { - let outcome = state.topology.accept(servers, at); - let credentials = credentials_of(state.topology.servers(), &internal); - if !outcome.kept_previous_snapshot() { - state.topology_loaded_at = Some(at); - state.topology_servers = state.topology.servers().len(); - } - (outcome, credentials) - } - }); - - match outcome { - RefreshOutcome::Applied { epoch, servers } => { - if let Err(error) = fleet.install(credentials).await { - tracing::error!(error = %error, "the livekit fleet refused a topology snapshot"); - } - tracing::info!( - backend = store.backend(), - epoch = epoch.get(), - servers, - "topology applied" - ); - } - RefreshOutcome::Unchanged { .. } => {} - RefreshOutcome::RejectedEmpty => { - tracing::error!( - backend = store.backend(), - "the topology load returned zero servers, which is refused: an empty fleet would \ - make every room's media side look empty rather than unreadable" - ); - } - RefreshOutcome::Failed => {} - } -} - -pub async fn discover_rooms(shared: &Shared, fleet: &SharedFleet, clock: &dyn Clock) -where - L: LiveKitApi + Fleet + Send + Sync, -{ - let now = clock.now(); - let locations = shared.with_state(|state| { - state - .topology - .servers() - .iter() - .map(|server| server.location().clone()) - .collect::>() - }); - if locations.is_empty() { - return; - } - - let admitted = shared.with_state_mut(|state| { - let mut admitted: Vec = Vec::new(); - if !state.governor.livekit_inflight().try_acquire() { - return admitted; - } - for location in &locations { - if !state.server_health.may_probe(location, now) { - continue; - } - if !state.governor.server_inflight(location).try_acquire() { - continue; - } - if !state - .governor - .try_acquire_server_read(location, DISCOVERY_LIST_ROOMS_COST, now) - { - state.governor.server_inflight(location).release(); - continue; - } - if !state.governor.try_acquire_many( - BudgetClass::LiveKitRead, - DISCOVERY_LIST_ROOMS_COST, - now, - ) { - state - .governor - .refund_server_read(location, DISCOVERY_LIST_ROOMS_COST); - state.governor.server_inflight(location).release(); - break; - } - admitted.push(location.clone()); - } - if admitted.is_empty() { - state.governor.livekit_inflight().release(); - } - admitted - }); - - let results = list_fleet_rooms(fleet, &admitted, |_| true).await; - let at = clock.now(); - - for (_, list) in &results { - let outcome = match list { - ServerRoomList::Listed { .. } => "ok", - ServerRoomList::Unreadable(fault) => fault.label(), - }; - shared.metrics().record_livekit_call("list_rooms", outcome); - } - - shared.with_state_mut(|state| { - if !admitted.is_empty() { - state.governor.livekit_inflight().release(); - } - for location in &admitted { - state.governor.server_inflight(location).release(); - } - }); - - shared.with_state_mut(|state| { - for (location, list) in &results { - state.server_health.record(location, list.outcome(), at); - if list.fault().is_some_and(LiveKitFault::is_auth_failure) { - state.topology_refresh_requested = true; - } - let filled = list.is_complete() - && state - .suspicion - .hole_for(location) - .is_some_and(|hole| hole.since().get() <= now.get()) - && state.suspicion.clear_hole(location); - if filled { - tracing::info!( - region = location.region.as_str(), - server = location.server.as_str(), - "a full room list read after the dropped webhook closed that server's hole" - ); - } - } - let pass = state.directory.apply_pass(&results, at); - for room in state.directory.drain_relocated() { - state - .suspicion - .note_room(room, SuspicionSource::Discovery, at); - } - if pass.unreadable_servers() > 0 { - tracing::debug!( - servers = pass.servers, - unreadable = pass.unreadable_servers(), - "a discovery pass was incomplete, so no location was withdrawn from those servers" - ); - } - }); -} - -pub async fn connect_topology(service: &ServiceConfig) -> anyhow::Result> { - match service.database_backend { - DatabaseBackend::Postgres => { - let config = fluxer_svc::postgres::PostgresConfig::from_service_config(service); - let store = crate::topology::postgres::PostgresTopology::connect(&config).await?; - Ok(Arc::new(store)) - } - DatabaseBackend::Cassandra => connect_scylla_topology(service).await, - } -} - -#[cfg(feature = "scylla")] -async fn connect_scylla_topology( - service: &ServiceConfig, -) -> anyhow::Result> { - let config = fluxer_svc::scylla::ScyllaConfig::from_service_config(service); - let store = crate::topology::scylla::ScyllaTopology::connect(&config).await?; - Ok(Arc::new(store)) -} - -#[cfg(not(feature = "scylla"))] -async fn connect_scylla_topology( - _service: &ServiceConfig, -) -> anyhow::Result> { - anyhow::bail!( - "FLUXER_DATABASE_BACKEND selects cassandra but this binary was built without the scylla feature" - ) -} - -pub fn build_livekit( - servers: &[VoiceServer], - internal: &InternalEndpoint, -) -> anyhow::Result { - TwirpLiveKit::new(credentials_of(servers, internal), TwirpTimeouts::DEFAULT) -} diff --git a/fluxer_recon/src/singleton.rs b/fluxer_recon/src/singleton.rs deleted file mode 100644 index 9e37519f1..000000000 --- a/fluxer_recon/src/singleton.rs +++ /dev/null @@ -1,245 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -use fluxer_svc::transport::{Transport, TransportMessage, TransportSubscriber, reply_message}; - -use crate::control::INSTANCE_SUBJECT; -use crate::ids::Millis; -use crate::runtime::{Shared, monotonic_now, wall_now_ms}; - -pub const SINGLETON_PROBE_INTERVAL_MS: u64 = 30_000; -pub const SINGLETON_PROBE_DEADLINE_MS: u64 = 750; -pub const PEER_CLAMP_RELEASE_AFTER_MS: u64 = 3 * SINGLETON_PROBE_INTERVAL_MS; -pub const RESUBSCRIBE_BACKOFF_MS: u64 = 1_000; -pub const RESUBSCRIBE_BACKOFF_CEILING_MS: u64 = 30_000; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum SingletonLayer { - Deployment, - RuntimeDetector, - SameTurnAuthorization, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum SingletonResponse { - RefuseToBoot, - DegradeToConstructive, - NoActionCorrectnessHolds, -} - -impl SingletonResponse { - pub const fn terminates_a_running_process(self) -> bool { - match self { - Self::RefuseToBoot | Self::DegradeToConstructive | Self::NoActionCorrectnessHolds => { - false - } - } - } -} - -impl SingletonLayer { - pub const ALL: [Self; 3] = [ - Self::Deployment, - Self::RuntimeDetector, - Self::SameTurnAuthorization, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::Deployment => "deployment", - Self::RuntimeDetector => "runtime_detector", - Self::SameTurnAuthorization => "same_turn_authorization", - } - } - - pub const fn response(self) -> SingletonResponse { - match self { - Self::Deployment => SingletonResponse::RefuseToBoot, - Self::RuntimeDetector => SingletonResponse::DegradeToConstructive, - Self::SameTurnAuthorization => SingletonResponse::NoActionCorrectnessHolds, - } - } - - pub const fn acts_before_serving(self) -> bool { - matches!(self, Self::Deployment) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct InstanceIdentity { - pub id: String, - pub started_at: u64, -} - -impl InstanceIdentity { - pub fn generate() -> Self { - static SEQUENCE: AtomicU64 = AtomicU64::new(0); - let started_at = wall_now_ms(); - let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed); - let entropy = mix(u64::from(std::process::id())) - ^ mix(started_at) - ^ mix(monotonic_now().get()) - ^ mix(sequence); - Self { - id: format!("{entropy:016x}"), - started_at, - } - } - - pub fn announce(&self) -> InstanceAnnounce { - InstanceAnnounce { - instance_id: self.id.clone(), - started_at: self.started_at, - } - } - - pub fn probe(&self) -> InstanceProbe { - InstanceProbe { - instance_id: self.id.clone(), - } - } -} - -const fn mix(value: u64) -> u64 { - let mut hash = value ^ 0x9e37_79b9_7f4a_7c15; - hash = (hash ^ (hash >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); - hash = (hash ^ (hash >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); - hash ^ (hash >> 31) -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct InstanceProbe { - pub instance_id: String, -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct InstanceAnnounce { - pub instance_id: String, - pub started_at: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PeerVerdict { - SelfEcho, - Peer, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PeerRecord { - pub instance_id: String, - pub started_at: u64, - pub first_seen_at: Millis, - pub last_seen_at: Millis, - pub detections: u64, -} - -pub fn should_answer_probe(own: &InstanceIdentity, probe: &InstanceProbe) -> bool { - probe.instance_id != own.id -} - -pub fn peer_verdict(own: &InstanceIdentity, announce: &InstanceAnnounce) -> PeerVerdict { - if announce.instance_id == own.id { - PeerVerdict::SelfEcho - } else { - PeerVerdict::Peer - } -} - -pub const fn next_backoff_ms(previous: u64) -> u64 { - let doubled = previous.saturating_mul(2); - if doubled > RESUBSCRIBE_BACKOFF_CEILING_MS { - RESUBSCRIBE_BACKOFF_CEILING_MS - } else { - doubled - } -} - -pub async fn run_singleton(shared: Shared, transport: T) -> anyhow::Result<()> { - let announce = serde_json::to_vec(&shared.instance().announce())?; - let probe = serde_json::to_vec(&shared.instance().probe())?; - let deadline = Duration::from_millis(SINGLETON_PROBE_DEADLINE_MS); - let mut ticker = tokio::time::interval(Duration::from_millis(SINGLETON_PROBE_INTERVAL_MS)); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut backoff_ms = RESUBSCRIBE_BACKOFF_MS; - - loop { - let mut subscription = match transport.subscribe(INSTANCE_SUBJECT).await { - Ok(subscription) => { - backoff_ms = RESUBSCRIBE_BACKOFF_MS; - subscription - } - Err(error) => { - tracing::error!( - error = %error, - subject = INSTANCE_SUBJECT, - retry_in_ms = backoff_ms, - "the singleton detector could not subscribe, retrying rather than ending the \ - task, because a restart loop is worse than a detector that is briefly deaf" - ); - tokio::time::sleep(Duration::from_millis(backoff_ms)).await; - backoff_ms = next_backoff_ms(backoff_ms); - continue; - } - }; - - tracing::info!( - subject = INSTANCE_SUBJECT, - instance_id = shared.instance().id, - "singleton detector listening" - ); - - loop { - tokio::select! { - message = subscription.next() => { - let Some(message) = message else { - tracing::warn!("singleton subscription ended, will re-subscribe"); - break; - }; - let requester: InstanceProbe = match serde_json::from_slice(message.payload()) { - Ok(requester) => requester, - Err(error) => { - tracing::debug!(error = %error, "ignoring an undecodable instance probe"); - continue; - } - }; - if !should_answer_probe(shared.instance(), &requester) { - continue; - } - if let Err(error) = reply_message(&message, &transport, &announce).await { - tracing::debug!(error = %error, "failed to answer an instance probe"); - } - } - _ = ticker.tick() => { - let reply = transport.request(INSTANCE_SUBJECT, &probe, deadline).await; - let Ok(bytes) = reply else { - continue; - }; - let Ok(peer) = serde_json::from_slice::(&bytes) else { - continue; - }; - if matches!(peer_verdict(shared.instance(), &peer), PeerVerdict::SelfEcho) { - continue; - } - shared.note_peer(&peer, monotonic_now()); - tracing::error!( - peer_instance_id = peer.instance_id, - peer_started_at = peer.started_at, - own_instance_id = shared.instance().id, - layer = SingletonLayer::RuntimeDetector.label(), - response = ?SingletonLayer::RuntimeDetector.response(), - "a second recon instance answered, degrading below the destructive lane \ - and continuing to serve" - ); - } - _ = transport.wait_for_reconnect() => { - tracing::info!("NATS reconnected, re-subscribing the singleton detector"); - break; - } - } - } - } -} diff --git a/fluxer_recon/src/suspicion.rs b/fluxer_recon/src/suspicion.rs deleted file mode 100644 index 23f9fe95b..000000000 --- a/fluxer_recon/src/suspicion.rs +++ /dev/null @@ -1,356 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::BTreeMap; - -use crate::config::ReconConfig; -use crate::ids::{Location, Millis, RoomKey}; -use crate::ledger::{LOCATION_MAP_BYTES, MAP_ENTRY_OVERHEAD_BYTES}; - -pub const SUSPICION_ENTRY_BYTES: u64 = - (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SuspicionSource { - Webhook, - WebhookDrop, - Census, - Discovery, - Divergence, - Control, -} - -impl SuspicionSource { - pub const ALL: [Self; 6] = [ - Self::Webhook, - Self::WebhookDrop, - Self::Census, - Self::Discovery, - Self::Divergence, - Self::Control, - ]; - - pub const fn label(self) -> &'static str { - match self { - Self::Webhook => "webhook", - Self::WebhookDrop => "webhook_drop", - Self::Census => "census", - Self::Discovery => "discovery", - Self::Divergence => "divergence", - Self::Control => "control", - } - } - - pub const fn index(self) -> usize { - match self { - Self::Webhook => 0, - Self::WebhookDrop => 1, - Self::Census => 2, - Self::Discovery => 3, - Self::Divergence => 4, - Self::Control => 5, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct SuspicionHint { - room: RoomKey, - source: SuspicionSource, - at: Millis, -} - -impl SuspicionHint { - pub const fn new(room: RoomKey, source: SuspicionSource, at: Millis) -> Self { - Self { room, source, at } - } - - pub const fn room(&self) -> RoomKey { - self.room - } - - pub const fn source(&self) -> SuspicionSource { - self.source - } - - pub const fn at(&self) -> Millis { - self.at - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Raised { - Raised, - Extended, - Refused, -} - -impl Raised { - pub const fn label(self) -> &'static str { - match self { - Self::Raised => "raised", - Self::Extended => "extended", - Self::Refused => "refused", - } - } - - pub const fn is_held(self) -> bool { - matches!(self, Self::Raised | Self::Extended) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct SuspicionLimits { - pub hold_ms: u64, - pub max_rooms: usize, -} - -impl SuspicionLimits { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - hold_ms: config.suspicion_hold_ms, - max_rooms: config.max_hot_rooms, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Hold { - source: SuspicionSource, - since: Millis, - until: Millis, - hints: u32, -} - -impl Hold { - pub const fn source(&self) -> SuspicionSource { - self.source - } - - pub const fn since(&self) -> Millis { - self.since - } - - pub const fn until(&self) -> Millis { - self.until - } - - pub const fn hints(&self) -> u32 { - self.hints - } - - pub const fn is_held(&self, now: Millis) -> bool { - now.get() < self.until.get() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ServerHole { - dropped: u32, - since: Millis, - events: u32, -} - -impl ServerHole { - pub const fn dropped(&self) -> u32 { - self.dropped - } - - pub const fn since(&self) -> Millis { - self.since - } - - pub const fn events(&self) -> u32 { - self.events - } -} - -#[derive(Clone, Debug)] -pub struct SuspicionLane { - limits: SuspicionLimits, - holds: BTreeMap, - raised: Vec, - holes: BTreeMap, - by_source: [u64; SuspicionSource::ALL.len()], - raised_total: u64, - extended_total: u64, - refused_total: u64, - expired_total: u64, - holes_total: u64, -} - -impl SuspicionLane { - pub fn tracked_bytes(&self) -> u64 { - (self.holds.len() as u64) - .saturating_mul(SUSPICION_ENTRY_BYTES) - .saturating_add((self.holes.len() as u64).saturating_mul(LOCATION_MAP_BYTES)) - } - - pub const fn new(limits: SuspicionLimits) -> Self { - Self { - limits, - holds: BTreeMap::new(), - raised: Vec::new(), - holes: BTreeMap::new(), - by_source: [0; SuspicionSource::ALL.len()], - raised_total: 0, - extended_total: 0, - refused_total: 0, - expired_total: 0, - holes_total: 0, - } - } - - pub fn from_config(config: &ReconConfig) -> Self { - Self::new(SuspicionLimits::from_config(config)) - } - - pub const fn limits(&self) -> SuspicionLimits { - self.limits - } - - pub const fn raised_total(&self) -> u64 { - self.raised_total - } - - pub const fn extended_total(&self) -> u64 { - self.extended_total - } - - pub const fn refused_total(&self) -> u64 { - self.refused_total - } - - pub const fn expired_total(&self) -> u64 { - self.expired_total - } - - pub const fn holes_total(&self) -> u64 { - self.holes_total - } - - pub const fn hints_from(&self, source: SuspicionSource) -> u64 { - self.by_source[source.index()] - } - - pub fn held(&self) -> usize { - self.holds.len() - } - - pub fn is_empty(&self) -> bool { - self.holds.is_empty() - } - - pub fn hold_for(&self, room: &RoomKey) -> Option<&Hold> { - self.holds.get(room) - } - - pub fn is_suspect(&self, room: &RoomKey, now: Millis) -> bool { - self.holds.get(room).is_some_and(|hold| hold.is_held(now)) - } - - pub fn suspect_rooms(&self, now: Millis) -> Vec { - self.holds - .iter() - .filter(|(_, hold)| hold.is_held(now)) - .map(|(room, _)| *room) - .collect() - } - - pub fn note(&mut self, hint: SuspicionHint) -> Raised { - self.by_source[hint.source.index()] = self.by_source[hint.source.index()].saturating_add(1); - let until = hint.at.saturating_add_millis(self.limits.hold_ms); - - if let Some(hold) = self.holds.get_mut(&hint.room) { - hold.until = until; - hold.source = hint.source; - hold.hints = hold.hints.saturating_add(1); - self.extended_total = self.extended_total.saturating_add(1); - return Raised::Extended; - } - - if self.holds.len() >= self.limits.max_rooms { - self.refused_total = self.refused_total.saturating_add(1); - return Raised::Refused; - } - - self.holds.insert( - hint.room, - Hold { - source: hint.source, - since: hint.at, - until, - hints: 1, - }, - ); - self.raised.push(hint.room); - self.raised_total = self.raised_total.saturating_add(1); - Raised::Raised - } - - pub fn note_room(&mut self, room: RoomKey, source: SuspicionSource, now: Millis) -> Raised { - self.note(SuspicionHint::new(room, source, now)) - } - - pub fn drain_raised(&mut self) -> Vec { - std::mem::take(&mut self.raised) - } - - pub fn expire(&mut self, now: Millis) -> usize { - let expired: Vec = self - .holds - .iter() - .filter(|(_, hold)| !hold.is_held(now)) - .map(|(room, _)| *room) - .collect(); - for room in &expired { - self.holds.remove(room); - } - self.expired_total = self.expired_total.saturating_add(expired.len() as u64); - expired.len() - } - - pub fn forget(&mut self, room: &RoomKey) -> bool { - self.raised.retain(|held| held != room); - self.holds.remove(room).is_some() - } - - pub fn note_dropped(&mut self, location: &Location, dropped: u32, now: Millis) -> bool { - if dropped == 0 { - return false; - } - match self.holes.get_mut(location) { - Some(hole) => { - hole.dropped = hole.dropped.saturating_add(dropped); - hole.events = hole.events.saturating_add(1); - } - None => { - self.holes.insert( - location.clone(), - ServerHole { - dropped, - since: now, - events: 1, - }, - ); - self.holes_total = self.holes_total.saturating_add(1); - } - } - true - } - - pub fn hole_for(&self, location: &Location) -> Option<&ServerHole> { - self.holes.get(location) - } - - pub fn holed_servers(&self) -> Vec { - self.holes.keys().cloned().collect() - } - - pub fn clear_hole(&mut self, location: &Location) -> bool { - self.holes.remove(location).is_some() - } - - pub fn holes(&self) -> usize { - self.holes.len() - } -} diff --git a/fluxer_recon/src/topology/mod.rs b/fluxer_recon/src/topology/mod.rs deleted file mode 100644 index 29d891024..000000000 --- a/fluxer_recon/src/topology/mod.rs +++ /dev/null @@ -1,453 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -pub mod postgres; - -#[cfg(feature = "scylla")] -pub mod scylla; - -use std::future::Future; -use std::pin::Pin; - -use crate::config::ReconConfig; -use crate::evidence::{ServerRecord, TopologyFreshness, TopologyLens}; -use crate::ids::{ApiSecret, Epoch, IdError, Location, Millis, RegionId, ServerId}; - -pub const VOICE_SERVERS_TABLE: &str = "voice_servers"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum RowError { - #[error("the {0} column is missing")] - MissingColumn(&'static str), - #[error("the {0} column is empty")] - EmptyColumn(&'static str), - #[error("the {column} column is not a valid identifier: {source}")] - InvalidColumn { - column: &'static str, - source: IdError, - }, - #[error("the row cannot be decoded as a voice_servers row")] - Undecodable, -} - -#[derive(Debug, thiserror::Error)] -pub enum TopologyError { - #[error("the topology backend is unavailable: {0}")] - Unavailable(String), - #[error("the topology query failed: {0}")] - Query(String), - #[error("a voice_servers row is malformed: {0}")] - Row(#[from] RowError), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct VoiceServerRow { - pub region_id: String, - pub server_id: String, - pub endpoint: String, - pub api_key: String, - pub api_secret: String, - pub is_active: bool, -} - -impl VoiceServerRow { - pub fn into_server(self) -> Result { - let region = RegionId::new(self.region_id.trim()).map_err(|source| match source { - IdError::Empty => RowError::EmptyColumn("region_id"), - IdError::ContainsWhitespace | IdError::ContainsUnderscore => RowError::InvalidColumn { - column: "region_id", - source, - }, - })?; - let server = ServerId::new(self.server_id.trim()).map_err(|source| match source { - IdError::Empty => RowError::EmptyColumn("server_id"), - IdError::ContainsWhitespace | IdError::ContainsUnderscore => RowError::InvalidColumn { - column: "server_id", - source, - }, - })?; - - let endpoint = self.endpoint.trim(); - if endpoint.is_empty() { - return Err(RowError::EmptyColumn("endpoint")); - } - let api_key = self.api_key.trim(); - if api_key.is_empty() { - return Err(RowError::EmptyColumn("api_key")); - } - if self.api_secret.is_empty() { - return Err(RowError::EmptyColumn("api_secret")); - } - - Ok(VoiceServer { - location: Location::new(region, server), - endpoint: Box::from(endpoint), - api_key: Box::from(api_key), - api_secret: ApiSecret::new(&self.api_secret), - is_active: self.is_active, - }) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct InternalEndpoint { - pub url: Option, - pub default_region_id: Option, -} - -impl InternalEndpoint { - pub fn resolve<'a>(&'a self, server: &'a VoiceServer) -> &'a str { - let (Some(url), Some(region)) = (self.url.as_deref(), self.default_region_id.as_deref()) - else { - return server.endpoint(); - }; - if server.region().as_str() != region - || server.server().as_str() != format!("{region}-server-1") - { - return server.endpoint(); - } - url - } -} - -pub fn default_region_id(blob: &str) -> Option { - let trimmed = blob.trim(); - if trimmed.is_empty() { - return None; - } - let parsed: serde_json::Value = serde_json::from_str(trimmed).ok()?; - match parsed { - serde_json::Value::String(id) => Some(id), - serde_json::Value::Object(fields) => fields - .get("id") - .and_then(serde_json::Value::as_str) - .map(str::to_owned), - _ => None, - } - .filter(|id| !id.trim().is_empty()) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct VoiceServer { - location: Location, - endpoint: Box, - api_key: Box, - api_secret: ApiSecret, - is_active: bool, -} - -impl VoiceServer { - pub const fn location(&self) -> &Location { - &self.location - } - - pub const fn region(&self) -> &RegionId { - &self.location.region - } - - pub const fn server(&self) -> &ServerId { - &self.location.server - } - - pub fn endpoint(&self) -> &str { - &self.endpoint - } - - pub fn api_key(&self) -> &str { - &self.api_key - } - - pub const fn api_secret(&self) -> &ApiSecret { - &self.api_secret - } - - pub const fn is_active(&self) -> bool { - self.is_active - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TopologySnapshot { - servers: Vec, - records: Vec, - loaded_at: Millis, - epoch: Epoch, -} - -impl TopologySnapshot { - pub fn servers(&self) -> &[VoiceServer] { - &self.servers - } - - pub fn records(&self) -> &[ServerRecord] { - &self.records - } - - pub const fn loaded_at(&self) -> Millis { - self.loaded_at - } - - pub const fn epoch(&self) -> Epoch { - self.epoch - } - - pub fn len(&self) -> usize { - self.servers.len() - } - - pub fn is_empty(&self) -> bool { - self.servers.is_empty() - } - - pub fn draining(&self) -> Vec { - self.records - .iter() - .filter(|record| record.removed_at.is_some()) - .map(|record| record.location.clone()) - .collect() - } - - pub fn server(&self, location: &Location) -> Option<&VoiceServer> { - self.servers - .iter() - .find(|server| &server.location == location) - } - - pub fn locations(&self) -> Vec { - self.servers - .iter() - .map(|server| server.location.clone()) - .collect() - } - - pub const fn age_ms(&self, now: Millis) -> u64 { - now.saturating_since(self.loaded_at) - } - - pub const fn is_stale(&self, now: Millis, max_age_ms: u64) -> bool { - self.age_ms(now) > max_age_ms - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TopologyLimits { - pub max_age_ms: u64, - pub drain_grace_ms: u64, -} - -impl TopologyLimits { - pub const fn from_config(config: &ReconConfig) -> Self { - Self { - max_age_ms: config.topology_max_age_ms, - drain_grace_ms: config.topology_drain_grace_ms, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RefreshOutcome { - Applied { epoch: Epoch, servers: usize }, - Unchanged { epoch: Epoch }, - RejectedEmpty, - Failed, -} - -impl RefreshOutcome { - pub const fn label(self) -> &'static str { - match self { - Self::Applied { .. } => "applied", - Self::Unchanged { .. } => "unchanged", - Self::RejectedEmpty => "rejected_empty", - Self::Failed => "failed", - } - } - - pub const fn kept_previous_snapshot(self) -> bool { - match self { - Self::Applied { .. } | Self::Unchanged { .. } => false, - Self::RejectedEmpty | Self::Failed => true, - } - } -} - -#[derive(Clone, Debug)] -pub struct TopologyCache { - limits: TopologyLimits, - snapshot: Option, - epoch: Epoch, - last_failure_at: Option, - applied_total: u64, - unchanged_total: u64, - rejected_empty_total: u64, - failed_total: u64, -} - -impl TopologyCache { - pub const fn new(limits: TopologyLimits) -> Self { - Self { - limits, - snapshot: None, - epoch: Epoch::FIRST, - last_failure_at: None, - applied_total: 0, - unchanged_total: 0, - rejected_empty_total: 0, - failed_total: 0, - } - } - - pub const fn limits(&self) -> TopologyLimits { - self.limits - } - - pub const fn snapshot(&self) -> Option<&TopologySnapshot> { - self.snapshot.as_ref() - } - - pub const fn epoch(&self) -> Epoch { - self.epoch - } - - pub const fn last_failure_at(&self) -> Option { - self.last_failure_at - } - - pub const fn applied_total(&self) -> u64 { - self.applied_total - } - - pub const fn unchanged_total(&self) -> u64 { - self.unchanged_total - } - - pub const fn rejected_empty_total(&self) -> u64 { - self.rejected_empty_total - } - - pub const fn failed_total(&self) -> u64 { - self.failed_total - } - - pub fn records(&self) -> &[ServerRecord] { - self.snapshot - .as_ref() - .map_or(&[], TopologySnapshot::records) - } - - pub fn servers(&self) -> &[VoiceServer] { - self.snapshot - .as_ref() - .map_or(&[], TopologySnapshot::servers) - } - - pub fn server(&self, location: &Location) -> Option<&VoiceServer> { - self.snapshot - .as_ref() - .and_then(|snapshot| snapshot.server(location)) - } - - pub fn age_ms(&self, now: Millis) -> Option { - self.snapshot.as_ref().map(|snapshot| snapshot.age_ms(now)) - } - - pub fn is_stale(&self, now: Millis) -> bool { - self.snapshot - .as_ref() - .is_none_or(|snapshot| snapshot.is_stale(now, self.limits.max_age_ms)) - } - - pub fn freshness(&self, now: Millis) -> TopologyFreshness { - TopologyFreshness { - age_ms: self.age_ms(now).unwrap_or(u64::MAX), - max_age_ms: self.limits.max_age_ms, - } - } - - pub fn lens(&self, now: Millis) -> TopologyLens<'_> { - TopologyLens { - servers: self.records(), - now, - drain_grace_ms: self.limits.drain_grace_ms, - } - } - - pub fn note_failure(&mut self, now: Millis) -> RefreshOutcome { - self.failed_total = self.failed_total.saturating_add(1); - self.last_failure_at = Some(now); - RefreshOutcome::Failed - } - - pub fn accept(&mut self, servers: Vec, now: Millis) -> RefreshOutcome { - if servers.is_empty() { - self.rejected_empty_total = self.rejected_empty_total.saturating_add(1); - return RefreshOutcome::RejectedEmpty; - } - - let mut servers = servers; - servers.sort_by(|left, right| left.location.cmp(&right.location)); - servers.dedup_by(|left, right| left.location == right.location); - let records = self.records_for(&servers, now); - - let unchanged = self - .snapshot - .as_ref() - .is_some_and(|snapshot| snapshot.servers == servers && snapshot.records == records); - if unchanged { - self.unchanged_total = self.unchanged_total.saturating_add(1); - if let Some(snapshot) = self.snapshot.as_mut() { - snapshot.loaded_at = now; - } - return RefreshOutcome::Unchanged { epoch: self.epoch }; - } - - self.epoch = self.epoch.next(); - self.applied_total = self.applied_total.saturating_add(1); - let count = servers.len(); - self.snapshot = Some(TopologySnapshot { - servers, - records, - loaded_at: now, - epoch: self.epoch, - }); - RefreshOutcome::Applied { - epoch: self.epoch, - servers: count, - } - } - - fn records_for(&self, servers: &[VoiceServer], now: Millis) -> Vec { - let mut records: Vec = servers - .iter() - .map(|server| ServerRecord { - location: server.location.clone(), - removed_at: None, - }) - .collect(); - - if let Some(previous) = self.snapshot.as_ref() { - for record in &previous.records { - if servers - .iter() - .any(|server| server.location == record.location) - { - continue; - } - records.push(ServerRecord { - location: record.location.clone(), - removed_at: Some(record.removed_at.unwrap_or(now)), - }); - } - } - - records.sort_by(|left, right| left.location.cmp(&right.location)); - records - } -} - -pub type LoadFuture<'a> = - Pin, TopologyError>> + Send + 'a>>; - -pub trait TopologyStore: Send + Sync { - fn load(&self) -> LoadFuture<'_>; - - fn backend(&self) -> &'static str; -} diff --git a/fluxer_recon/src/topology/postgres.rs b/fluxer_recon/src/topology/postgres.rs deleted file mode 100644 index 4dec563ca..000000000 --- a/fluxer_recon/src/topology/postgres.rs +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use serde::Deserialize; -use serde_json::Value; - -use fluxer_svc::postgres::{KvClient, PostgresConfig, connect as connect_pool, decode_row}; - -use super::{ - LoadFuture, RowError, TopologyError, TopologyStore, VOICE_SERVERS_TABLE, VoiceServer, - VoiceServerRow, -}; - -pub const BACKEND: &str = "postgres"; -const WHOLE_TABLE_PREFIX: &str = ""; - -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(default)] -struct VoiceServerKvRow { - region_id: Option, - server_id: Option, - endpoint: Option, - api_key: Option, - api_secret: Option, - is_active: Option, -} - -fn column(value: Option, name: &'static str) -> Result { - value.ok_or(RowError::MissingColumn(name)) -} - -pub fn row_from_json(value: Value) -> Result { - let decoded = decode_row(value).map_err(|_| RowError::Undecodable)?; - if !decoded.is_object() { - return Err(RowError::Undecodable); - } - let row: VoiceServerKvRow = - serde_json::from_value(decoded).map_err(|_| RowError::Undecodable)?; - - Ok(VoiceServerRow { - region_id: column(row.region_id, "region_id")?, - server_id: column(row.server_id, "server_id")?, - endpoint: column(row.endpoint, "endpoint")?, - api_key: column(row.api_key, "api_key")?, - api_secret: column(row.api_secret, "api_secret")?, - is_active: row.is_active.unwrap_or(false), - }) -} - -#[derive(Clone)] -pub struct PostgresTopology { - kv: KvClient, -} - -impl PostgresTopology { - pub fn new(kv: KvClient) -> Self { - Self { kv } - } - - pub async fn connect(config: &PostgresConfig) -> anyhow::Result { - let pool = connect_pool(config).await?; - Ok(Self::new(KvClient::new(pool, config)?)) - } - - async fn read(&self) -> Result, TopologyError> { - let rows = self - .kv - .get_row_key_prefix_rows(VOICE_SERVERS_TABLE, WHOLE_TABLE_PREFIX) - .await - .map_err(|error| TopologyError::Query(error.to_string()))?; - - let mut servers = Vec::with_capacity(rows.len()); - for (_, value) in rows { - servers.push(row_from_json(value)?.into_server()?); - } - Ok(servers) - } -} - -impl std::fmt::Debug for PostgresTopology { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("PostgresTopology") - .field("table", &VOICE_SERVERS_TABLE) - .finish() - } -} - -impl TopologyStore for PostgresTopology { - fn load(&self) -> LoadFuture<'_> { - Box::pin(async move { self.read().await }) - } - - fn backend(&self) -> &'static str { - BACKEND - } -} diff --git a/fluxer_recon/src/topology/scylla.rs b/fluxer_recon/src/topology/scylla.rs deleted file mode 100644 index cf32d95c9..000000000 --- a/fluxer_recon/src/topology/scylla.rs +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::sync::Arc; - -use scylla::DeserializeRow; -use scylla::client::session::Session; -use scylla::statement::prepared::PreparedStatement; - -use fluxer_svc::scylla::{ScyllaConfig, connect}; - -use super::{LoadFuture, RowError, TopologyError, TopologyStore, VoiceServer, VoiceServerRow}; - -pub const BACKEND: &str = "scylla"; -pub const VOICE_SERVERS_CQL: &str = - "SELECT region_id, server_id, endpoint, api_key, api_secret, is_active FROM voice_servers"; - -#[derive(Debug, DeserializeRow)] -struct VoiceServerCqlRow { - region_id: Option, - server_id: Option, - endpoint: Option, - api_key: Option, - api_secret: Option, - is_active: Option, -} - -fn column(value: Option, name: &'static str) -> Result { - value.ok_or(RowError::MissingColumn(name)) -} - -fn row_from_cql(row: VoiceServerCqlRow) -> Result { - Ok(VoiceServerRow { - region_id: column(row.region_id, "region_id")?, - server_id: column(row.server_id, "server_id")?, - endpoint: column(row.endpoint, "endpoint")?, - api_key: column(row.api_key, "api_key")?, - api_secret: column(row.api_secret, "api_secret")?, - is_active: row.is_active.unwrap_or(false), - }) -} - -#[derive(Clone)] -pub struct ScyllaTopology { - session: Arc, - statement: PreparedStatement, -} - -impl ScyllaTopology { - pub async fn new(session: Arc) -> anyhow::Result { - let statement = session.prepare(VOICE_SERVERS_CQL).await?; - Ok(Self { session, statement }) - } - - pub async fn connect(config: &ScyllaConfig) -> anyhow::Result { - Self::new(connect(config).await?).await - } - - async fn read(&self) -> Result, TopologyError> { - let result = self - .session - .execute_unpaged(&self.statement, ()) - .await - .map_err(|error| TopologyError::Unavailable(error.to_string()))?; - let rows = result - .into_rows_result() - .map_err(|error| TopologyError::Query(error.to_string()))?; - - let mut servers = Vec::new(); - for row in rows - .rows::() - .map_err(|error| TopologyError::Query(error.to_string()))? - { - let row = row.map_err(|error| TopologyError::Query(error.to_string()))?; - servers.push(row_from_cql(row)?.into_server()?); - } - Ok(servers) - } -} - -impl std::fmt::Debug for ScyllaTopology { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("ScyllaTopology") - .field("statement", &VOICE_SERVERS_CQL) - .finish() - } -} - -impl TopologyStore for ScyllaTopology { - fn load(&self) -> LoadFuture<'_> { - Box::pin(async move { self.read().await }) - } - - fn backend(&self) -> &'static str { - BACKEND - } -} diff --git a/fluxer_recon/src/turn.rs b/fluxer_recon/src/turn.rs deleted file mode 100644 index fc657bf6b..000000000 --- a/fluxer_recon/src/turn.rs +++ /dev/null @@ -1,597 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::collections::{BTreeMap, BTreeSet}; - -use crate::ids::{Millis, RoomKey, TurnId}; -use crate::ledger::MAP_ENTRY_OVERHEAD_BYTES; - -pub const SEQUENCER_ENTRY_BYTES: u64 = - (size_of::() + size_of::() + MAP_ENTRY_OVERHEAD_BYTES) as u64; - -const FIRST_TURN: TurnId = TurnId::new(1); -const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Digest(u64); - -impl Default for Digest { - fn default() -> Self { - Self::new() - } -} - -impl Digest { - pub const fn new() -> Self { - Self(FNV_OFFSET) - } - - pub const fn bytes(self, bytes: &[u8]) -> Self { - let mut hash = self.0; - let mut index = 0; - while index < bytes.len() { - hash ^= bytes[index] as u64; - hash = hash.wrapping_mul(FNV_PRIME); - index += 1; - } - Self(hash) - } - - pub const fn text(self, text: &str) -> Self { - self.bytes(text.as_bytes()) - } - - pub const fn number(self, value: u64) -> Self { - Self(self.0 ^ mix(value)).bytes(&[0xff]) - } - - pub const fn flag(self, value: bool) -> Self { - self.number(value as u64) - } - - pub const fn finish(self) -> u64 { - self.0 - } -} - -pub fn digest_bytes(bytes: &[u8]) -> u64 { - Digest::new().bytes(bytes).finish() -} - -pub fn digest_str(text: &str) -> u64 { - Digest::new().text(text).finish() -} - -const fn mix(value: u64) -> u64 { - let mut hash = value; - hash ^= hash >> 30; - hash = hash.wrapping_mul(0xbf58_476d_1ce4_e5b9); - hash ^= hash >> 27; - hash = hash.wrapping_mul(0x94d0_49bb_1331_11eb); - hash ^ (hash >> 31) -} - -fn fold_digest(receipts: &[ReadReceipt]) -> u64 { - let mut digest = Digest::new(); - for receipt in receipts { - digest = digest - .number(receipt.digest) - .number(u64::from(receipt.ticket)); - } - digest.finish() -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum TurnPurpose { - RoomTurn, - Preflight, -} - -impl TurnPurpose { - pub const fn label(self) -> &'static str { - match self { - Self::RoomTurn => "room_turn", - Self::Preflight => "preflight", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TurnScope { - pub room: RoomKey, - pub purpose: TurnPurpose, -} - -impl TurnScope { - pub const fn room_turn(room: RoomKey) -> Self { - Self { - room, - purpose: TurnPurpose::RoomTurn, - } - } - - pub const fn preflight(room: RoomKey) -> Self { - Self { - room, - purpose: TurnPurpose::Preflight, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ReadSource { - GatewayVoiceStates, - GatewayPendingJoins, - MediaParticipants, - MediaRoomList, - Census, -} - -impl ReadSource { - pub const fn label(self) -> &'static str { - match self { - Self::GatewayVoiceStates => "gateway_voice_states", - Self::GatewayPendingJoins => "gateway_pending_joins", - Self::MediaParticipants => "media_participants", - Self::MediaRoomList => "media_room_list", - Self::Census => "census", - } - } -} - -impl std::fmt::Display for ReadSource { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.label()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ReadReceipt { - ticket: u32, - source: ReadSource, - at: Millis, - digest: u64, -} - -impl ReadReceipt { - pub const fn ticket(self) -> u32 { - self.ticket - } - - pub const fn source(self) -> ReadSource { - self.source - } - - pub const fn at(self) -> Millis { - self.at - } - - pub const fn digest(self) -> u64 { - self.digest - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum TurnError { - #[error("a read taken in turn {found} cannot be used in turn {expected}")] - StaleRead { expected: TurnId, found: TurnId }, - #[error("turn {turn} took {issued} reads but the observation accounts for {accounted}")] - UnaccountedReads { - turn: TurnId, - issued: usize, - accounted: usize, - }, - #[error("turn {turn} opened at {opened_at} cannot be sealed at {sealed_at}")] - SealedBeforeOpened { - turn: TurnId, - opened_at: Millis, - sealed_at: Millis, - }, - #[error("turn {turn} was never issued by this sequencer")] - UnknownTurn { turn: TurnId }, - #[error("turn {turn} carries a {read} read taken at {at}, outside the turn")] - ReadOutsideTurn { - turn: TurnId, - read: ReadSource, - at: Millis, - }, -} - -#[derive(Debug)] -#[must_use = "an opened turn must be sealed or abandoned"] -pub struct TurnToken { - turn: TurnId, - scope: TurnScope, - opened_at: Millis, - receipts: Vec, - discarded: BTreeSet, -} - -impl TurnToken { - pub const fn turn(&self) -> TurnId { - self.turn - } - - pub const fn scope(&self) -> TurnScope { - self.scope - } - - pub const fn room(&self) -> RoomKey { - self.scope.room - } - - pub const fn opened_at(&self) -> Millis { - self.opened_at - } - - pub fn reads(&self) -> &[ReadReceipt] { - &self.receipts - } - - pub fn read(&mut self, source: ReadSource, at: Millis, digest: u64, value: T) -> Fresh { - let ticket = u32::try_from(self.receipts.len()).unwrap_or(u32::MAX); - self.receipts.push(ReadReceipt { - ticket, - source, - at, - digest, - }); - Fresh { - turn: self.turn, - tickets: vec![ticket], - value, - } - } - - pub fn discard(&mut self, value: Fresh) -> Result { - if value.turn != self.turn { - return Err(TurnError::StaleRead { - expected: self.turn, - found: value.turn, - }); - } - self.discarded.extend(value.tickets.iter().copied()); - Ok(value.value) - } -} - -#[derive(Debug)] -#[must_use = "a read must be sealed into its turn or discarded"] -pub struct Fresh { - turn: TurnId, - tickets: Vec, - value: T, -} - -impl Fresh { - pub fn turn(&self) -> TurnId { - self.turn - } - - pub fn peek(&self) -> &T { - &self.value - } - - pub fn map(self, transform: F) -> Fresh - where - F: FnOnce(T) -> U, - { - Fresh { - turn: self.turn, - tickets: self.tickets, - value: transform(self.value), - } - } - - pub fn zip(self, other: Fresh) -> Result, TurnError> { - if self.turn != other.turn { - return Err(TurnError::StaleRead { - expected: self.turn, - found: other.turn, - }); - } - let mut tickets = self.tickets; - tickets.extend(other.tickets); - tickets.sort_unstable(); - tickets.dedup(); - Ok(Fresh { - turn: self.turn, - tickets, - value: (self.value, other.value), - }) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TurnRecord { - turn: TurnId, - scope: TurnScope, - opened_at: Millis, - sealed_at: Millis, - digest: u64, - reads: Vec, -} - -impl TurnRecord { - pub const fn turn(&self) -> TurnId { - self.turn - } - - pub const fn scope(&self) -> TurnScope { - self.scope - } - - pub const fn room(&self) -> RoomKey { - self.scope.room - } - - pub const fn opened_at(&self) -> Millis { - self.opened_at - } - - pub const fn sealed_at(&self) -> Millis { - self.sealed_at - } - - pub const fn digest(&self) -> u64 { - self.digest - } - - pub fn reads(&self) -> &[ReadReceipt] { - &self.reads - } - - pub const fn duration_ms(&self) -> u64 { - self.sealed_at.saturating_since(self.opened_at) - } - - pub const fn age_ms(&self, now: Millis) -> u64 { - now.saturating_since(self.sealed_at) - } - - pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { - self.age_ms(now) <= max_age_ms - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Observed { - record: TurnRecord, - value: T, -} - -impl Observed { - pub const fn record(&self) -> &TurnRecord { - &self.record - } - - pub const fn turn(&self) -> TurnId { - self.record.turn - } - - pub const fn scope(&self) -> TurnScope { - self.record.scope - } - - pub const fn room(&self) -> RoomKey { - self.record.scope.room - } - - pub const fn opened_at(&self) -> Millis { - self.record.opened_at - } - - pub const fn sealed_at(&self) -> Millis { - self.record.sealed_at - } - - pub const fn digest(&self) -> u64 { - self.record.digest - } - - pub fn reads(&self) -> &[ReadReceipt] { - &self.record.reads - } - - pub const fn age_ms(&self, now: Millis) -> u64 { - self.record.age_ms(now) - } - - pub const fn is_fresh(&self, now: Millis, max_age_ms: u64) -> bool { - self.record.is_fresh(now, max_age_ms) - } - - pub const fn value(&self) -> &T { - &self.value - } - - pub fn into_parts(self) -> (TurnRecord, T) { - (self.record, self.value) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoomTurns { - last_opened: TurnId, - last_sealed: Option, -} - -impl RoomTurns { - pub const fn last_opened(&self) -> TurnId { - self.last_opened - } - - pub fn last_sealed(&self) -> Option<&TurnRecord> { - self.last_sealed.as_ref() - } -} - -#[derive(Clone, Debug)] -pub struct TurnSequencer { - next: TurnId, - opened_total: u64, - sealed_total: u64, - abandoned_total: u64, - rooms: BTreeMap, -} - -impl Default for TurnSequencer { - fn default() -> Self { - Self::new() - } -} - -impl TurnSequencer { - pub fn tracked_bytes(&self) -> u64 { - (self.rooms.len() as u64).saturating_mul(SEQUENCER_ENTRY_BYTES) - } - - pub const fn new() -> Self { - Self { - next: FIRST_TURN, - opened_total: 0, - sealed_total: 0, - abandoned_total: 0, - rooms: BTreeMap::new(), - } - } - - pub const fn next_turn(&self) -> TurnId { - self.next - } - - pub const fn opened_total(&self) -> u64 { - self.opened_total - } - - pub const fn sealed_total(&self) -> u64 { - self.sealed_total - } - - pub const fn abandoned_total(&self) -> u64 { - self.abandoned_total - } - - pub fn tracked_rooms(&self) -> usize { - self.rooms.len() - } - - pub fn history(&self, room: &RoomKey) -> Option<&RoomTurns> { - self.rooms.get(room) - } - - pub fn last_sealed(&self, room: &RoomKey) -> Option<&TurnRecord> { - self.rooms.get(room).and_then(RoomTurns::last_sealed) - } - - pub fn is_latest_sealed(&self, room: &RoomKey, turn: TurnId) -> bool { - self.last_sealed(room) - .is_some_and(|record| record.turn == turn) - } - - pub fn forget(&mut self, room: &RoomKey) -> bool { - self.rooms.remove(room).is_some() - } - - pub fn retain(&mut self, mut keep: F) - where - F: FnMut(&RoomKey) -> bool, - { - self.rooms.retain(|room, _| keep(room)); - } - - pub fn open(&mut self, scope: TurnScope, at: Millis) -> TurnToken { - let turn = self.next; - self.next = self.next.next(); - self.opened_total = self.opened_total.saturating_add(1); - self.rooms - .entry(scope.room) - .and_modify(|history| history.last_opened = turn) - .or_insert(RoomTurns { - last_opened: turn, - last_sealed: None, - }); - TurnToken { - turn, - scope, - opened_at: at, - receipts: Vec::new(), - discarded: BTreeSet::new(), - } - } - - pub fn seal( - &mut self, - token: TurnToken, - at: Millis, - value: Fresh, - ) -> Result, TurnError> { - if token.turn >= self.next { - return Err(TurnError::UnknownTurn { turn: token.turn }); - } - if value.turn != token.turn { - return Err(TurnError::StaleRead { - expected: token.turn, - found: value.turn, - }); - } - if at < token.opened_at { - return Err(TurnError::SealedBeforeOpened { - turn: token.turn, - opened_at: token.opened_at, - sealed_at: at, - }); - } - - for receipt in &token.receipts { - if receipt.at < token.opened_at || receipt.at > at { - return Err(TurnError::ReadOutsideTurn { - turn: token.turn, - read: receipt.source, - at: receipt.at, - }); - } - } - - let accounted: BTreeSet = value - .tickets - .iter() - .copied() - .chain(token.discarded.iter().copied()) - .collect(); - if accounted.len() != token.receipts.len() { - return Err(TurnError::UnaccountedReads { - turn: token.turn, - issued: token.receipts.len(), - accounted: accounted.len(), - }); - } - - let record = TurnRecord { - turn: token.turn, - scope: token.scope, - opened_at: token.opened_at, - sealed_at: at, - digest: fold_digest(&token.receipts), - reads: token.receipts, - }; - self.sealed_total = self.sealed_total.saturating_add(1); - self.rooms - .entry(record.scope.room) - .and_modify(|history| history.last_sealed = Some(record.clone())) - .or_insert_with(|| RoomTurns { - last_opened: record.turn, - last_sealed: Some(record.clone()), - }); - - Ok(Observed { - record, - value: value.value, - }) - } - - pub fn abandon(&mut self, token: TurnToken) -> TurnId { - self.abandoned_total = self.abandoned_total.saturating_add(1); - token.turn - } -} diff --git a/fluxer_recon/src/webhook.rs b/fluxer_recon/src/webhook.rs deleted file mode 100644 index 16cef8114..000000000 --- a/fluxer_recon/src/webhook.rs +++ /dev/null @@ -1,481 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use std::net::SocketAddr; - -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode, header}; -use base64::prelude::*; -use hmac::{Hmac, KeyInit, Mac}; -use serde::Deserialize; -use serde_json::Value; -use sha2::{Digest, Sha256}; - -use crate::clock::SharedClock; -use crate::ids::RoomKey; -use crate::ids::{ApiSecret, Location, Millis}; -use crate::names::{ParticipantIdentity, parse_participant_identity, parse_room_name}; -use crate::runtime::{RuntimeState, Shared}; -use crate::suspicion::{Raised, SuspicionSource}; - -type HmacSha256 = Hmac; - -pub const WEBHOOK_PATH: &str = "/livekit/webhook"; -pub const MAX_WEBHOOK_BODY_BYTES: usize = 64 * 1024; - -const HEADER_ALG: &str = "HS256"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -pub enum WebhookError { - #[error("the webhook token is not three dot separated parts")] - Malformed, - #[error("the webhook token is not signed with HS256")] - UnsupportedHeader, - #[error("a webhook token part is not base64url")] - BadBase64, - #[error("the webhook token does not decode as json")] - BadJson, - #[error("the webhook token signature does not verify")] - BadSignature, - #[error("the webhook token names an api key this fleet does not carry")] - UnknownIssuer, - #[error("the webhook token carries no body digest")] - MissingDigest, - #[error("the webhook body does not match the digest the token signed")] - DigestMismatch, - #[error("the webhook token has expired")] - Expired, - #[error("the webhook body does not decode as json")] - BadBody, -} - -impl WebhookError { - pub const fn label(self) -> &'static str { - match self { - Self::Malformed => "malformed", - Self::UnsupportedHeader => "unsupported_header", - Self::BadBase64 => "bad_base64", - Self::BadJson => "bad_json", - Self::BadSignature => "bad_signature", - Self::UnknownIssuer => "unknown_issuer", - Self::MissingDigest => "missing_digest", - Self::DigestMismatch => "digest_mismatch", - Self::Expired => "expired", - Self::BadBody => "bad_body", - } - } - - pub const fn is_authentication_failure(self) -> bool { - match self { - Self::Malformed - | Self::UnsupportedHeader - | Self::BadBase64 - | Self::BadJson - | Self::BadSignature - | Self::UnknownIssuer - | Self::MissingDigest - | Self::DigestMismatch - | Self::Expired => true, - Self::BadBody => false, - } - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -struct TokenHeader { - alg: String, -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] -pub struct WebhookClaims { - #[serde(default)] - pub iss: String, - #[serde(default)] - pub exp: u64, - #[serde(default)] - pub sha256: Option, -} - -fn decode_part(part: &str) -> Result, WebhookError> { - BASE64_URL_SAFE_NO_PAD - .decode(part.as_bytes()) - .or_else(|_| BASE64_STANDARD.decode(part.as_bytes())) - .map_err(|_| WebhookError::BadBase64) -} - -fn decode_digest(value: &str) -> Option> { - BASE64_STANDARD - .decode(value.as_bytes()) - .or_else(|_| BASE64_URL_SAFE_NO_PAD.decode(value.as_bytes())) - .ok() -} - -pub fn body_digest(body: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(body); - BASE64_STANDARD.encode(hasher.finalize()) -} - -pub fn verify_webhook_token( - token: &str, - api_key: &str, - api_secret: &ApiSecret, - body: &[u8], - now_unix_seconds: u64, -) -> Result { - let token = token.trim(); - let token = token.strip_prefix("Bearer ").unwrap_or(token).trim(); - - let mut parts = token.split('.'); - let (Some(header_b64), Some(payload_b64), Some(signature_b64), None) = - (parts.next(), parts.next(), parts.next(), parts.next()) - else { - return Err(WebhookError::Malformed); - }; - - let header_bytes = decode_part(header_b64)?; - let header: TokenHeader = - serde_json::from_slice(&header_bytes).map_err(|_| WebhookError::BadJson)?; - if header.alg != HEADER_ALG { - return Err(WebhookError::UnsupportedHeader); - } - - let payload_bytes = decode_part(payload_b64)?; - let claims: WebhookClaims = - serde_json::from_slice(&payload_bytes).map_err(|_| WebhookError::BadJson)?; - if claims.iss != api_key { - return Err(WebhookError::UnknownIssuer); - } - - let signature = decode_part(signature_b64)?; - let mut mac = HmacSha256::new_from_slice(api_secret.expose().as_bytes()) - .expect("hmac accepts any key length"); - mac.update(header_b64.as_bytes()); - mac.update(b"."); - mac.update(payload_b64.as_bytes()); - mac.verify_slice(&signature) - .map_err(|_| WebhookError::BadSignature)?; - - let Some(claimed) = claims.sha256.as_deref() else { - return Err(WebhookError::MissingDigest); - }; - let (Some(claimed), Some(actual)) = (decode_digest(claimed), decode_digest(&body_digest(body))) - else { - return Err(WebhookError::DigestMismatch); - }; - if claimed != actual { - return Err(WebhookError::DigestMismatch); - } - - if claims.exp > 0 && now_unix_seconds >= claims.exp { - return Err(WebhookError::Expired); - } - - Ok(claims) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum WebhookEventKind { - RoomStarted, - RoomFinished, - ParticipantJoined, - ParticipantLeft, - TrackPublished, - TrackUnpublished, - Unrecognised, -} - -impl WebhookEventKind { - pub const ALL: [Self; 7] = [ - Self::RoomStarted, - Self::RoomFinished, - Self::ParticipantJoined, - Self::ParticipantLeft, - Self::TrackPublished, - Self::TrackUnpublished, - Self::Unrecognised, - ]; - - pub const fn wire(self) -> &'static str { - match self { - Self::RoomStarted => "room_started", - Self::RoomFinished => "room_finished", - Self::ParticipantJoined => "participant_joined", - Self::ParticipantLeft => "participant_left", - Self::TrackPublished => "track_published", - Self::TrackUnpublished => "track_unpublished", - Self::Unrecognised => "unrecognised", - } - } - - pub fn from_wire(value: &str) -> Self { - match value { - "room_started" => Self::RoomStarted, - "room_finished" => Self::RoomFinished, - "participant_joined" => Self::ParticipantJoined, - "participant_left" => Self::ParticipantLeft, - "track_published" => Self::TrackPublished, - "track_unpublished" => Self::TrackUnpublished, - _ => Self::Unrecognised, - } - } - - pub const fn changes_the_roster(self) -> bool { - match self { - Self::RoomFinished - | Self::ParticipantJoined - | Self::ParticipantLeft - | Self::TrackPublished - | Self::TrackUnpublished => true, - Self::RoomStarted | Self::Unrecognised => false, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct WebhookEvent { - pub kind: WebhookEventKind, - pub room: Option, - pub identity: Option, - pub num_dropped: u32, - pub created_at_unix_seconds: u64, - pub unparseable_room: bool, - pub unparseable_identity: bool, -} - -fn text_of(value: &Value) -> Option<&str> { - value.as_str() -} - -fn number_of(value: &Value) -> Option { - match value { - Value::Number(number) => number.as_u64(), - Value::String(raw) => raw.parse::().ok(), - Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None, - } -} - -pub fn decode_event(body: &[u8]) -> Result { - let value: Value = serde_json::from_slice(body).map_err(|_| WebhookError::BadBody)?; - if !value.is_object() { - return Err(WebhookError::BadBody); - } - - let kind = value - .get("event") - .and_then(text_of) - .map_or(WebhookEventKind::Unrecognised, WebhookEventKind::from_wire); - - let named_room = value.get("room").and_then(|room| room.get("name")); - let room = named_room.and_then(text_of).map(parse_room_name); - let named_identity = value - .get("participant") - .and_then(|participant| participant.get("identity")); - let identity = named_identity - .and_then(text_of) - .map(parse_participant_identity); - - Ok(WebhookEvent { - kind, - room: room - .as_ref() - .and_then(|parsed| parsed.as_ref().ok()) - .copied(), - identity: identity - .as_ref() - .and_then(|parsed| parsed.as_ref().ok()) - .cloned(), - num_dropped: value - .get("numDropped") - .or_else(|| value.get("num_dropped")) - .and_then(number_of) - .and_then(|dropped| u32::try_from(dropped).ok()) - .unwrap_or(0), - created_at_unix_seconds: value - .get("createdAt") - .or_else(|| value.get("created_at")) - .and_then(number_of) - .unwrap_or(0), - unparseable_room: named_room.is_some() && room.is_none_or(|parsed| parsed.is_err()), - unparseable_identity: named_identity.is_some() - && identity.is_none_or(|parsed| parsed.is_err()), - }) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Ingested { - pub raised: Option, - pub holed: bool, -} - -impl Ingested { - pub const fn nothing() -> Self { - Self { - raised: None, - holed: false, - } - } -} - -pub fn ingest( - state: &mut RuntimeState, - event: &WebhookEvent, - location: &Location, - at: Millis, -) -> Ingested { - let holed = state - .suspicion - .note_dropped(location, event.num_dropped, at); - let source = if holed { - SuspicionSource::WebhookDrop - } else { - SuspicionSource::Webhook - }; - - let raised = match event.room { - None => None, - Some(room) => { - if !holed && !event.kind.changes_the_roster() { - None - } else { - state.directory.note_suspicion(room, at); - Some(state.suspicion.note_room(room, source, at)) - } - } - }; - - Ingested { raised, holed } -} - -#[derive(Clone)] -pub struct WebhookLane { - shared: Shared, - clock: SharedClock, -} - -impl WebhookLane { - pub const fn new(shared: Shared, clock: SharedClock) -> Self { - Self { shared, clock } - } -} - -impl std::fmt::Debug for WebhookLane { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.debug_struct("WebhookLane").finish() - } -} - -fn credentials(shared: &Shared) -> Vec<(Location, Box, ApiSecret)> { - shared.with_state(|state| { - state - .topology - .servers() - .iter() - .map(|server| { - ( - server.location().clone(), - Box::from(server.api_key()), - server.api_secret().clone(), - ) - }) - .collect() - }) -} - -pub fn authenticate( - shared: &Shared, - token: &str, - body: &[u8], - now_unix_seconds: u64, -) -> Result { - let mut failure = WebhookError::UnknownIssuer; - for (location, api_key, api_secret) in credentials(shared) { - match verify_webhook_token(token, &api_key, &api_secret, body, now_unix_seconds) { - Ok(_) => return Ok(location), - Err(WebhookError::UnknownIssuer) => {} - Err(error) => failure = error, - } - } - Err(failure) -} - -pub fn accept( - shared: &Shared, - clock: &SharedClock, - token: &str, - body: &[u8], -) -> Result { - let wall_now = clock.wall_now().get() / 1_000; - let location = authenticate(shared, token, body, wall_now)?; - let event = decode_event(body)?; - let at = clock.now(); - - let ingested = shared.with_state_mut(|state| ingest(state, &event, &location, at)); - - let age_seconds = if event.created_at_unix_seconds == 0 { - 0.0 - } else { - wall_now.saturating_sub(event.created_at_unix_seconds) as f64 - }; - shared - .metrics() - .set_webhook_last_event_age(location.server.as_str(), age_seconds); - - Ok(ingested) -} - -async fn receive( - State(lane): State, - headers: HeaderMap, - body: axum::body::Bytes, -) -> StatusCode { - if body.len() > MAX_WEBHOOK_BODY_BYTES { - return StatusCode::PAYLOAD_TOO_LARGE; - } - let token = headers - .get(header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - - match accept(&lane.shared, &lane.clock, token, &body) { - Ok(ingested) => { - tracing::debug!( - raised = ?ingested.raised, - holed = ingested.holed, - "a livekit webhook raised suspicion, which only reorders reads" - ); - StatusCode::NO_CONTENT - } - Err(error) => { - tracing::warn!( - error = error.label(), - "a livekit webhook was refused, so detection stays on the polling path" - ); - if error.is_authentication_failure() { - StatusCode::UNAUTHORIZED - } else { - StatusCode::BAD_REQUEST - } - } - } -} - -pub fn router(shared: Shared, clock: SharedClock) -> axum::Router { - axum::Router::new() - .route(WEBHOOK_PATH, axum::routing::post(receive)) - .with_state(WebhookLane::new(shared, clock)) -} - -pub async fn run_webhook( - shared: Shared, - clock: SharedClock, - address: SocketAddr, -) -> anyhow::Result<()> { - let listener = tokio::net::TcpListener::bind(address).await?; - tracing::info!( - addr = %address, - path = WEBHOOK_PATH, - "webhook listener accepting livekit events, which are suspicion only" - ); - axum::serve(listener, router(shared, clock)).await?; - Ok(()) -} diff --git a/fluxer_svc/src/metrics.rs b/fluxer_svc/src/metrics.rs index 86ac5cdfa..b725b4de4 100644 --- a/fluxer_svc/src/metrics.rs +++ b/fluxer_svc/src/metrics.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; const ORDERING: Ordering = Ordering::Relaxed; -pub type AdditionalMetricsRenderer = Arc; +pub(crate) type AdditionalMetricsRenderer = Arc; const HISTOGRAM_BUCKETS_MS: &[u64] = &[ 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, @@ -92,7 +92,7 @@ impl Default for ServiceMetrics { } impl ServiceMetrics { - pub fn with_additional_renderer(renderer: AdditionalMetricsRenderer) -> Self { + pub(crate) fn with_additional_renderer(renderer: AdditionalMetricsRenderer) -> Self { Self { additional_renderer: Some(renderer), ..Self::default() diff --git a/tools/ci/src/ci_workflow.rs b/tools/ci/src/ci_workflow.rs index de041e58e..a2c8aca7e 100644 --- a/tools/ci/src/ci_workflow.rs +++ b/tools/ci/src/ci_workflow.rs @@ -378,11 +378,6 @@ mod tests { "fluxer-messages", include_str!("../../../fluxer_messages/Dockerfile"), ), - ( - "fluxer_recon", - "fluxer-recon", - include_str!("../../../fluxer_recon/Dockerfile"), - ), ( "fluxer_snowflakes", "fluxer-snowflakes", diff --git a/tools/ci/src/image_set.rs b/tools/ci/src/image_set.rs index d56493151..8b6151a24 100644 --- a/tools/ci/src/image_set.rs +++ b/tools/ci/src/image_set.rs @@ -68,10 +68,6 @@ const COMPONENTS: &[Component] = &[ image: "fluxer-messages", services: &["messages", "messages-shard"], }, - Component { - image: "fluxer-recon", - services: &[], - }, Component { image: "fluxer-snowflakes", services: &["snowflakes", "snowflakes-shard"], @@ -1117,13 +1113,13 @@ mod tests { let bootstrapping = unpublished_moving_tags("ghcr.io/fluxerapp", "fluxer-docs", &tags, |reference| { - !reference.starts_with("ghcr.io/fluxerapp/fluxer-recon:") + !reference.starts_with("ghcr.io/fluxerapp/fluxer-gifs:") }); assert_eq!( bootstrapping, [ - "fluxer-recon has no published v1 image, so v1 stays an incomplete set until the build-recon workflow runs", - "fluxer-recon has no published latest image, so latest stays an incomplete set until the build-recon workflow runs", + "fluxer-gifs has no published v1 image, so v1 stays an incomplete set until the build-gifs workflow runs", + "fluxer-gifs has no published latest image, so latest stays an incomplete set until the build-gifs workflow runs", ] ); } @@ -1132,8 +1128,8 @@ mod tests { fn unpublished_moving_tags_never_reports_the_component_being_promoted() { let tags = ["v1".to_string()]; let warnings = - unpublished_moving_tags("ghcr.io/fluxerapp", "fluxer-recon", &tags, |reference| { - !reference.starts_with("ghcr.io/fluxerapp/fluxer-recon:") + unpublished_moving_tags("ghcr.io/fluxerapp", "fluxer-gifs", &tags, |reference| { + !reference.starts_with("ghcr.io/fluxerapp/fluxer-gifs:") }); assert!(warnings.is_empty()); }