pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-08 03:32:27 +09:00
Code Issues Packages Projects Releases Wiki Activity

feat(gateway,admin): Expand stats for metrics (#3064)

Browse Source
This commit is contained in:
Tarek
2026-10-02 12:58:16 +02:00
committed by GitHub
parent 840dc3dfa5
commit 632067b552
13 changed files with 175 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_admin/openapi-admin.json
+27
View File
@@ -12059,6 +12059,9 @@
"properties": {
"status": {"type": "string", "minLength": 1, "maxLength": 256},
"sessions": {"$ref": "#/components/schemas/Int32Type"},
"session_resumes_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
"websocket_dispatches_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
"websocket_dispatch_drops_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
"guilds": {"$ref": "#/components/schemas/Int32Type"},
"presences": {"$ref": "#/components/schemas/Int32Type"},
"calls": {"$ref": "#/components/schemas/Int32Type"},
@@ -12085,6 +12088,24 @@
"node_id": {"type": "string", "minLength": 1, "maxLength": 256},
"status": {"type": "string", "minLength": 1, "maxLength": 256},
"sessions": {"$ref": "#/components/schemas/Int32Type"},
"session_resumes_total": {
"nullable": true,
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"websocket_dispatches_total": {
"nullable": true,
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"websocket_dispatch_drops_total": {
"nullable": true,
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"guilds": {"$ref": "#/components/schemas/Int32Type"},
"presences": {"$ref": "#/components/schemas/Int32Type"},
"calls": {"$ref": "#/components/schemas/Int32Type"},
@@ -12138,6 +12159,9 @@
"node_id",
"status",
"sessions",
"session_resumes_total",
"websocket_dispatches_total",
"websocket_dispatch_drops_total",
"guilds",
"presences",
"calls",
@@ -12153,6 +12177,9 @@
"required": [
"status",
"sessions",
"session_resumes_total",
"websocket_dispatches_total",
"websocket_dispatch_drops_total",
"guilds",
"presences",
"calls",
fluxer_admin/src/templates/pages/gateway_nodes.rs
+8
View File
@@ -25,6 +25,10 @@ pub(crate) fn stat_card(label: &str, value: &str) -> Markup {
pub(crate) fn node_stats_section(data: &serde_json::Value, expanded: bool, base: &str) -> Markup {
let sessions = data.get("sessions").and_then(|v| v.as_u64()).unwrap_or(0);
let reconnects: u64 = data
.get("session_resumes_total")
.and_then(|v| v.as_u64())
.unwrap_or(0);
let guilds = data.get("guilds").and_then(|v| v.as_u64()).unwrap_or(0);
let presences = data.get("presences").and_then(|v| v.as_u64()).unwrap_or(0);
let calls = data.get("calls").and_then(|v| v.as_u64()).unwrap_or(0);
@@ -64,6 +68,7 @@ pub(crate) fn node_stats_section(data: &serde_json::Value, expanded: bool, base:
div class="grid grid-cols-2 gap-3 sm:gap-4 md:grid-cols-3 lg:grid-cols-6" {
(stat_card("Nodes", &node_count.to_string()))
(stat_card("Sessions", &sessions.to_string()))
(stat_card("Reconnects", &reconnects.to_string()))
(stat_card("Guilds", &guilds.to_string()))
(stat_card("Presences", &presences.to_string()))
(stat_card("Calls", &calls.to_string()))
@@ -83,6 +88,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
tr {
th class="px-6 py-3 text-left text-neutral-600 text-xs uppercase" { "Node" }
th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Sessions" }
th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Session Resumes" }
th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Guilds" }
th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Presences" }
th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Calls" }
@@ -95,6 +101,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
@let label = format_node_id(node_id, i);
@let status = node.get("status").and_then(|v| v.as_str()).unwrap_or("-");
@let ns = node.get("sessions").and_then(|v| v.as_u64()).unwrap_or(0);
@let nsr = node.get("session_resumes_total").and_then(|v| v.as_u64()).unwrap_or(0);
@let ng = node.get("guilds").and_then(|v| v.as_u64()).unwrap_or(0);
@let np = node.get("presences").and_then(|v| v.as_u64()).unwrap_or(0);
@let nc = node.get("calls").and_then(|v| v.as_u64()).unwrap_or(0);
@@ -105,6 +112,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
div class="text-neutral-500 text-xs" { (status) }
}
td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (ns) }
td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (nsr) }
td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (ng) }
td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (np) }
td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (nc) }
fluxer_api/src/api/infrastructure/GatewayService.ts
+2
View File
@@ -1744,6 +1744,8 @@ export class GatewayService {
}
async getNodeStats(): Promise<GatewayNodeStats> {
const a = this.call<GatewayNodeStats>('process.node_stats', {});
console.log(a.then((stats) => console.log('Node stats:', stats)));
return this.call<GatewayNodeStats>('process.node_stats', {});
}
fluxer_api/src/api/infrastructure/IGatewayService.ts
+6
View File
@@ -62,6 +62,9 @@ export interface GatewayNodeStatsEntry {
node_id: string;
status: string;
sessions: number;
session_resumes_total: number | null;
websocket_dispatches_total: number | null;
websocket_dispatch_drops_total: number | null;
guilds: number;
presences: number;
calls: number;
@@ -74,6 +77,9 @@ export interface GatewayNodeStatsEntry {
export interface GatewayNodeStats {
status: string;
sessions: number;
session_resumes_total: number;
websocket_dispatches_total: number;
websocket_dispatch_drops_total: number;
guilds: number;
presences: number;
calls: number;
fluxer_api/src/api/test/NoopGatewayService.ts
+3
View File
@@ -968,6 +968,9 @@ export class NoopGatewayService extends IGatewayService {
return {
status: 'ok',
sessions: 0,
session_resumes_total: 0,
websocket_dispatches_total: 0,
websocket_dispatch_drops_total: 0,
guilds: 0,
presences: 0,
calls: 0,
fluxer_docs/scripts/VerifyDocsStyle.ts
+1
View File
@@ -269,6 +269,7 @@ const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRul
['admin-api/applications.mdx', {'table-identifier': 1}],
['admin-api/bulk-jobs.mdx', {'table-fit': 1}],
['admin-api/discovery.mdx', {'table-identifier': 1}],
['admin-api/gateway.mdx', {'table-identifier': 2}],
['admin-api/guilds.mdx', {'table-identifier': 3}],
['admin-api/index.mdx', {'table-identifier': 3}],
['admin-api/instance.mdx', {'table-identifier': 8}],
fluxer_docs/src/content/docs/admin-api/gateway.mdx
+9
View File
@@ -24,6 +24,9 @@ One snapshot of the Gateway cluster, taken across the nodes the request polled.
| --- | --- | --- |
| status<sup>1</sup> | string | The aggregate health of the cluster |
| sessions | integer | The connected client Gateway sessions, summed across nodes |
| session_resumes_total | integer | The total number of session resumes, summed across nodes |
| websocket_dispatches_total | integer | The total number of WebSocket dispatches, summed across nodes |
| websocket_dispatch_drops_total | integer | The total number of WebSocket dispatches dropped, summed across nodes |
| guilds | integer | The live guild processes, summed across nodes |
| presences | integer | The tracked presences, summed across nodes |
| calls | integer | The live calls, summed across nodes |
@@ -48,6 +51,9 @@ One snapshot of the Gateway cluster, taken across the nodes the request polled.
{
"status": "healthy",
"sessions": 4820,
"session_resumes_total": 1234,
"websocket_dispatches_total": 1234567,
"websocket_dispatch_drops_total": 123,
"guilds": 1913,
"presences": 4611,
"calls": 12,
@@ -71,6 +77,9 @@ One entry for each node the request polled, including nodes that did not answer
| node_id<sup>1</sup> | string | The name the node reports for itself |
| status<sup>2</sup> | string | The health this node reported |
| sessions | integer | The connected client Gateway sessions on this node |
| session_resumes_total | integer | The total number of session resumes on this node |
| websocket_dispatches_total | integer | The total number of WebSocket dispatches on this node |
| websocket_dispatch_drops_total | integer | The total number of WebSocket dispatches dropped on this node |
| guilds | integer | The live guild processes on this node |
| presences | integer | The tracked presences on this node |
| calls | integer | The live calls on this node |
fluxer_gateway/src/gateway/gateway_cluster_metrics.erl
+64 -2
View File
@@ -8,6 +8,12 @@
record_discovery_resolve_failure/0,
record_membership_transition/1,
record_owner_resolution/1,
record_resume/0,
resumes_total/0,
record_dispatch/0,
dispatches_total/0,
record_dispatch_drop/0,
dispatch_drops_total/0,
snapshot/0,
reset_for_tests/0
]).
@@ -18,7 +24,10 @@
-define(MEMBERSHIP_DOWN_IDX, 3).
-define(OWNER_SELF_IDX, 4).
-define(OWNER_PEER_IDX, 5).
-define(COUNTER_COUNT, 5).
-define(RESUMES_IDX, 6).
-define(DISPATCHES_IDX, 7).
-define(DISPATCH_DROPS_IDX, 8).
-define(COUNTER_COUNT, 8).
-spec init() -> ok.
init() ->
@@ -41,11 +50,38 @@ record_owner_resolution(self) ->
record_owner_resolution(peer) ->
add(?OWNER_PEER_IDX, 1).
-spec record_resume() -> ok.
record_resume() ->
add(?RESUMES_IDX, 1).
-spec resumes_total() -> non_neg_integer().
resumes_total() ->
counters:get(counters_ref(), ?RESUMES_IDX).
-spec record_dispatch() -> ok.
record_dispatch() ->
add(?DISPATCHES_IDX, 1).
-spec dispatches_total() -> non_neg_integer().
dispatches_total() ->
counters:get(counters_ref(), ?DISPATCHES_IDX).
-spec record_dispatch_drop() -> ok.
record_dispatch_drop() ->
add(?DISPATCH_DROPS_IDX, 1).
-spec dispatch_drops_total() -> non_neg_integer().
dispatch_drops_total() ->
counters:get(counters_ref(), ?DISPATCH_DROPS_IDX).
-spec snapshot() -> map().
snapshot() ->
Counters = counters_ref(),
#{
<<"gateway_cluster_member_count">> => gateway_cluster_membership:alive_count(),
<<"session_resumes_total">> => counters:get(Counters, ?RESUMES_IDX),
<<"websocket_dispatches_total">> => counters:get(Counters, ?DISPATCHES_IDX),
<<"websocket_dispatch_drops_total">> => counters:get(Counters, ?DISPATCH_DROPS_IDX),
<<"gateway_cluster_discovery_resolve_failures_total">> =>
counters:get(Counters, ?DISCOVERY_RESOLVE_FAILURES_IDX),
<<"gateway_cluster_membership_transitions_total">> => #{
@@ -83,7 +119,7 @@ valid_counters_or_new(Counters) ->
validate_counters(Counters) ->
try
Ref = eqwalizer:dynamic_cast(Counters),
_ = counters:get(Ref, ?DISCOVERY_RESOLVE_FAILURES_IDX),
_ = counters:get(Ref, ?COUNTER_COUNT),
{ok, Ref}
catch
_:_ -> error
@@ -108,6 +144,7 @@ snapshot_defaults_to_zero_test() ->
persistent_term:erase({gateway_cluster_membership, members}),
Snapshot = snapshot(),
?assertEqual(1, maps:get(<<"gateway_cluster_member_count">>, Snapshot)),
?assertEqual(0, maps:get(<<"session_resumes_total">>, Snapshot)),
?assertEqual(0, maps:get(<<"gateway_cluster_discovery_resolve_failures_total">>, Snapshot)),
Membership = maps:get(<<"gateway_cluster_membership_transitions_total">>, Snapshot),
?assertEqual(0, maps:get(<<"up">>, Membership)),
@@ -117,16 +154,41 @@ records_counters_test() ->
reset_for_tests(),
record_discovery_resolve_failure(),
record_membership_transition(up),
record_resume(),
record_membership_transition(down),
record_owner_resolution(self),
record_owner_resolution(peer),
Snapshot = snapshot(),
?assertEqual(1, maps:get(<<"gateway_cluster_discovery_resolve_failures_total">>, Snapshot)),
Membership = maps:get(<<"gateway_cluster_membership_transitions_total">>, Snapshot),
?assertEqual(1, maps:get(<<"session_resumes_total">>, Snapshot)),
Owners = maps:get(<<"gateway_node_router_owner_resolutions_total">>, Snapshot),
?assertEqual(1, maps:get(<<"up">>, Membership)),
?assertEqual(1, maps:get(<<"down">>, Membership)),
?assertEqual(1, maps:get(<<"self">>, Owners)),
?assertEqual(1, maps:get(<<"peer">>, Owners)).
dispatch_counter_test() ->
reset_for_tests(),
?assertEqual(0, dispatches_total()),
record_dispatch(),
record_dispatch(),
?assertEqual(2, dispatches_total()),
?assertEqual(2, maps:get(<<"websocket_dispatches_total">>, snapshot())).
dispatch_drop_counter_test() ->
reset_for_tests(),
?assertEqual(0, dispatch_drops_total()),
record_dispatch_drop(),
record_dispatch_drop(),
?assertEqual(2, dispatch_drops_total()),
?assertEqual(0, dispatches_total()),
?assertEqual(2, maps:get(<<"websocket_dispatch_drops_total">>, snapshot())).
undersized_counters_test() ->
persistent_term:put(?COUNTERS_KEY, counters:new(5, [write_concurrency])),
record_dispatch(),
?assertEqual(1, dispatches_total()),
reset_for_tests().
-endif.
fluxer_gateway/src/gateway/gateway_handler_dispatch.erl
+4
View File
@@ -138,8 +138,10 @@ dispatch_pre_encoded(Event, EncodedData, Seq, #{compress_ctx := CompressCtx} = S
case gateway_compress:compress(JsonFrame, Ctx) of
{ok, Compressed, NewCompressCtx} ->
Frame = gateway_handler_encode:make_frame(Compressed, text, NewCompressCtx),
gateway_cluster_metrics:record_dispatch(),
{[Frame], State#{compress_ctx => NewCompressCtx}};
{error, _Reason} ->
gateway_cluster_metrics:record_dispatch_drop(),
{ok, State}
end.
@@ -156,9 +158,11 @@ dispatch_standard(Event, Data, Seq, State) ->
},
case gateway_handler_encode:encode_and_compress(Message, State) of
{ok, Frame, NewState} ->
gateway_cluster_metrics:record_dispatch(),
trigger_gc_after_large_dispatch(Data),
{[Frame], NewState};
{error, _Reason} ->
gateway_cluster_metrics:record_dispatch_drop(),
{ok, State}
end.
fluxer_gateway/src/gateway/gateway_handler_resume.erl
+1
View File
@@ -158,6 +158,7 @@ finalize_resume(Pid, Seq, MissedEvents, GwTimings0, State) ->
),
ResumedData = put_resumed_gateway_timings(session_is_staff(Pid), GwTimings),
SocketPid ! {dispatch, resumed, ResumedData, Seq},
gateway_cluster_metrics:record_resume(),
erlang:garbage_collect(self(), [{type, major}]),
{ok, State#{
session_pid => Pid,
fluxer_gateway/src/gateway/gateway_rpc_misc_session.erl
+23
View File
@@ -30,6 +30,7 @@ get_local_node_id() ->
-spec get_local_node_stats() -> map().
get_local_node_stats() ->
SessionCount = get_manager_count(session_manager),
ResumeCount = gateway_cluster_metrics:resumes_total(),
GuildCount = get_manager_count(guild_manager),
PresenceCount = get_manager_count(presence_manager),
CallCount = get_manager_count(call_manager),
@@ -41,6 +42,9 @@ get_local_node_stats() ->
<<"node_id">> => node_id(node()),
<<"status">> => <<"healthy">>,
<<"sessions">> => SessionCount,
<<"session_resumes_total">> => ResumeCount,
<<"websocket_dispatches_total">> => gateway_cluster_metrics:dispatches_total(),
<<"websocket_dispatch_drops_total">> => gateway_cluster_metrics:dispatch_drops_total(),
<<"guilds">> => GuildCount,
<<"presences">> => PresenceCount,
<<"calls">> => CallCount,
@@ -180,6 +184,13 @@ aggregate_node_stats(NodeStats) ->
#{
<<"status">> => aggregate_status(SortedNodes),
<<"sessions">> => sum_stat(SortedNodes, <<"sessions">>),
<<"session_resumes_total">> => sum_stat(SortedNodes, <<"session_resumes_total">>),
<<"websocket_dispatches_total">> => sum_stat(
SortedNodes, <<"websocket_dispatches_total">>
),
<<"websocket_dispatch_drops_total">> => sum_stat(
SortedNodes, <<"websocket_dispatch_drops_total">>
),
<<"guilds">> => sum_stat(SortedNodes, <<"guilds">>),
<<"presences">> => sum_stat(SortedNodes, <<"presences">>),
<<"calls">> => sum_stat(SortedNodes, <<"calls">>),
@@ -266,6 +277,9 @@ unavailable_node_stats(TargetNode) ->
<<"node_id">> => node_id(TargetNode),
<<"status">> => <<"unavailable">>,
<<"sessions">> => null,
<<"session_resumes_total">> => null,
<<"websocket_dispatches_total">> => null,
<<"websocket_dispatch_drops_total">> => null,
<<"guilds">> => null,
<<"presences">> => null,
<<"calls">> => null,
@@ -315,6 +329,9 @@ aggregate_node_stats_sums_cluster_totals_test() ->
<<"node_id">> => <<"[email protected]">>,
<<"status">> => <<"healthy">>,
<<"sessions">> => 10,
<<"session_resumes_total">> => 1,
<<"websocket_dispatches_total">> => 7,
<<"websocket_dispatch_drops_total">> => 7,
<<"guilds">> => 20,
<<"presences">> => 30,
<<"calls">> => 40,
@@ -327,6 +344,9 @@ aggregate_node_stats_sums_cluster_totals_test() ->
<<"node_id">> => <<"[email protected]">>,
<<"status">> => <<"healthy">>,
<<"sessions">> => 3,
<<"session_resumes_total">> => 1,
<<"websocket_dispatches_total">> => 7,
<<"websocket_dispatch_drops_total">> => 7,
<<"guilds">> => 4,
<<"presences">> => 5,
<<"calls">> => 6,
@@ -336,7 +356,10 @@ aggregate_node_stats_sums_cluster_totals_test() ->
<<"uptime_seconds">> => 90
},
A = aggregate_node_stats([N1, N2]),
?assertEqual(14, maps:get(<<"websocket_dispatches_total">>, A)),
?assertEqual(14, maps:get(<<"websocket_dispatch_drops_total">>, A)),
?assertEqual(13, maps:get(<<"sessions">>, A)),
?assertEqual(2, maps:get(<<"session_resumes_total">>, A)),
?assertEqual(24, maps:get(<<"guilds">>, A)),
?assertEqual(35, maps:get(<<"presences">>, A)),
?assertEqual(46, maps:get(<<"calls">>, A)),
fluxer_gateway/src/gateway/metrics_handler.erl
+21
View File
@@ -51,6 +51,9 @@ render_metrics() ->
-spec render_gateway_gauges() -> iolist().
render_gateway_gauges() ->
Sessions = safe_apply_int(fun session_manager:session_count/0),
DispatchDrops = safe_apply_int(fun gateway_cluster_metrics:dispatch_drops_total/0),
Dispatches = safe_apply_int(fun gateway_cluster_metrics:dispatches_total/0),
Resumes = safe_apply_int(fun gateway_cluster_metrics:resumes_total/0),
Guilds = safe_apply_int(fun guild_manager:local_guild_count/0),
VoiceCounts = safe_apply_map(fun voice_state_counts_cache:get_local_counts/0),
CallIds = safe_apply_list(fun call_manager:local_call_ids/0),
@@ -64,12 +67,30 @@ render_gateway_gauges() ->
<<"Connected WebSocket sessions">>,
integer_to_binary(Sessions)
),
format_metric(
<<"fluxer_gateway_session_resumes_total">>,
<<"counter">>,
<<"Successful WebSocket session resumes">>,
integer_to_binary(Resumes)
),
format_metric(
<<"fluxer_gateway_guilds_total">>,
<<"gauge">>,
<<"Locally loaded guilds">>,
integer_to_binary(Guilds)
),
format_metric(
<<"fluxer_gateway_websocket_dispatches_total">>,
<<"counter">>,
<<"WebSocket dispatch frames handed to the transport, including replays">>,
integer_to_binary(Dispatches)
),
format_metric(
<<"fluxer_gateway_websocket_dispatch_drops_total">>,
<<"counter">>,
<<"WebSocket dispatches dropped due to encoding or compression errors">>,
integer_to_binary(DispatchDrops)
),
render_voice_metrics(VoiceCounts),
format_metric(
<<"fluxer_gateway_calls_total">>,
packages/schema/src/domains/admin/AdminSchemas.ts
+6
View File
@@ -1133,6 +1133,9 @@ export type ReloadAllGuildsResponse = z.infer<typeof ReloadAllGuildsResponse>;
export const NodeStatsResponse = z.object({
status: createStringType(1, 256),
sessions: Int32Type,
session_resumes_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
websocket_dispatches_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
websocket_dispatch_drops_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
guilds: Int32Type,
presences: Int32Type,
calls: Int32Type,
@@ -1151,6 +1154,9 @@ export const NodeStatsResponse = z.object({
node_id: createStringType(1, 256),
status: createStringType(1, 256),
sessions: Int32Type,
session_resumes_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
websocket_dispatches_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
websocket_dispatch_drops_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
guilds: Int32Type,
presences: Int32Type,
calls: Int32Type,