feat(gateway,admin): Expand stats for metrics (#3064)

fluxer_admin/openapi-admin.json

@@ -12059,6 +12059,9 @@
 				"properties": {
 					"status": {"type": "string", "minLength": 1, "maxLength": 256},
 					"sessions": {"$ref": "#/components/schemas/Int32Type"},
+					"session_resumes_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
+					"websocket_dispatches_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
+					"websocket_dispatch_drops_total": {"type": "integer", "minimum": 0, "maximum": 9007199254740991},
 					"guilds": {"$ref": "#/components/schemas/Int32Type"},
 					"presences": {"$ref": "#/components/schemas/Int32Type"},
 					"calls": {"$ref": "#/components/schemas/Int32Type"},
@@ -12085,6 +12088,24 @@
 								"node_id": {"type": "string", "minLength": 1, "maxLength": 256},
 								"status": {"type": "string", "minLength": 1, "maxLength": 256},
 								"sessions": {"$ref": "#/components/schemas/Int32Type"},
+								"session_resumes_total": {
+									"nullable": true,
+									"type": "integer",
+									"minimum": 0,
+									"maximum": 9007199254740991
+								},
+								"websocket_dispatches_total": {
+									"nullable": true,
+									"type": "integer",
+									"minimum": 0,
+									"maximum": 9007199254740991
+								},
+								"websocket_dispatch_drops_total": {
+									"nullable": true,
+									"type": "integer",
+									"minimum": 0,
+									"maximum": 9007199254740991
+								},
 								"guilds": {"$ref": "#/components/schemas/Int32Type"},
 								"presences": {"$ref": "#/components/schemas/Int32Type"},
 								"calls": {"$ref": "#/components/schemas/Int32Type"},
@@ -12138,6 +12159,9 @@
 								"node_id",
 								"status",
 								"sessions",
+								"session_resumes_total",
+								"websocket_dispatches_total",
+								"websocket_dispatch_drops_total",
 								"guilds",
 								"presences",
 								"calls",
@@ -12153,6 +12177,9 @@
 				"required": [
 					"status",
 					"sessions",
+					"session_resumes_total",
+					"websocket_dispatches_total",
+					"websocket_dispatch_drops_total",
 					"guilds",
 					"presences",
 					"calls",

fluxer_admin/src/templates/pages/gateway_nodes.rs

@@ -25,6 +25,10 @@ pub(crate) fn stat_card(label: &str, value: &str) -> Markup {
 
 pub(crate) fn node_stats_section(data: &serde_json::Value, expanded: bool, base: &str) -> Markup {
     let sessions = data.get("sessions").and_then(|v| v.as_u64()).unwrap_or(0);
+    let reconnects: u64 = data
+        .get("session_resumes_total")
+        .and_then(|v| v.as_u64())
+        .unwrap_or(0);
     let guilds = data.get("guilds").and_then(|v| v.as_u64()).unwrap_or(0);
     let presences = data.get("presences").and_then(|v| v.as_u64()).unwrap_or(0);
     let calls = data.get("calls").and_then(|v| v.as_u64()).unwrap_or(0);
@@ -64,6 +68,7 @@ pub(crate) fn node_stats_section(data: &serde_json::Value, expanded: bool, base:
                     div class="grid grid-cols-2 gap-3 sm:gap-4 md:grid-cols-3 lg:grid-cols-6" {
                         (stat_card("Nodes", &node_count.to_string()))
                         (stat_card("Sessions", &sessions.to_string()))
+                        (stat_card("Reconnects", &reconnects.to_string()))
                         (stat_card("Guilds", &guilds.to_string()))
                         (stat_card("Presences", &presences.to_string()))
                         (stat_card("Calls", &calls.to_string()))
@@ -83,6 +88,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
                     tr {
                         th class="px-6 py-3 text-left text-neutral-600 text-xs uppercase" { "Node" }
                         th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Sessions" }
+                        th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Session Resumes" }
                         th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Guilds" }
                         th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Presences" }
                         th class="px-6 py-3 text-right text-neutral-600 text-xs uppercase" { "Calls" }
@@ -95,6 +101,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
                         @let label = format_node_id(node_id, i);
                         @let status = node.get("status").and_then(|v| v.as_str()).unwrap_or("-");
                         @let ns = node.get("sessions").and_then(|v| v.as_u64()).unwrap_or(0);
+                        @let nsr = node.get("session_resumes_total").and_then(|v| v.as_u64()).unwrap_or(0);
                         @let ng = node.get("guilds").and_then(|v| v.as_u64()).unwrap_or(0);
                         @let np = node.get("presences").and_then(|v| v.as_u64()).unwrap_or(0);
                         @let nc = node.get("calls").and_then(|v| v.as_u64()).unwrap_or(0);
@@ -105,6 +112,7 @@ pub(crate) fn node_stats_table(nodes: &[serde_json::Value]) -> Markup {
                                 div class="text-neutral-500 text-xs" { (status) }
                             }
                             td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (ns) }
+                            td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (nsr) }
                             td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (ng) }
                             td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (np) }
                             td class="whitespace-nowrap px-6 py-4 text-right text-sm" { (nc) }

fluxer_api/src/api/infrastructure/GatewayService.ts

@@ -1744,6 +1744,8 @@ export class GatewayService {
 	}
 
 	async getNodeStats(): Promise<GatewayNodeStats> {
+		const a = this.call<GatewayNodeStats>('process.node_stats', {});
+		console.log(a.then((stats) => console.log('Node stats:', stats)));
 		return this.call<GatewayNodeStats>('process.node_stats', {});
 	}
 

fluxer_api/src/api/infrastructure/IGatewayService.ts

@@ -62,6 +62,9 @@ export interface GatewayNodeStatsEntry {
 	node_id: string;
 	status: string;
 	sessions: number;
+	session_resumes_total: number | null;
+	websocket_dispatches_total: number | null;
+	websocket_dispatch_drops_total: number | null;
 	guilds: number;
 	presences: number;
 	calls: number;
@@ -74,6 +77,9 @@ export interface GatewayNodeStatsEntry {
 export interface GatewayNodeStats {
 	status: string;
 	sessions: number;
+	session_resumes_total: number;
+	websocket_dispatches_total: number;
+	websocket_dispatch_drops_total: number;
 	guilds: number;
 	presences: number;
 	calls: number;

fluxer_api/src/api/test/NoopGatewayService.ts

@@ -968,6 +968,9 @@ export class NoopGatewayService extends IGatewayService {
 		return {
 			status: 'ok',
 			sessions: 0,
+			session_resumes_total: 0,
+			websocket_dispatches_total: 0,
+			websocket_dispatch_drops_total: 0,
 			guilds: 0,
 			presences: 0,
 			calls: 0,

fluxer_docs/scripts/VerifyDocsStyle.ts

@@ -269,6 +269,7 @@ const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRul
 	['admin-api/applications.mdx', {'table-identifier': 1}],
 	['admin-api/bulk-jobs.mdx', {'table-fit': 1}],
 	['admin-api/discovery.mdx', {'table-identifier': 1}],
+	['admin-api/gateway.mdx', {'table-identifier': 2}],
 	['admin-api/guilds.mdx', {'table-identifier': 3}],
 	['admin-api/index.mdx', {'table-identifier': 3}],
 	['admin-api/instance.mdx', {'table-identifier': 8}],

fluxer_docs/src/content/docs/admin-api/gateway.mdx

@@ -24,6 +24,9 @@ One snapshot of the Gateway cluster, taken across the nodes the request polled.
 | --- | --- | --- |
 | status<sup>1</sup> | string | The aggregate health of the cluster |
 | sessions | integer | The connected client Gateway sessions, summed across nodes |
+| session_resumes_total | integer | The total number of session resumes, summed across nodes |
+| websocket_dispatches_total | integer | The total number of WebSocket dispatches, summed across nodes |
+| websocket_dispatch_drops_total | integer | The total number of WebSocket dispatches dropped, summed across nodes |
 | guilds | integer | The live guild processes, summed across nodes |
 | presences | integer | The tracked presences, summed across nodes |
 | calls | integer | The live calls, summed across nodes |
@@ -48,6 +51,9 @@ One snapshot of the Gateway cluster, taken across the nodes the request polled.
 {
   "status": "healthy",
   "sessions": 4820,
+  "session_resumes_total": 1234,
+  "websocket_dispatches_total": 1234567,
+  "websocket_dispatch_drops_total": 123,
   "guilds": 1913,
   "presences": 4611,
   "calls": 12,
@@ -71,6 +77,9 @@ One entry for each node the request polled, including nodes that did not answer
 | node_id<sup>1</sup> | string | The name the node reports for itself |
 | status<sup>2</sup> | string | The health this node reported |
 | sessions | integer | The connected client Gateway sessions on this node |
+| session_resumes_total | integer | The total number of session resumes on this node |
+| websocket_dispatches_total | integer | The total number of WebSocket dispatches on this node |
+| websocket_dispatch_drops_total | integer | The total number of WebSocket dispatches dropped on this node |
 | guilds | integer | The live guild processes on this node |
 | presences | integer | The tracked presences on this node |
 | calls | integer | The live calls on this node |

fluxer_gateway/src/gateway/gateway_cluster_metrics.erl

@@ -8,6 +8,12 @@
     record_discovery_resolve_failure/0,
     record_membership_transition/1,
     record_owner_resolution/1,
+    record_resume/0,
+    resumes_total/0,
+    record_dispatch/0,
+    dispatches_total/0,
+    record_dispatch_drop/0,
+    dispatch_drops_total/0,
     snapshot/0,
     reset_for_tests/0
 ]).
@@ -18,7 +24,10 @@
 -define(MEMBERSHIP_DOWN_IDX, 3).
 -define(OWNER_SELF_IDX, 4).
 -define(OWNER_PEER_IDX, 5).
--define(COUNTER_COUNT, 5).
+-define(RESUMES_IDX, 6).
+-define(DISPATCHES_IDX, 7).
+-define(DISPATCH_DROPS_IDX, 8).
+-define(COUNTER_COUNT, 8).
 
 -spec init() -> ok.
 init() ->
@@ -41,11 +50,38 @@ record_owner_resolution(self) ->
 record_owner_resolution(peer) ->
     add(?OWNER_PEER_IDX, 1).
 
+-spec record_resume() -> ok.
+record_resume() ->
+    add(?RESUMES_IDX, 1).
+
+-spec resumes_total() -> non_neg_integer().
+resumes_total() ->
+    counters:get(counters_ref(), ?RESUMES_IDX).
+
+-spec record_dispatch() -> ok.
+record_dispatch() ->
+    add(?DISPATCHES_IDX, 1).
+
+-spec dispatches_total() -> non_neg_integer().
+dispatches_total() ->
+    counters:get(counters_ref(), ?DISPATCHES_IDX).
+
+-spec record_dispatch_drop() -> ok.
+record_dispatch_drop() ->
+    add(?DISPATCH_DROPS_IDX, 1).
+
+-spec dispatch_drops_total() -> non_neg_integer().
+dispatch_drops_total() ->
+    counters:get(counters_ref(), ?DISPATCH_DROPS_IDX).
+
 -spec snapshot() -> map().
 snapshot() ->
     Counters = counters_ref(),
     #{
         <<"gateway_cluster_member_count">> => gateway_cluster_membership:alive_count(),
+        <<"session_resumes_total">> => counters:get(Counters, ?RESUMES_IDX),
+        <<"websocket_dispatches_total">> => counters:get(Counters, ?DISPATCHES_IDX),
+        <<"websocket_dispatch_drops_total">> => counters:get(Counters, ?DISPATCH_DROPS_IDX),
         <<"gateway_cluster_discovery_resolve_failures_total">> =>
             counters:get(Counters, ?DISCOVERY_RESOLVE_FAILURES_IDX),
         <<"gateway_cluster_membership_transitions_total">> => #{
@@ -83,7 +119,7 @@ valid_counters_or_new(Counters) ->
 validate_counters(Counters) ->
     try
         Ref = eqwalizer:dynamic_cast(Counters),
-        _ = counters:get(Ref, ?DISCOVERY_RESOLVE_FAILURES_IDX),
+        _ = counters:get(Ref, ?COUNTER_COUNT),
         {ok, Ref}
     catch
         _:_ -> error
@@ -108,6 +144,7 @@ snapshot_defaults_to_zero_test() ->
     persistent_term:erase({gateway_cluster_membership, members}),
     Snapshot = snapshot(),
     ?assertEqual(1, maps:get(<<"gateway_cluster_member_count">>, Snapshot)),
+    ?assertEqual(0, maps:get(<<"session_resumes_total">>, Snapshot)),
     ?assertEqual(0, maps:get(<<"gateway_cluster_discovery_resolve_failures_total">>, Snapshot)),
     Membership = maps:get(<<"gateway_cluster_membership_transitions_total">>, Snapshot),
     ?assertEqual(0, maps:get(<<"up">>, Membership)),
@@ -117,16 +154,41 @@ records_counters_test() ->
     reset_for_tests(),
     record_discovery_resolve_failure(),
     record_membership_transition(up),
+    record_resume(),
     record_membership_transition(down),
     record_owner_resolution(self),
     record_owner_resolution(peer),
     Snapshot = snapshot(),
     ?assertEqual(1, maps:get(<<"gateway_cluster_discovery_resolve_failures_total">>, Snapshot)),
     Membership = maps:get(<<"gateway_cluster_membership_transitions_total">>, Snapshot),
+    ?assertEqual(1, maps:get(<<"session_resumes_total">>, Snapshot)),
     Owners = maps:get(<<"gateway_node_router_owner_resolutions_total">>, Snapshot),
     ?assertEqual(1, maps:get(<<"up">>, Membership)),
     ?assertEqual(1, maps:get(<<"down">>, Membership)),
     ?assertEqual(1, maps:get(<<"self">>, Owners)),
     ?assertEqual(1, maps:get(<<"peer">>, Owners)).
 
+dispatch_counter_test() ->
+    reset_for_tests(),
+    ?assertEqual(0, dispatches_total()),
+    record_dispatch(),
+    record_dispatch(),
+    ?assertEqual(2, dispatches_total()),
+    ?assertEqual(2, maps:get(<<"websocket_dispatches_total">>, snapshot())).
+
+dispatch_drop_counter_test() ->
+    reset_for_tests(),
+    ?assertEqual(0, dispatch_drops_total()),
+    record_dispatch_drop(),
+    record_dispatch_drop(),
+    ?assertEqual(2, dispatch_drops_total()),
+    ?assertEqual(0, dispatches_total()),
+    ?assertEqual(2, maps:get(<<"websocket_dispatch_drops_total">>, snapshot())).
+
+undersized_counters_test() ->
+    persistent_term:put(?COUNTERS_KEY, counters:new(5, [write_concurrency])),
+    record_dispatch(),
+    ?assertEqual(1, dispatches_total()),
+    reset_for_tests().
+
 -endif.

fluxer_gateway/src/gateway/gateway_handler_dispatch.erl

@@ -138,8 +138,10 @@ dispatch_pre_encoded(Event, EncodedData, Seq, #{compress_ctx := CompressCtx} = S
     case gateway_compress:compress(JsonFrame, Ctx) of
         {ok, Compressed, NewCompressCtx} ->
             Frame = gateway_handler_encode:make_frame(Compressed, text, NewCompressCtx),
+            gateway_cluster_metrics:record_dispatch(),
             {[Frame], State#{compress_ctx => NewCompressCtx}};
         {error, _Reason} ->
+            gateway_cluster_metrics:record_dispatch_drop(),
             {ok, State}
     end.
 
@@ -156,9 +158,11 @@ dispatch_standard(Event, Data, Seq, State) ->
     },
     case gateway_handler_encode:encode_and_compress(Message, State) of
         {ok, Frame, NewState} ->
+            gateway_cluster_metrics:record_dispatch(),
             trigger_gc_after_large_dispatch(Data),
             {[Frame], NewState};
         {error, _Reason} ->
+            gateway_cluster_metrics:record_dispatch_drop(),
             {ok, State}
     end.
 

fluxer_gateway/src/gateway/gateway_handler_resume.erl

@@ -158,6 +158,7 @@ finalize_resume(Pid, Seq, MissedEvents, GwTimings0, State) ->
     ),
     ResumedData = put_resumed_gateway_timings(session_is_staff(Pid), GwTimings),
     SocketPid ! {dispatch, resumed, ResumedData, Seq},
+    gateway_cluster_metrics:record_resume(),
     erlang:garbage_collect(self(), [{type, major}]),
     {ok, State#{
         session_pid => Pid,

fluxer_gateway/src/gateway/gateway_rpc_misc_session.erl

@@ -30,6 +30,7 @@ get_local_node_id() ->
 -spec get_local_node_stats() -> map().
 get_local_node_stats() ->
     SessionCount = get_manager_count(session_manager),
+    ResumeCount = gateway_cluster_metrics:resumes_total(),
     GuildCount = get_manager_count(guild_manager),
     PresenceCount = get_manager_count(presence_manager),
     CallCount = get_manager_count(call_manager),
@@ -41,6 +42,9 @@ get_local_node_stats() ->
         <<"node_id">> => node_id(node()),
         <<"status">> => <<"healthy">>,
         <<"sessions">> => SessionCount,
+        <<"session_resumes_total">> => ResumeCount,
+        <<"websocket_dispatches_total">> => gateway_cluster_metrics:dispatches_total(),
+        <<"websocket_dispatch_drops_total">> => gateway_cluster_metrics:dispatch_drops_total(),
         <<"guilds">> => GuildCount,
         <<"presences">> => PresenceCount,
         <<"calls">> => CallCount,
@@ -180,6 +184,13 @@ aggregate_node_stats(NodeStats) ->
     #{
         <<"status">> => aggregate_status(SortedNodes),
         <<"sessions">> => sum_stat(SortedNodes, <<"sessions">>),
+        <<"session_resumes_total">> => sum_stat(SortedNodes, <<"session_resumes_total">>),
+        <<"websocket_dispatches_total">> => sum_stat(
+            SortedNodes, <<"websocket_dispatches_total">>
+        ),
+        <<"websocket_dispatch_drops_total">> => sum_stat(
+            SortedNodes, <<"websocket_dispatch_drops_total">>
+        ),
         <<"guilds">> => sum_stat(SortedNodes, <<"guilds">>),
         <<"presences">> => sum_stat(SortedNodes, <<"presences">>),
         <<"calls">> => sum_stat(SortedNodes, <<"calls">>),
@@ -266,6 +277,9 @@ unavailable_node_stats(TargetNode) ->
         <<"node_id">> => node_id(TargetNode),
         <<"status">> => <<"unavailable">>,
         <<"sessions">> => null,
+        <<"session_resumes_total">> => null,
+        <<"websocket_dispatches_total">> => null,
+        <<"websocket_dispatch_drops_total">> => null,
         <<"guilds">> => null,
         <<"presences">> => null,
         <<"calls">> => null,
@@ -315,6 +329,9 @@ aggregate_node_stats_sums_cluster_totals_test() ->
         <<"node_id">> => <<"[email protected]">>,
         <<"status">> => <<"healthy">>,
         <<"sessions">> => 10,
+        <<"session_resumes_total">> => 1,
+        <<"websocket_dispatches_total">> => 7,
+        <<"websocket_dispatch_drops_total">> => 7,
         <<"guilds">> => 20,
         <<"presences">> => 30,
         <<"calls">> => 40,
@@ -327,6 +344,9 @@ aggregate_node_stats_sums_cluster_totals_test() ->
         <<"node_id">> => <<"[email protected]">>,
         <<"status">> => <<"healthy">>,
         <<"sessions">> => 3,
+        <<"session_resumes_total">> => 1,
+        <<"websocket_dispatches_total">> => 7,
+        <<"websocket_dispatch_drops_total">> => 7,
         <<"guilds">> => 4,
         <<"presences">> => 5,
         <<"calls">> => 6,
@@ -336,7 +356,10 @@ aggregate_node_stats_sums_cluster_totals_test() ->
         <<"uptime_seconds">> => 90
     },
     A = aggregate_node_stats([N1, N2]),
+    ?assertEqual(14, maps:get(<<"websocket_dispatches_total">>, A)),
+    ?assertEqual(14, maps:get(<<"websocket_dispatch_drops_total">>, A)),
     ?assertEqual(13, maps:get(<<"sessions">>, A)),
+    ?assertEqual(2, maps:get(<<"session_resumes_total">>, A)),
     ?assertEqual(24, maps:get(<<"guilds">>, A)),
     ?assertEqual(35, maps:get(<<"presences">>, A)),
     ?assertEqual(46, maps:get(<<"calls">>, A)),

fluxer_gateway/src/gateway/metrics_handler.erl

@@ -51,6 +51,9 @@ render_metrics() ->
 -spec render_gateway_gauges() -> iolist().
 render_gateway_gauges() ->
     Sessions = safe_apply_int(fun session_manager:session_count/0),
+    DispatchDrops = safe_apply_int(fun gateway_cluster_metrics:dispatch_drops_total/0),
+    Dispatches = safe_apply_int(fun gateway_cluster_metrics:dispatches_total/0),
+    Resumes = safe_apply_int(fun gateway_cluster_metrics:resumes_total/0),
     Guilds = safe_apply_int(fun guild_manager:local_guild_count/0),
     VoiceCounts = safe_apply_map(fun voice_state_counts_cache:get_local_counts/0),
     CallIds = safe_apply_list(fun call_manager:local_call_ids/0),
@@ -64,12 +67,30 @@ render_gateway_gauges() ->
             <<"Connected WebSocket sessions">>,
             integer_to_binary(Sessions)
         ),
+        format_metric(
+            <<"fluxer_gateway_session_resumes_total">>,
+            <<"counter">>,
+            <<"Successful WebSocket session resumes">>,
+            integer_to_binary(Resumes)
+        ),
         format_metric(
             <<"fluxer_gateway_guilds_total">>,
             <<"gauge">>,
             <<"Locally loaded guilds">>,
             integer_to_binary(Guilds)
         ),
+        format_metric(
+            <<"fluxer_gateway_websocket_dispatches_total">>,
+            <<"counter">>,
+            <<"WebSocket dispatch frames handed to the transport, including replays">>,
+            integer_to_binary(Dispatches)
+        ),
+        format_metric(
+            <<"fluxer_gateway_websocket_dispatch_drops_total">>,
+            <<"counter">>,
+            <<"WebSocket dispatches dropped due to encoding or compression errors">>,
+            integer_to_binary(DispatchDrops)
+        ),
         render_voice_metrics(VoiceCounts),
         format_metric(
             <<"fluxer_gateway_calls_total">>,

packages/schema/src/domains/admin/AdminSchemas.ts

@@ -1133,6 +1133,9 @@ export type ReloadAllGuildsResponse = z.infer<typeof ReloadAllGuildsResponse>;
 export const NodeStatsResponse = z.object({
 	status: createStringType(1, 256),
 	sessions: Int32Type,
+	session_resumes_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
+	websocket_dispatches_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
+	websocket_dispatch_drops_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
 	guilds: Int32Type,
 	presences: Int32Type,
 	calls: Int32Type,
@@ -1151,6 +1154,9 @@ export const NodeStatsResponse = z.object({
 				node_id: createStringType(1, 256),
 				status: createStringType(1, 256),
 				sessions: Int32Type,
+				session_resumes_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
+				websocket_dispatches_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
+				websocket_dispatch_drops_total: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
 				guilds: Int32Type,
 				presences: Int32Type,
 				calls: Int32Type,