diff --git a/fluxer_gateway/src/gateway/gateway_http_client_response.erl b/fluxer_gateway/src/gateway/gateway_http_client_response.erl index 6b8ec7591..dc69b48e4 100644 --- a/fluxer_gateway/src/gateway/gateway_http_client_response.erl +++ b/fluxer_gateway/src/gateway/gateway_http_client_response.erl @@ -38,7 +38,7 @@ allow_circuit_request(CircuitKey, RecoveryTimeoutMs) -> -spec update_circuit_state_direct({atom(), binary()}, response(), pos_integer()) -> ok. update_circuit_state_direct(CircuitKey, Result, FailureThreshold) -> Now = erlang:system_time(millisecond), - IsFailure = is_countable_failure(Result), + IsFailure = is_countable_circuit_failure(CircuitKey, Result), record_result(CircuitKey, IsFailure, Now, FailureThreshold). -spec acquire_inflight_slot(atom(), pos_integer()) -> ok | {error, overloaded}. @@ -185,14 +185,29 @@ retry_update_counter(Table, Key, Op) -> error:badarg -> {error, badarg} end. --spec is_countable_failure(response()) -> boolean(). -is_countable_failure({error, nxdomain}) -> false; -is_countable_failure({error, {failed_connect, _}}) -> false; -is_countable_failure({error, timeout}) -> false; -is_countable_failure({error, {timeout, _}}) -> false; -is_countable_failure({error, _}) -> true; -is_countable_failure({ok, StatusCode, _, _}) when StatusCode >= 500 -> true; -is_countable_failure(_) -> false. +-spec is_countable_circuit_failure({atom(), binary()}, response()) -> boolean(). +is_countable_circuit_failure({rpc, _Host}, Result) -> + is_countable_failure_with_transport(Result); +is_countable_circuit_failure({_Workload, _Host}, Result) -> + is_countable_failure_without_transport(Result). + +-spec is_countable_failure_without_transport(response()) -> boolean(). +is_countable_failure_without_transport({error, nxdomain}) -> false; +is_countable_failure_without_transport({error, {failed_connect, _}}) -> false; +is_countable_failure_without_transport({error, timeout}) -> false; +is_countable_failure_without_transport({error, {timeout, _}}) -> false; +is_countable_failure_without_transport({error, _}) -> true; +is_countable_failure_without_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true; +is_countable_failure_without_transport(_) -> false. + +-spec is_countable_failure_with_transport(response()) -> boolean(). +is_countable_failure_with_transport({error, nxdomain}) -> true; +is_countable_failure_with_transport({error, {failed_connect, _}}) -> true; +is_countable_failure_with_transport({error, timeout}) -> true; +is_countable_failure_with_transport({error, {timeout, _}}) -> true; +is_countable_failure_with_transport({error, _}) -> true; +is_countable_failure_with_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true; +is_countable_failure_with_transport(_) -> false. -spec record_result({atom(), binary()}, boolean(), integer(), pos_integer()) -> ok. record_result(CircuitKey, IsFailure, Now, FailureThreshold) -> @@ -314,3 +329,34 @@ safe_delete(Table, Key) -> catch error:badarg -> ok end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +circuit_ignores_transport_failures_for_non_rpc_workloads_test() -> + ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, nxdomain})), + ?assertEqual( + false, is_countable_circuit_failure({push, <<"h">>}, {error, {failed_connect, []}}) + ), + ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, timeout})), + ?assertEqual( + false, is_countable_circuit_failure({push, <<"h">>}, {error, {timeout, connect}}) + ), + ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {error, closed})), + ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {ok, 500, [], <<>>})), + ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {ok, 200, [], <<>>})). + +circuit_counts_transport_failures_for_rpc_test() -> + ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, nxdomain})), + ?assertEqual( + true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {failed_connect, []}}) + ), + ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, timeout})), + ?assertEqual( + true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {timeout, connect}}) + ), + ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, closed})), + ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 500, [], <<>>})), + ?assertEqual(false, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 200, [], <<>>})). + +-endif. diff --git a/fluxer_gateway/src/guild/guild.erl b/fluxer_gateway/src/guild/guild.erl index 00d190746..b44133945 100644 --- a/fluxer_gateway/src/guild/guild.erl +++ b/fluxer_gateway/src/guild/guild.erl @@ -9,6 +9,7 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(HIBERNATE_TIMEOUT, 60000). +-define(VOICE_MEMBERS_TABLE_WARNED, {?MODULE, voice_members_table_unavailable}). -type guild_state() :: map(). -type call_reply() :: @@ -46,6 +47,10 @@ handle_call(export_handoff_state, _From, State) -> {reply, {ok, guild_handoff:export_handoff_state(State)}, State}; handle_call({get_cached_voice_state_by_connection, ConnectionId}, _From, State) -> handle_cached_voice_state_call(ConnectionId, State); +handle_call({get_guild_id}, _From, State) -> + {reply, maps:get(id, State, undefined), State}; +handle_call({get_voice_guild_state}, _From, State) -> + {reply, voice_guild_state(State), State}; handle_call({dispatch, Request}, _From, State) -> handle_dispatch_call(Request, State); handle_call({reload, NewData}, _From, State) -> @@ -603,6 +608,112 @@ require_map(Value) when is_map(Value) -> require_map(Value) -> erlang:error({badmap, Value}). +-spec voice_guild_state(guild_state()) -> map(). +voice_guild_state(State) -> + case maps:get(data, State, #{}) of + #{members_ets := Tab} = Data -> + case voice_members_table_healthy(Tab) of + true -> + ok = clear_voice_members_table_warning(), + project_voice_guild_state(Data, State); + false -> + voice_members_table_unavailable(State) + end; + _ -> + voice_members_table_unavailable(State) + end. + +-spec voice_members_table_healthy(term()) -> boolean(). +voice_members_table_healthy(Tab) when is_reference(Tab) -> + try ets:info(eqwalizer:dynamic_cast(Tab), owner) of + Owner when is_pid(Owner) -> Owner =:= self(); + _ -> false + catch + error:badarg -> false + end; +voice_members_table_healthy(_Tab) -> + false. + +-spec project_voice_guild_state(map(), guild_state()) -> map(). +project_voice_guild_state(Data, State) -> + Projected = maps:with(voice_guild_state_keys(), State), + project_voice_sessions(Projected#{data => maps:with(voice_guild_data_keys(), Data)}). + +%% The sessions map keeps every key it had, so fold order over it is unchanged and the +%% dispatch order of every voice broadcast is unchanged. Only session ENTRIES are narrowed. +-spec project_voice_sessions(map()) -> map(). +project_voice_sessions(#{sessions := Sessions} = Projected) when is_map(Sessions) -> + Projected#{sessions => maps:map(fun project_voice_session/2, Sessions)}; +project_voice_sessions(Projected) -> + Projected. + +-spec project_voice_session(term(), term()) -> term(). +project_voice_session(_SessionId, Session) when is_map(Session) -> + maps:without(voice_session_drop_keys(), Session); +project_voice_session(_SessionId, Session) -> + Session. + +%% Every reader reachable from the voice guild state takes only pending_connect, user_id, +%% viewable_channels and pid out of a session entry: guild_sessions:filter_sessions_for_channel, +%% guild_voice_broadcast and guild_virtual_channel_access. These five are read nowhere on that path. +-spec voice_session_drop_keys() -> [atom()]. +voice_session_drop_keys() -> + [active_guilds, user_roles, mref, bot, is_staff]. + +-spec voice_guild_state_keys() -> [atom() | binary()]. +voice_guild_state_keys() -> + [ + id, + <<"id">>, + sessions, + guild_pid, + voice_states, + pending_voice_connections, + recently_disconnected_voice_states, + e2ee_room_keys, + virtual_channel_access, + virtual_channel_access_pending, + virtual_channel_access_preserve, + virtual_channel_access_move_pending, + test_perm_fun, + test_force_disconnect_fun, + test_livekit_fun, + test_permission_sync_fun + ]. + +-spec voice_guild_data_keys() -> [atom() | binary()]. +voice_guild_data_keys() -> + [ + <<"id">>, + <<"guild">>, + <<"roles">>, + <<"role_index">>, + role_perms_cache, + <<"channels">>, + <<"channel_index">>, + overwrite_perms_cache, + members_ets + ]. + +-spec voice_members_table_unavailable(guild_state()) -> map(). +voice_members_table_unavailable(State) -> + ok = log_voice_members_table_unavailable(maps:get(id, State, undefined)), + State. + +-spec log_voice_members_table_unavailable(term()) -> ok. +log_voice_members_table_unavailable(GuildId) -> + case erlang:put(?VOICE_MEMBERS_TABLE_WARNED, true) of + true -> + ok; + _ -> + logger:warning("guild_voice_members_table_unavailable: guild_id=~p", [GuildId]) + end. + +-spec clear_voice_members_table_warning() -> ok. +clear_voice_members_table_warning() -> + _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED), + ok. + -spec maybe_report_crash(term(), term()) -> ok. maybe_report_crash(normal, _) -> ok; @@ -630,4 +741,349 @@ handle_non_voice_exit_other_linked_stops_guild_test() -> handle_non_voice_exit(OtherPid, boom, State) ). +voice_guild_state_pins_projected_key_set_test() -> + with_voice_projection_state(fun(Full) -> + Projected = voice_guild_state(Full), + ?assertEqual( + lists:sort([ + id, + <<"id">>, + data, + sessions, + guild_pid, + voice_states, + pending_voice_connections, + recently_disconnected_voice_states, + e2ee_room_keys, + virtual_channel_access, + virtual_channel_access_pending, + virtual_channel_access_preserve, + virtual_channel_access_move_pending + ]), + lists:sort(maps:keys(Projected)) + ), + ?assertEqual( + lists:sort([ + <<"id">>, + <<"guild">>, + <<"roles">>, + <<"role_index">>, + role_perms_cache, + <<"channels">>, + <<"channel_index">>, + overwrite_perms_cache, + members_ets + ]), + lists:sort(maps:keys(maps:get(data, Projected))) + ) + end). + +voice_guild_state_drops_non_voice_payload_test() -> + with_voice_projection_state(fun(Full) -> + Projected = voice_guild_state(Full), + ?assertNot(maps:is_key(member_list_subscriptions, Projected)), + ?assertNot(maps:is_key(presence_subscriptions, Projected)), + ?assertNot(maps:is_key(connected_user_ids, Projected)), + Data = maps:get(data, Projected), + ?assertNot(maps:is_key(<<"members">>, Data)), + ?assertNot(maps:is_key(members_normalized, Data)), + ?assertNot(maps:is_key(<<"member_role_index">>, Data)), + ?assertNot(maps:is_key(<<"emojis">>, Data)) + end). + +voice_guild_state_preserves_member_permissions_test() -> + with_voice_projection_state(fun(Full) -> + Projected = voice_guild_state(Full), + Expected = guild_permissions:get_member_permissions(10, 500, Full), + ?assert(Expected > 0), + ?assert(permission_bits:has(Expected, constants:speak_permission())), + ?assertEqual(Expected, guild_permissions:get_member_permissions(10, 500, Projected)), + ?assertEqual( + voice_utils:compute_voice_permissions(10, 500, Full), + voice_utils:compute_voice_permissions(10, 500, Projected) + ), + ?assertEqual( + guild_permissions:find_member_by_user_id(10, Full), + guild_permissions:find_member_by_user_id(10, Projected) + ), + ?assertEqual( + guild_permissions:can_view_channel(10, 500, undefined, Full), + guild_permissions:can_view_channel(10, 500, undefined, Projected) + ) + end). + +voice_guild_state_absent_member_still_resolves_to_zero_test() -> + with_voice_projection_state(fun(Full) -> + Projected = voice_guild_state(Full), + ?assertEqual(undefined, guild_permissions:find_member_by_user_id(11, Projected)), + ?assertEqual(0, guild_permissions:get_member_permissions(11, 500, Projected)), + ?assertEqual( + guild_permissions:get_member_permissions(11, 500, Full), + guild_permissions:get_member_permissions(11, 500, Projected) + ) + end). + +voice_guild_state_put_member_keeps_permissions_resolvable_test() -> + with_voice_projection_state(fun(Full) -> + Projected = voice_guild_state(Full), + Data = maps:get(data, Projected), + Member = guild_permissions:find_member_by_user_id(10, Projected), + ?assert(is_map(Member)), + Rebuilt = guild_data_index:put_member(Member#{<<"mute">> => true}, Data), + RebuiltState = Projected#{data => Rebuilt}, + ?assertEqual(1, map_size(maps:get(<<"members">>, Rebuilt))), + ?assertEqual(1, map_size(maps:get(members_normalized, Rebuilt))), + ?assertEqual(maps:get(members_ets, Data), maps:get(members_ets, Rebuilt)), + ?assertEqual( + guild_permissions:get_member_permissions(10, 500, Full), + guild_permissions:get_member_permissions(10, 500, RebuiltState) + ), + ?assertMatch( + #{<<"mute">> := true}, + guild_permissions:find_member_by_user_id(10, RebuiltState) + ), + ?assertEqual( + guild_permissions:can_view_channel(10, 500, undefined, Full), + guild_permissions:can_view_channel(10, 500, undefined, RebuiltState) + ) + end). + +voice_guild_state_without_members_table_returns_full_state_test() -> + with_voice_projection_state(fun(Full) -> + Data = maps:get(data, Full), + Broken = Full#{data => maps:remove(members_ets, Data)}, + _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED), + ?assertEqual(Broken, voice_guild_state(Broken)), + ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)), + _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED) + end). + +voice_members_table_warning_rearms_after_recovery_test() -> + with_voice_projection_state(fun(Full) -> + Data = maps:get(data, Full), + Broken = Full#{data => maps:remove(members_ets, Data)}, + _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED), + _ = voice_guild_state(Broken), + ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)), + _ = voice_guild_state(Broken), + ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)), + _ = voice_guild_state(Full), + ?assertEqual(undefined, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)), + _ = voice_guild_state(Broken), + ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)), + _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED) + end). + +voice_guild_state_session_entries_keep_only_reader_keys_test() -> + with_voice_projection_state(fun(Base) -> + Full = Base#{sessions => voice_projection_sessions()}, + Projected = voice_guild_state(Full), + Sessions = maps:get(sessions, Projected), + Session = maps:get(<<"s-view">>, Sessions), + ?assertEqual( + lists:sort([session_id, user_id, pid, pending_connect, viewable_channels]), + lists:sort(maps:keys(Session)) + ), + ?assertNot(maps:is_key(active_guilds, Session)), + ?assertNot(maps:is_key(user_roles, Session)), + ?assertNot(maps:is_key(mref, Session)), + ?assertNot(maps:is_key(bot, Session)), + ?assertNot(maps:is_key(is_staff, Session)) + end). + +voice_guild_state_session_projection_matches_reference_test() -> + with_voice_projection_state(fun(Base) -> + Full = Base#{sessions => voice_projection_sessions()}, + Reference = reference_voice_guild_state(Full), + Projected = voice_guild_state(Full), + ?assertEqual(maps:remove(sessions, Reference), maps:remove(sessions, Projected)), + RefSessions = maps:get(sessions, Reference), + NewSessions = maps:get(sessions, Projected), + ?assertEqual(maps:keys(RefSessions), maps:keys(NewSessions)), + ?assertEqual( + maps:map(fun project_voice_session/2, RefSessions), + NewSessions + ) + end). + +voice_guild_state_session_readers_see_identical_results_test() -> + with_voice_projection_state(fun(Base) -> + Full = Base#{sessions => voice_projection_sessions()}, + Reference = reference_voice_guild_state(Full), + Projected = voice_guild_state(Full), + Expected = voice_session_reader_result(500, Reference), + ?assertEqual(Expected, voice_session_reader_result(500, Projected)), + {SessionIds, Pids} = Expected, + ?assertEqual([<<"s-cached">>, <<"s-perm">>, <<"s-view">>], lists:sort(SessionIds)), + ?assertEqual([self(), self(), self()], Pids), + Hidden = voice_session_reader_result(501, Reference), + ?assertEqual(Hidden, voice_session_reader_result(501, Projected)), + ?assertEqual({[<<"s-cached">>], [self()]}, Hidden) + end). + +voice_guild_state_session_projection_boundaries_test() -> + with_voice_projection_state(fun(Base) -> + Bare = #{user_id => 10, pid => self()}, + Full = Base#{sessions => #{<<"s-bare">> => Bare, <<"s-odd">> => not_a_map}}, + Sessions = maps:get(sessions, voice_guild_state(Full)), + ?assertEqual(Bare, maps:get(<<"s-bare">>, Sessions)), + ?assertEqual(not_a_map, maps:get(<<"s-odd">>, Sessions)), + NoSessions = maps:remove(sessions, Base), + ?assertNot(maps:is_key(sessions, voice_guild_state(NoSessions))), + BadSessions = Base#{sessions => not_a_map}, + ?assertEqual(not_a_map, maps:get(sessions, voice_guild_state(BadSessions))) + end). + +voice_guild_state_preserves_session_fold_order_at_scale_test() -> + with_voice_projection_state(fun voice_session_fold_order_scenario/1). + +%% 200 entries forces the hashmap representation, where iteration order is driven by key +%% hashes. maps:map/2 rewrites values only, so the key set and therefore the order is identical. +voice_session_fold_order_scenario(Base) -> + Sessions = maps:from_list(lists:map(fun voice_scale_session/1, lists:seq(1, 200))), + Full = Base#{sessions => Sessions}, + Projected = voice_guild_state(Full), + ?assertEqual(maps:keys(Sessions), maps:keys(maps:get(sessions, Projected))), + ?assertEqual( + voice_session_reader_result(500, reference_voice_guild_state(Full)), + voice_session_reader_result(500, Projected) + ). + +voice_scale_session(N) -> + SessionId = integer_to_binary(N), + {SessionId, voice_projection_session(SessionId, 10, #{500 => true}, false)}. + +voice_session_drop_keys_never_drops_a_reader_key_test() -> + Dropped = voice_session_drop_keys(), + Readers = [pending_connect, user_id, viewable_channels, pid], + ?assertEqual([], [K || K <- Readers, lists:member(K, Dropped)]). + +%% Reproduces the projection exactly as it was before session entries were narrowed, so the +%% tests above compare the new path against a live oracle rather than a hand-written literal. +reference_voice_guild_state(State) -> + Data = maps:get(data, State, #{}), + Projected = maps:with(voice_guild_state_keys(), State), + Projected#{data => maps:with(voice_guild_data_keys(), Data)}. + +%% The reader tuple is everything the voice path can observe about the sessions map: which +%% sessions match, in which fold order, and which pids the broadcast dispatches to. +voice_session_reader_result(ChannelId, State) -> + Sessions = maps:get(sessions, State, #{}), + Pairs = guild_sessions:filter_sessions_for_channel(Sessions, ChannelId, undefined, State), + {[Sid || {Sid, _S} <- Pairs], [maps:get(pid, S) || {_Sid, S} <- Pairs]}. + +voice_projection_sessions() -> + #{ + <<"s-view">> => voice_projection_session(<<"s-view">>, 10, #{500 => true}, false), + <<"s-perm">> => voice_projection_session(<<"s-perm">>, 10, #{}, false), + <<"s-pending">> => voice_projection_session(<<"s-pending">>, 10, #{500 => true}, true), + <<"s-stranger">> => voice_projection_session(<<"s-stranger">>, 11, #{}, false), + <<"s-cached">> => voice_projection_session(<<"s-cached">>, 10, #{501 => true}, false) + }. + +voice_projection_session(SessionId, UserId, ViewableChannels, Pending) -> + #{ + session_id => SessionId, + user_id => UserId, + pid => self(), + mref => make_ref(), + active_guilds => sets:from_list([42, 43, 44]), + user_roles => [77], + bot => false, + is_staff => false, + pending_connect => Pending, + viewable_channels => ViewableChannels + }. + +voice_members_table_warning_key_is_module_scoped_test() -> + ?assertEqual({?MODULE, voice_members_table_unavailable}, ?VOICE_MEMBERS_TABLE_WARNED), + ?assertNot(is_atom(?VOICE_MEMBERS_TABLE_WARNED)). + +with_voice_projection_state(Fun) -> + Tab = ets:new(guild_members_data, [set, public, {read_concurrency, true}]), + try + Fun(voice_projection_state(Tab)) + after + ets:delete(Tab) + end. + +voice_projection_state(Tab) -> + Data = voice_projection_data(), + maps:foreach( + fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end, + maps:get(members_normalized, Data) + ), + #{ + id => 42, + <<"id">> => <<"42">>, + data => Data#{members_ets => Tab, <<"emojis">> => [], channels_stale => true}, + sessions => #{}, + guild_pid => self(), + voice_states => #{}, + pending_voice_connections => #{}, + recently_disconnected_voice_states => #{}, + e2ee_room_keys => #{}, + virtual_channel_access => #{}, + virtual_channel_access_pending => #{}, + virtual_channel_access_preserve => #{}, + virtual_channel_access_move_pending => #{}, + presence_subscriptions => #{}, + member_list_subscriptions => #{}, + connected_user_ids => sets:new(), + counts => #{member_count => 1} + }. + +voice_projection_data() -> + Everyone = #{ + <<"id">> => <<"42">>, + <<"name">> => <<"@everyone">>, + <<"permissions">> => integer_to_binary(constants:view_channel_permission()) + }, + Speaker = #{ + <<"id">> => <<"77">>, + <<"name">> => <<"speaker">>, + <<"permissions">> => integer_to_binary( + constants:connect_permission() bor constants:speak_permission() + ) + }, + Channel = #{ + <<"id">> => <<"500">>, + <<"type">> => 2, + <<"name">> => <<"General">>, + <<"permission_overwrites">> => [ + #{ + <<"id">> => <<"77">>, + <<"type">> => 0, + <<"allow">> => integer_to_binary(constants:stream_permission()), + <<"deny">> => <<"0">> + } + ] + }, + HiddenChannel = #{ + <<"id">> => <<"501">>, + <<"type">> => 2, + <<"name">> => <<"Hidden">>, + <<"permission_overwrites">> => [ + #{ + <<"id">> => <<"42">>, + <<"type">> => 0, + <<"allow">> => <<"0">>, + <<"deny">> => integer_to_binary(constants:view_channel_permission()) + } + ] + }, + Member = #{ + <<"user">> => #{<<"id">> => <<"10">>, <<"username">> => <<"speaker">>}, + <<"roles">> => [<<"77">>], + <<"mute">> => false, + <<"deaf">> => false + }, + guild_data_index:normalize_data(#{ + <<"id">> => <<"42">>, + <<"guild">> => #{<<"id">> => <<"42">>, <<"owner_id">> => <<"9999">>}, + <<"roles">> => [Everyone, Speaker], + <<"channels">> => [Channel, HiddenChannel], + <<"members">> => [Member] + }). + -endif. diff --git a/fluxer_gateway/src/guild/guild_connect_async.erl b/fluxer_gateway/src/guild/guild_connect_async.erl index a0d81a736..b75bbe960 100644 --- a/fluxer_gateway/src/guild/guild_connect_async.erl +++ b/fluxer_gateway/src/guild/guild_connect_async.erl @@ -644,3 +644,61 @@ send_result(GuildId, Attempt, Result0, SessionPid) -> end, SessionPid ! {guild_connect_result, GuildId, Attempt, Reply}, ok. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +pending_connect_state(Sessions) -> + #{ + id => 42, + sessions => Sessions, + session_connect_pending => #{}, + session_connect_queue => queue:new(), + session_connect_inflight => ?SESSION_CONNECT_MAX_WORKERS, + session_connect_max_queue => ?SESSION_CONNECT_DEFAULT_MAX_QUEUE + }. + +pending_connect_request(SessionId, UserId) -> + #{ + session_id => SessionId, + user_id => UserId, + session_pid => self(), + bot => false, + is_staff => false, + active_guilds => sets:new() + }. + +enqueued_session_entry(SessionId, UserId, State0) -> + State1 = enqueue_session_connect_async( + 42, 1, pending_connect_request(SessionId, UserId), #{}, State0 + ), + maps:get(SessionId, maps:get(sessions, State1)). + +enqueue_marks_existing_session_pending_connect_test() -> + SessionId = <<"s-existing">>, + UserId = 21, + MRef = make_ref(), + Existing = #{ + session_id => SessionId, + user_id => UserId, + pid => self(), + mref => MRef, + pending_connect => false, + active_guilds => sets:new() + }, + Entry = enqueued_session_entry( + SessionId, UserId, pending_connect_state(#{SessionId => Existing}) + ), + ?assertEqual(true, maps:get(pending_connect, Entry)), + ?assertEqual(MRef, maps:get(mref, Entry)), + ?assertEqual(UserId, maps:get(user_id, Entry)). + +enqueue_creates_pending_session_entry_test() -> + SessionId = <<"s-fresh">>, + UserId = 22, + Entry = enqueued_session_entry(SessionId, UserId, pending_connect_state(#{})), + demonitor(maps:get(mref, Entry), [flush]), + ?assertEqual(true, maps:get(pending_connect, Entry)), + ?assertEqual(UserId, maps:get(user_id, Entry)). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_data.erl b/fluxer_gateway/src/guild/guild_data.erl index 6b2b69ccc..0854cc23e 100644 --- a/fluxer_gateway/src/guild/guild_data.erl +++ b/fluxer_gateway/src/guild/guild_data.erl @@ -25,6 +25,7 @@ -define(CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS, [ <<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids ]). +-define(CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, [active_guilds, user_roles, viewable_channels]). -export_type([guild_state/0, guild_reply/1, user_id/0]). @@ -167,15 +168,56 @@ build_connect_snapshot(Item, State) -> ], State ), - maybe_trim_connect_snapshot(Item, Base, State). + project_snapshot_sessions(maybe_trim_connect_snapshot(Item, Base, State)). + +-spec project_snapshot_sessions(map()) -> map(). +project_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) -> + Projected = maps:map( + fun(_SessionId, SessionData) -> project_snapshot_session(SessionData) end, + Sessions + ), + Snapshot#{sessions => Projected}; +project_snapshot_sessions(Snapshot) -> + Snapshot. + +-spec project_snapshot_session(term()) -> term(). +project_snapshot_session(SessionData) when is_map(SessionData) -> + maps:without(?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, SessionData); +project_snapshot_session(SessionData) -> + SessionData. -spec maybe_trim_connect_snapshot(map(), map(), guild_state()) -> map(). maybe_trim_connect_snapshot(Item, Base, State) -> - case has_members_ets(State) of + case should_trim_connect_snapshot(State) of true -> trim_connect_snapshot(Item, Base); false -> Base end. +-spec should_trim_connect_snapshot(guild_state()) -> boolean(). +should_trim_connect_snapshot(State) -> + case connect_snapshot_trim_member_threshold() of + undefined -> + false; + Threshold -> + member_count_at_least(Threshold, State) andalso has_members_ets(State) + end. + +-spec member_count_at_least(pos_integer(), guild_state()) -> boolean(). +member_count_at_least(Threshold, State) -> + case maps:get(member_count, State, undefined) of + Count when is_integer(Count) -> Count >= Threshold; + _ -> false + end. + +-spec connect_snapshot_trim_member_threshold() -> pos_integer() | undefined. +connect_snapshot_trim_member_threshold() -> + case + application:get_env(fluxer_gateway, connect_snapshot_trim_member_threshold, undefined) + of + N when is_integer(N), N > 0 -> N; + _ -> undefined + end. + -spec has_members_ets(guild_state()) -> boolean(). has_members_ets(#{data := #{members_ets := Tab}}) -> is_reference(Tab); has_members_ets(_) -> false. @@ -448,3 +490,107 @@ guild_id_wire_value(undefined) -> null; guild_id_wire_value(GuildId) -> integer_to_binary(GuildId). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +projection_snapshot() -> + #{ + sessions => #{ + <<"s1">> => #{ + session_id => <<"s1">>, + user_id => 7, + pid => self(), + bot => false, + is_staff => true, + pending_connect => false, + active_guilds => sets:new(), + user_roles => [1, 2], + viewable_channels => #{10 => true} + } + } + }. + +projection_entry(Snapshot) -> + maps:get(<<"s1">>, maps:get(sessions, Snapshot)). + +project_snapshot_sessions_drops_only_heavy_keys_test() -> + Entry = projection_entry(project_snapshot_sessions(projection_snapshot())), + ?assertEqual(7, maps:get(user_id, Entry)), + ?assertEqual(true, maps:get(is_staff, Entry)), + ?assertEqual(false, maps:get(pending_connect, Entry)), + ?assertEqual(<<"s1">>, maps:get(session_id, Entry)), + ?assertNot(maps:is_key(active_guilds, Entry)), + ?assertNot(maps:is_key(user_roles, Entry)), + ?assertNot(maps:is_key(viewable_channels, Entry)). + +projection_keeps_session_staff_lookup_test() -> + Projected = project_snapshot_sessions(projection_snapshot()), + ?assertEqual(true, guild_availability_check:is_user_staff(7, Projected)). + +projection_leaves_snapshot_data_untouched_test() -> + Data = #{<<"members">> => #{7 => #{}}, members_ets => make_ref()}, + Snapshot = maps:put(data, Data, projection_snapshot()), + ?assertEqual(Data, maps:get(data, project_snapshot_sessions(Snapshot))). + +projection_keeps_non_map_session_entries_test() -> + Snapshot = #{sessions => #{<<"broken">> => not_a_map}}, + ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)). + +projection_keeps_snapshot_without_sessions_test() -> + Snapshot = #{id => 42, member_count => 3}, + ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)). + +trim_state(MemberCount) -> + #{id => 42, member_count => MemberCount, data => #{members_ets => make_ref()}}. + +with_trim_threshold(Threshold, Fun) -> + application:set_env(fluxer_gateway, connect_snapshot_trim_member_threshold, Threshold), + try + Fun() + after + application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold) + end. + +trim_is_off_without_threshold_test() -> + application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold), + ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435))). + +trim_needs_member_count_at_threshold_test() -> + with_trim_threshold(20000, fun() -> + ?assertEqual(true, should_trim_connect_snapshot(trim_state(49435))), + ?assertEqual(true, should_trim_connect_snapshot(trim_state(20000))), + ?assertEqual(false, should_trim_connect_snapshot(trim_state(19999))), + NoCount = maps:remove(member_count, trim_state(1)), + ?assertEqual(false, should_trim_connect_snapshot(NoCount)) + end). + +trim_needs_members_ets_test() -> + with_trim_threshold(1, fun() -> + ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10, data => #{}})), + ?assertEqual( + false, + should_trim_connect_snapshot(#{member_count => 10, data => #{members_ets => 7}}) + ), + ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10})) + end). + +trim_ignores_invalid_threshold_test() -> + with_trim_threshold(0, fun() -> + ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435))) + end), + with_trim_threshold(not_an_integer, fun() -> + ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435))) + end). + +trim_and_projection_target_disjoint_keys_test() -> + ?assertEqual( + [], + [ + Key + || Key <- ?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, + lists:member(Key, ?CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS) + ] + ). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_data_index_members.erl b/fluxer_gateway/src/guild/guild_data_index_members.erl index f333c1ad1..e43d02739 100644 --- a/fluxer_gateway/src/guild/guild_data_index_members.erl +++ b/fluxer_gateway/src/guild/guild_data_index_members.erl @@ -9,6 +9,7 @@ member_list/1, member_count/1, member_ids/1, + sorted_member_ids/2, member_role_index/1, get_member/2, get_member_ets/2, @@ -84,6 +85,8 @@ member_list(Data) -> SortedIds = sorted_member_ids(Data, MemberMap), [Member || UserId <- SortedIds, {ok, Member} <- [maps:find(UserId, MemberMap)]]. +%% Every member mutation drops members_sorted_ids, so a cache that is present always +%% equals lists:sort(maps:keys(member_map(Data))) and callers may page it as-is. -spec sorted_member_ids(term(), #{user_id() => member()}) -> [user_id()]. sorted_member_ids(Data, MemberMap) when is_map(Data) -> case maps:find(members_sorted_ids, Data) of @@ -369,3 +372,50 @@ members_ets_table(#{members_ets := Tab}) -> Tab; members_ets_table(_) -> undefined. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +sorted_member_ids_equals_sorted_keys_test() -> + lists:foreach(fun assert_sorted_ids_match_keys/1, sorted_ids_fixtures()). + +assert_sorted_ids_match_keys(Data) -> + MemberMap = member_map(Data), + ?assertEqual(lists:sort(maps:keys(MemberMap)), sorted_member_ids(Data, MemberMap)). + +sorted_member_ids_cache_dropped_by_mutations_test() -> + Base = put_member_map(fixture_member_map([12, 3, 7]), #{}), + ?assertEqual([3, 7, 12], maps:get(members_sorted_ids, Base)), + ?assertNot(maps:is_key(members_sorted_ids, put_member(fixture_member(5), Base))), + ?assertNot(maps:is_key(members_sorted_ids, remove_member(3, Base))). + +sorted_ids_fixtures() -> + Base = put_member_map(fixture_member_map([12, 3, 7]), #{}), + Large = put_member_map(fixture_member_map(fixture_user_ids(40)), #{}), + [ + #{}, + #{<<"members">> => #{}}, + #{<<"members">> => fixture_member_list([9, 2, 5])}, + #{<<"members">> => #{<<"9">> => fixture_member(9), <<"2">> => fixture_member(2)}}, + #{<<"members">> => fixture_member_list(fixture_user_ids(40))}, + Base, + Large, + put_member(fixture_member(5), Base), + put_member(fixture_member(500), Large), + remove_member(3, Base), + put_member_list(fixture_member_list([4, 1]), Base) + ]. + +fixture_user_ids(Count) -> + lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)). + +fixture_member_map(UserIds) -> + maps:from_list([{UserId, fixture_member(UserId)} || UserId <- UserIds]). + +fixture_member_list(UserIds) -> + [fixture_member(UserId) || UserId <- UserIds]. + +fixture_member(UserId) -> + #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_data_members.erl b/fluxer_gateway/src/guild/guild_data_members.erl index e91c51331..35ff5caba 100644 --- a/fluxer_gateway/src/guild/guild_data_members.erl +++ b/fluxer_gateway/src/guild/guild_data_members.erl @@ -56,10 +56,15 @@ has_member(#{user_id := UserId}, State) -> -spec list_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}. list_guild_members(#{limit := Limit, offset := Offset}, State) -> Data = guild_data_index:ensure_data_map(State), - AllMembers = guild_data_index:member_list(Data), - TotalCount = length(AllMembers), - PaginatedMembers = paginate_members(AllMembers, Limit, Offset), - {reply, #{members => PaginatedMembers, total => TotalCount}, State}. + MemberMap = guild_data_index:member_map(Data), + SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap), + PageIds = paginate_members(SortedIds, Limit, Offset), + PaginatedMembers = lookup_members(PageIds, MemberMap), + {reply, #{members => PaginatedMembers, total => map_size(MemberMap)}, State}. + +-spec lookup_members([term()], map()) -> [guild_member()]. +lookup_members(UserIds, MemberMap) -> + [Member || UserId <- UserIds, {ok, Member} <- [maps:find(UserId, MemberMap)]]. -spec search_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}. search_guild_members(#{query := Query, limit := Limit}, State) when @@ -113,23 +118,69 @@ collect_query_match(Member, NormalizedQuery, Remaining, Acc, NextIterator) -> find_member_by_user_id(UserId, State) -> guild_permissions:find_member_by_user_id(UserId, State). --spec paginate_members([guild_member()], non_neg_integer(), non_neg_integer()) -> - [guild_member()]. -paginate_members(Members, Limit, Offset) -> - take_page(drop_members(Members, Offset), Limit, []). +-spec paginate_members([term()], non_neg_integer(), non_neg_integer()) -> [term()]. +paginate_members(Items, Limit, Offset) -> + take_page(drop_items(Items, Offset), Limit, []). --spec drop_members([guild_member()], non_neg_integer()) -> [guild_member()]. -drop_members(Members, 0) -> - Members; -drop_members([], _Offset) -> +-spec drop_items([term()], non_neg_integer()) -> [term()]. +drop_items(Items, 0) -> + Items; +drop_items([], _Offset) -> []; -drop_members([_ | Rest], Offset) -> - drop_members(Rest, Offset - 1). +drop_items([_ | Rest], Offset) -> + drop_items(Rest, Offset - 1). --spec take_page([guild_member()], non_neg_integer(), [guild_member()]) -> [guild_member()]. +-spec take_page([term()], non_neg_integer(), [term()]) -> [term()]. take_page(_Remaining, 0, Acc) -> lists:reverse(Acc); take_page([], _Limit, Acc) -> lists:reverse(Acc); -take_page([Member | Rest], Limit, Acc) -> - take_page(Rest, Limit - 1, [Member | Acc]). +take_page([Item | Rest], Limit, Acc) -> + take_page(Rest, Limit - 1, [Item | Acc]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +list_guild_members_matches_full_list_paging_test() -> + lists:foreach(fun assert_list_page_matches/1, list_paging_cases()). + +assert_list_page_matches({State, Limit, Offset}) -> + {reply, Reply, _} = list_guild_members(#{limit => Limit, offset => Offset}, State), + ?assertEqual(legacy_list_guild_members(Limit, Offset, State), Reply). + +legacy_list_guild_members(Limit, Offset, State) -> + Data = guild_data_index:ensure_data_map(State), + AllMembers = guild_data_index:member_list(Data), + #{members => paginate_members(AllMembers, Limit, Offset), total => length(AllMembers)}. + +list_paging_cases() -> + [ + {State, Limit, Offset} + || State <- list_paging_states(), + Limit <- [0, 1, 2, 5, 99], + Offset <- [0, 1, 3, 5, 6, 20, 39, 40, 41, 99] + ]. + +list_paging_states() -> + Members = paging_members(paging_user_ids(40)), + Cached = guild_data_index:normalize_data(#{<<"members">> => Members}), + [ + #{data => Cached}, + #{data => guild_data_index:put_member(paging_member(500), Cached)}, + #{data => guild_data_index:remove_member(7, Cached)}, + #{data => #{<<"members">> => Members}}, + #{data => guild_data_index:normalize_data(#{<<"members">> => []})}, + #{data => #{<<"members">> => #{}}}, + #{} + ]. + +paging_members(UserIds) -> + [paging_member(UserId) || UserId <- UserIds]. + +paging_user_ids(Count) -> + lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)). + +paging_member(UserId) -> + #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_data_wire.erl b/fluxer_gateway/src/guild/guild_data_wire.erl index dd7aee96b..cb972d164 100644 --- a/fluxer_gateway/src/guild/guild_data_wire.erl +++ b/fluxer_gateway/src/guild/guild_data_wire.erl @@ -5,186 +5,163 @@ -export([payload/1]). --type field_kind() :: snowflake | permission | snowflake_list | id_string | generic. --type path() :: [binary()]. +-type value_kind() :: scalar | scalar_list | maybe_scalar_list | id | restrict | generic. +-type key_kind() :: drop | value_kind(). -spec payload(term()) -> term(). payload(Value) -> - payload(Value, []). + fast_payload(Value, false). --spec payload(term(), path()) -> term(). -payload(Value, Path) when is_map(Value) -> +-spec fast_payload(term(), boolean()) -> term(). +fast_payload(Value, Restricted) when is_map(Value) -> maps:fold( fun(Key, FieldValue, Acc) -> - payload_map_field(Path, Key, FieldValue, Acc) + fast_map_field(Restricted, Key, FieldValue, Acc) end, #{}, Value ); -payload(Value, Path) when is_list(Value) -> - [payload(Item, Path) || Item <- Value]; -payload(Value, _Path) -> +fast_payload(Value, Restricted) when is_list(Value) -> + [fast_payload(Item, Restricted) || Item <- Value]; +fast_payload(Value, _Restricted) -> Value. --spec payload_map_field(path(), term(), term(), map()) -> map(). -payload_map_field(Path, Key, FieldValue, Acc) -> - KeyBinary = key_binary(Key), - case keep_payload_field(KeyBinary) of - true -> Acc#{payload_key(Key) => payload_field(Path, Key, KeyBinary, FieldValue)}; - false -> Acc - end. +-spec fast_map_field(boolean(), term(), term(), map()) -> map(). +fast_map_field(Restricted, Key, Value, Acc) when is_binary(Key) -> + fast_put(binary_key_kind(Key), Restricted, Key, Value, Acc); +fast_map_field(Restricted, Key, Value, Acc) when is_atom(Key) -> + Name = atom_to_binary(Key, utf8), + fast_put(named_or_suffix_kind(Name), Restricted, Name, Value, Acc); +fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key), Key > 0 -> + fast_put(maybe_scalar_list, Restricted, integer_to_binary(Key), Value, Acc); +fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key) -> + fast_put(generic, Restricted, integer_to_binary(Key), Value, Acc); +fast_map_field(Restricted, Key, Value, Acc) -> + fast_put(generic, Restricted, Key, Value, Acc). --spec keep_payload_field(binary() | undefined) -> boolean(). -keep_payload_field(<<"recipient_ids">>) -> false; -keep_payload_field(<<"role_index">>) -> false; -keep_payload_field(<<"channel_index">>) -> false; -keep_payload_field(<<"member_role_index">>) -> false; -keep_payload_field(<<"role_perms_cache">>) -> false; -keep_payload_field(<<"overwrite_perms_cache">>) -> false; -keep_payload_field(_KeyBinary) -> true. +-spec fast_put(key_kind(), boolean(), term(), term(), map()) -> map(). +fast_put(drop, _Restricted, _Key, _Value, Acc) -> + Acc; +fast_put(Kind, Restricted, Key, Value, Acc) -> + Acc#{Key => fast_field(Kind, Restricted, Value)}. --spec payload_key(term()) -> term(). -payload_key(Key) when is_integer(Key) -> - integer_to_binary(Key); -payload_key(Key) when is_atom(Key) -> - atom_to_binary(Key, utf8); -payload_key(Key) -> - Key. +-spec fast_field(key_kind(), boolean(), term()) -> term(). +fast_field(generic, Restricted, Value) -> + fast_payload(Value, Restricted); +fast_field(scalar, Restricted, Value) -> + fast_scalar(Value, Restricted); +fast_field(scalar_list, Restricted, Value) -> + fast_scalar_list(Value, Restricted); +fast_field(maybe_scalar_list, Restricted, Value) -> + fast_maybe_scalar_list(Value, Restricted); +fast_field(id, false, Value) -> + fast_scalar(Value, false); +fast_field(id, true, Value) -> + fast_payload(Value, true); +fast_field(restrict, _Restricted, Value) -> + fast_payload(Value, true). --spec payload_field(path(), term(), binary() | undefined, term()) -> term(). -payload_field(Path, Key, KeyBinary, Value) -> - FieldPath = path_push(KeyBinary, Path), - case field_kind(Path, Key, KeyBinary, Value) of - snowflake -> payload_snowflake(Value, FieldPath); - permission -> payload_permission(Value, FieldPath); - snowflake_list -> payload_snowflake_list(Value, FieldPath); - id_string -> payload_id_string(Value, FieldPath); - generic -> payload(Value, FieldPath) - end. - --spec payload_snowflake(term(), path()) -> term(). -payload_snowflake(Value, _Path) when is_integer(Value) -> +-spec fast_scalar(term(), boolean()) -> term(). +fast_scalar(Value, _Restricted) when is_integer(Value) -> integer_to_binary(Value); -payload_snowflake(Value, Path) -> - payload(Value, Path). +fast_scalar(Value, Restricted) -> + fast_payload(Value, Restricted). --spec payload_permission(term(), path()) -> term(). -payload_permission(Value, _Path) when is_integer(Value) -> - integer_to_binary(Value); -payload_permission(Value, Path) -> - payload(Value, Path). +-spec fast_scalar_list(term(), boolean()) -> term(). +fast_scalar_list(Values, Restricted) when is_list(Values) -> + [fast_scalar(Item, Restricted) || Item <- Values]; +fast_scalar_list(Value, Restricted) -> + fast_payload(Value, Restricted). --spec payload_snowflake_list(term(), path()) -> term(). -payload_snowflake_list(Values, Path) when is_list(Values) -> - [payload_snowflake(Item, Path) || Item <- Values]; -payload_snowflake_list(Value, Path) -> - payload(Value, Path). - --spec payload_id_string(term(), path()) -> term(). -payload_id_string(Value, _Path) when is_integer(Value) -> - integer_to_binary(Value); -payload_id_string(Value, Path) -> - payload(Value, Path). - --spec field_kind(path(), term(), binary() | undefined, term()) -> field_kind(). -field_kind(Path, Key, KeyBinary, Value) -> - case is_snowflake_record_list_value(Key, Value) of - true -> snowflake_list; - false -> field_kind_binary(Path, KeyBinary, Value) +-spec fast_maybe_scalar_list(term(), boolean()) -> term(). +fast_maybe_scalar_list(Value, Restricted) -> + case is_fast_scalar_snowflake_list(Value) of + true -> fast_scalar_list(Value, Restricted); + false -> fast_payload(Value, Restricted) end. --spec field_kind_binary(path(), binary() | undefined, term()) -> field_kind(). -field_kind_binary(_Path, <<"permissions">>, _Value) -> - permission; -field_kind_binary(_Path, <<"allow">>, _Value) -> - permission; -field_kind_binary(_Path, <<"deny">>, _Value) -> - permission; -field_kind_binary(Path, Key, Value) when is_binary(Key) -> - field_kind_for_binary(Path, Key, Value); -field_kind_binary(_Path, _Key, _Value) -> - generic. +-spec is_fast_scalar_snowflake_list(term()) -> boolean(). +is_fast_scalar_snowflake_list([]) -> + true; +is_fast_scalar_snowflake_list(Values) when is_list(Values) -> + lists:all(fun is_fast_snowflake_scalar/1, Values); +is_fast_scalar_snowflake_list(_) -> + false. --spec field_kind_for_binary(path(), binary(), term()) -> field_kind(). -field_kind_for_binary(Path, Key, Value) -> - case is_snowflake_list_key(Key, Value) of - true -> snowflake_list; - false -> field_kind_scalar(Path, Key) +-spec is_fast_snowflake_scalar(term()) -> boolean(). +is_fast_snowflake_scalar(Value) when is_integer(Value), Value > 0 -> + true; +is_fast_snowflake_scalar(<> = Value) when First >= $1, First =< $9 -> + snowflake_id:is_valid(Value); +is_fast_snowflake_scalar(_) -> + false. + +-spec binary_key_kind(binary()) -> key_kind(). +binary_key_kind(<> = Key) when First >= $1, First =< $9 -> + numeric_binary_key_kind(Key); +binary_key_kind(Key) -> + named_or_suffix_kind(Key). + +-spec named_or_suffix_kind(binary()) -> key_kind(). +named_or_suffix_kind(Key) -> + case named_key_kind(Key) of + unknown -> suffix_key_kind(Key); + Kind -> Kind end. --spec field_kind_scalar(path(), binary()) -> snowflake | id_string | generic. -field_kind_scalar(Path, Key) -> - case is_snowflake_key(Path, Key) of - true -> snowflake; - false -> opaque_id_kind(Key) +-spec numeric_binary_key_kind(binary()) -> value_kind(). +numeric_binary_key_kind(Key) -> + case snowflake_id:is_valid(Key) of + true -> maybe_scalar_list; + false -> suffix_key_kind(Key) end. --spec opaque_id_kind(binary()) -> id_string | generic. -opaque_id_kind(Key) -> - case is_opaque_id_key(Key) of - true -> id_string; +-spec suffix_key_kind(binary()) -> value_kind(). +suffix_key_kind(Key) -> + case has_suffix(Key, <<"_ids">>) of + true -> scalar_list; + false -> trailing_id_key_kind(Key) + end. + +-spec trailing_id_key_kind(binary()) -> value_kind(). +trailing_id_key_kind(Key) -> + case has_suffix(Key, <<"_id">>) of + true -> scalar; false -> generic end. --spec is_snowflake_key(path(), binary()) -> boolean(). -is_snowflake_key(Path, <<"id">>) -> - not has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path); -is_snowflake_key(_Path, Key) -> - has_suffix(Key, <<"_id">>) andalso not is_opaque_id_key(Key). - --spec is_opaque_id_key(binary()) -> boolean(). -is_opaque_id_key(<<"session_id">>) -> true; -is_opaque_id_key(<<"connection_id">>) -> true; -is_opaque_id_key(<<"subscription_id">>) -> true; -is_opaque_id_key(<<"app_id">>) -> true; -is_opaque_id_key(<<"device_id">>) -> true; -is_opaque_id_key(<<"region_id">>) -> true; -is_opaque_id_key(<<"server_id">>) -> true; -is_opaque_id_key(<<"target_id">>) -> true; -is_opaque_id_key(_) -> false. - --spec is_snowflake_list_key(binary(), term()) -> boolean(). -is_snowflake_list_key(<<"mention_roles">>, _Value) -> true; -is_snowflake_list_key(<<"participants">>, _Value) -> true; -is_snowflake_list_key(<<"ringing">>, _Value) -> true; -is_snowflake_list_key(<<"nsfw_emojis">>, _Value) -> true; -is_snowflake_list_key(<<"pinned_dms">>, _Value) -> true; -is_snowflake_list_key(<<"restricted_guilds">>, _Value) -> true; -is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) -> true; -is_snowflake_list_key(<<"roles">>, Value) -> is_scalar_snowflake_list(Value); -is_snowflake_list_key(<<"mentions">>, Value) -> is_scalar_snowflake_list(Value); -is_snowflake_list_key(<<"recipients">>, Value) -> is_scalar_snowflake_list(Value); -is_snowflake_list_key(Key, _Value) -> has_suffix(Key, <<"_ids">>). - --spec is_snowflake_record_list_value(term(), term()) -> boolean(). -is_snowflake_record_list_value(Key, Value) -> - key_is_snowflake(Key) andalso is_scalar_snowflake_list(Value). - --spec key_is_snowflake(term()) -> boolean(). -key_is_snowflake(Key) when is_integer(Key), Key > 0 -> - true; -key_is_snowflake(<>) when First < $1; First > $9 -> - false; -key_is_snowflake(Key) when is_binary(Key) -> - snowflake_id:is_valid(Key); -key_is_snowflake(_) -> - false. - --spec is_scalar_snowflake_list(term()) -> boolean(). -is_scalar_snowflake_list([]) -> - true; -is_scalar_snowflake_list(Values) when is_list(Values) -> - lists:all(fun is_snowflake_scalar/1, Values); -is_scalar_snowflake_list(_) -> - false. - --spec is_snowflake_scalar(term()) -> boolean(). -is_snowflake_scalar(Value) when is_integer(Value), Value > 0 -> - true; -is_snowflake_scalar(Value) when is_binary(Value) -> - snowflake_id:is_valid(Value); -is_snowflake_scalar(_) -> - false. +-spec named_key_kind(binary()) -> key_kind() | unknown. +named_key_kind(<<"id">>) -> id; +named_key_kind(<<"permissions">>) -> scalar; +named_key_kind(<<"allow">>) -> scalar; +named_key_kind(<<"deny">>) -> scalar; +named_key_kind(<<"session_id">>) -> scalar; +named_key_kind(<<"connection_id">>) -> scalar; +named_key_kind(<<"subscription_id">>) -> scalar; +named_key_kind(<<"app_id">>) -> scalar; +named_key_kind(<<"device_id">>) -> scalar; +named_key_kind(<<"region_id">>) -> scalar; +named_key_kind(<<"server_id">>) -> scalar; +named_key_kind(<<"target_id">>) -> scalar; +named_key_kind(<<"mention_roles">>) -> scalar_list; +named_key_kind(<<"participants">>) -> scalar_list; +named_key_kind(<<"ringing">>) -> scalar_list; +named_key_kind(<<"nsfw_emojis">>) -> scalar_list; +named_key_kind(<<"pinned_dms">>) -> scalar_list; +named_key_kind(<<"restricted_guilds">>) -> scalar_list; +named_key_kind(<<"bot_restricted_guilds">>) -> scalar_list; +named_key_kind(<<"roles">>) -> maybe_scalar_list; +named_key_kind(<<"mentions">>) -> maybe_scalar_list; +named_key_kind(<<"recipients">>) -> maybe_scalar_list; +named_key_kind(<<"guild_folders">>) -> restrict; +named_key_kind(<<"rtc_regions">>) -> restrict; +named_key_kind(<<"recipient_ids">>) -> drop; +named_key_kind(<<"role_index">>) -> drop; +named_key_kind(<<"channel_index">>) -> drop; +named_key_kind(<<"member_role_index">>) -> drop; +named_key_kind(<<"role_perms_cache">>) -> drop; +named_key_kind(<<"overwrite_perms_cache">>) -> drop; +named_key_kind(_) -> unknown. -spec has_suffix(binary(), binary()) -> boolean(). has_suffix(Value, Suffix) -> @@ -204,20 +181,347 @@ has_suffix(Value, Suffix, PrefixSize, SuffixSize) -> _ -> false end. --spec key_binary(term()) -> binary() | undefined. -key_binary(Key) when is_binary(Key) -> - Key; -key_binary(Key) when is_atom(Key) -> +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +-type field_kind() :: snowflake | permission | snowflake_list | id_string | generic. +-type path() :: [binary()]. + +fast_payload_matches_original_test() -> + lists:foreach(fun assert_fast_matches_original/1, corpus()). + +fast_payload_matches_original_for_mixed_keys_test() -> + lists:foreach(fun assert_fast_matches_original/1, mixed_key_maps()). + +fast_payload_matches_original_under_restricted_paths_test() -> + lists:foreach(fun assert_fast_matches_original/1, restricted_corpus()). + +fast_payload_keeps_restricted_ids_opaque_test() -> + Data = #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => 7}]}, + ?assertEqual( + #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => <<"7">>}]}, + fast_payload(Data, false) + ). + +fast_payload_drops_internal_keys_test() -> + Data = #{<<"id">> => 1, <<"role_index">> => #{}, role_perms_cache => #{}}, + ?assertEqual(#{<<"id">> => <<"1">>}, fast_payload(Data, false)). + +pre_encoded_payload_passes_through_unchanged_test() -> + Data = {pre_encoded, <<"{}">>}, + ?assertEqual(Data, payload(Data)). + +assert_fast_matches_original(Value) -> + ?assertEqual({Value, reference_payload(Value, [])}, {Value, fast_payload(Value, false)}). + +corpus() -> + [#{Key => Value} || Key <- sample_keys(), Value <- sample_values()]. + +restricted_corpus() -> + lists:flatmap(fun restricted_variants/1, corpus()). + +restricted_variants(Map) -> + [ + #{<<"guild_folders">> => [Map]}, + #{<<"rtc_regions">> => Map}, + #{guild_folders => #{<<"nested">> => Map}}, + #{<<"data">> => [#{<<"rtc_regions">> => [Map]}]}, + #{<<"data">> => #{<<"more">> => Map}} + ]. + +mixed_key_maps() -> + [ + #{<<"id">> => 1, id => 2}, + #{<<"100">> => [1], 100 => [2]}, + #{<<"permissions">> => [1], permissions => 2}, + #{<<"recipient_ids">> => [1], recipient_ids => [2], <<"id">> => 3}, + #{1 => [2], -1 => [3], 0 => [4]}, + #{<<"guild_folders">> => [#{<<"id">> => 1, <<"guild_ids">> => [2], id => 3}]}, + #{<<"roles">> => [1, 2], <<"mentions">> => [#{<<"id">> => 3}], <<"deny">> => 4} + ]. + +sample_keys() -> + [ + <<"id">>, + <<"guild_id">>, + <<"session_id">>, + <<"connection_id">>, + <<"subscription_id">>, + <<"app_id">>, + <<"device_id">>, + <<"region_id">>, + <<"server_id">>, + <<"target_id">>, + <<"permissions">>, + <<"allow">>, + <<"deny">>, + <<"roles">>, + <<"mentions">>, + <<"recipients">>, + <<"ringing">>, + <<"mention_roles">>, + <<"participants">>, + <<"nsfw_emojis">>, + <<"pinned_dms">>, + <<"restricted_guilds">>, + <<"bot_restricted_guilds">>, + <<"recipient_ids">>, + <<"role_index">>, + <<"channel_index">>, + <<"member_role_index">>, + <<"role_perms_cache">>, + <<"overwrite_perms_cache">>, + <<"guild_folders">>, + <<"rtc_regions">>, + <<"guild_ids">>, + <<"username">>, + <<"_id">>, + <<"_ids">>, + <<"id_">>, + <<"9_ids">>, + <<"100">>, + <<"0100">>, + <<"1x">>, + <<>>, + id, + guild_id, + session_id, + roles, + permissions, + role_perms_cache, + guild_folders, + '100', + 100, + 0, + -1, + 1.5, + {tuple, key} + ]. + +sample_values() -> + [ + 42, + -7, + 0, + <<"text">>, + <<"42">>, + null, + true, + undefined, + [], + [1, 2], + [<<"3">>, 4], + [<<"abc">>], + [[1, 2]], + [#{<<"id">> => 5}], + #{<<"id">> => 6, <<"session_id">> => <<"s">>, <<"user_id">> => 7}, + #{<<"id">> => 8, <<"roles">> => [9], <<"role_index">> => #{}}, + #{<<"nested">> => #{<<"id">> => 10, <<"guild_ids">> => [11]}} + ]. + +-spec reference_payload(term(), path()) -> term(). +reference_payload(Value, Path) when is_map(Value) -> + maps:fold( + fun(Key, FieldValue, Acc) -> + reference_payload_map_field(Path, Key, FieldValue, Acc) + end, + #{}, + Value + ); +reference_payload(Value, Path) when is_list(Value) -> + [reference_payload(Item, Path) || Item <- Value]; +reference_payload(Value, _Path) -> + Value. + +-spec reference_payload_map_field(path(), term(), term(), map()) -> map(). +reference_payload_map_field(Path, Key, FieldValue, Acc) -> + case reference_keep_payload_field(Key) of + true -> + Acc#{reference_payload_key(Key) => reference_payload_field(Path, Key, FieldValue)}; + false -> + Acc + end. + +-spec reference_keep_payload_field(term()) -> boolean(). +reference_keep_payload_field(Key) -> + not lists:member(reference_key_binary(Key), [ + <<"recipient_ids">>, + <<"role_index">>, + <<"channel_index">>, + <<"member_role_index">>, + <<"role_perms_cache">>, + <<"overwrite_perms_cache">> + ]). + +-spec reference_payload_key(term()) -> term(). +reference_payload_key(Key) when is_integer(Key) -> + integer_to_binary(Key); +reference_payload_key(Key) when is_atom(Key) -> atom_to_binary(Key, utf8); -key_binary(_) -> +reference_payload_key(Key) -> + Key. + +-spec reference_payload_field(path(), term(), term()) -> term(). +reference_payload_field(Path, Key, Value) -> + FieldPath = reference_path_push(Key, Path), + case reference_field_kind(Path, Key, Value) of + snowflake -> reference_payload_snowflake(Value, FieldPath); + permission -> reference_payload_permission(Value, FieldPath); + snowflake_list -> reference_payload_snowflake_list(Value, FieldPath); + id_string -> reference_payload_id_string(Value, FieldPath); + generic -> reference_payload(Value, FieldPath) + end. + +-spec reference_payload_snowflake(term(), path()) -> term(). +reference_payload_snowflake(Value, _Path) when is_integer(Value) -> + integer_to_binary(Value); +reference_payload_snowflake(Value, Path) -> + reference_payload(Value, Path). + +-spec reference_payload_permission(term(), path()) -> term(). +reference_payload_permission(Value, _Path) when is_integer(Value) -> + integer_to_binary(Value); +reference_payload_permission(Value, Path) -> + reference_payload(Value, Path). + +-spec reference_payload_snowflake_list(term(), path()) -> term(). +reference_payload_snowflake_list(Values, Path) when is_list(Values) -> + [reference_payload_snowflake(Item, Path) || Item <- Values]; +reference_payload_snowflake_list(Value, Path) -> + reference_payload(Value, Path). + +-spec reference_payload_id_string(term(), path()) -> term(). +reference_payload_id_string(Value, _Path) when is_integer(Value) -> + integer_to_binary(Value); +reference_payload_id_string(Value, Path) -> + reference_payload(Value, Path). + +-spec reference_field_kind(path(), term(), term()) -> field_kind(). +reference_field_kind(Path, Key, Value) -> + case reference_is_snowflake_record_list_value(Key, Value) of + true -> snowflake_list; + false -> reference_field_kind_binary(Path, reference_key_binary(Key), Value) + end. + +-spec reference_field_kind_binary(path(), binary() | undefined, term()) -> field_kind(). +reference_field_kind_binary(_Path, <<"permissions">>, _Value) -> + permission; +reference_field_kind_binary(_Path, <<"allow">>, _Value) -> + permission; +reference_field_kind_binary(_Path, <<"deny">>, _Value) -> + permission; +reference_field_kind_binary(Path, Key, Value) when is_binary(Key) -> + reference_field_kind_for_binary(Path, Key, Value); +reference_field_kind_binary(_Path, _Key, _Value) -> + generic. + +-spec reference_field_kind_for_binary(path(), binary(), term()) -> field_kind(). +reference_field_kind_for_binary(Path, Key, Value) -> + case reference_is_snowflake_list_key(Key, Value) of + true -> snowflake_list; + false -> reference_field_kind_scalar(Path, Key) + end. + +-spec reference_field_kind_scalar(path(), binary()) -> snowflake | id_string | generic. +reference_field_kind_scalar(Path, Key) -> + case reference_is_snowflake_key(Path, Key) of + true -> snowflake; + false -> reference_opaque_id_kind(Key) + end. + +-spec reference_opaque_id_kind(binary()) -> id_string | generic. +reference_opaque_id_kind(Key) -> + case reference_is_opaque_id_key(Key) of + true -> id_string; + false -> generic + end. + +-spec reference_is_snowflake_key(path(), binary()) -> boolean(). +reference_is_snowflake_key(Path, <<"id">>) -> + not reference_has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path); +reference_is_snowflake_key(_Path, Key) -> + has_suffix(Key, <<"_id">>) andalso not reference_is_opaque_id_key(Key). + +-spec reference_is_opaque_id_key(binary()) -> boolean(). +reference_is_opaque_id_key(<<"session_id">>) -> true; +reference_is_opaque_id_key(<<"connection_id">>) -> true; +reference_is_opaque_id_key(<<"subscription_id">>) -> true; +reference_is_opaque_id_key(<<"app_id">>) -> true; +reference_is_opaque_id_key(<<"device_id">>) -> true; +reference_is_opaque_id_key(<<"region_id">>) -> true; +reference_is_opaque_id_key(<<"server_id">>) -> true; +reference_is_opaque_id_key(<<"target_id">>) -> true; +reference_is_opaque_id_key(_) -> false. + +-spec reference_is_snowflake_list_key(binary(), term()) -> boolean(). +reference_is_snowflake_list_key(<<"mention_roles">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"participants">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"ringing">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"nsfw_emojis">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"pinned_dms">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"restricted_guilds">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) -> + true; +reference_is_snowflake_list_key(<<"roles">>, Value) -> + reference_is_scalar_snowflake_list(Value); +reference_is_snowflake_list_key(<<"mentions">>, Value) -> + reference_is_scalar_snowflake_list(Value); +reference_is_snowflake_list_key(<<"recipients">>, Value) -> + reference_is_scalar_snowflake_list(Value); +reference_is_snowflake_list_key(Key, _Value) -> + has_suffix(Key, <<"_ids">>). + +-spec reference_is_snowflake_record_list_value(term(), term()) -> boolean(). +reference_is_snowflake_record_list_value(Key, Value) -> + reference_key_is_snowflake(Key) andalso reference_is_scalar_snowflake_list(Value). + +-spec reference_key_is_snowflake(term()) -> boolean(). +reference_key_is_snowflake(Key) when is_integer(Key), Key > 0 -> + true; +reference_key_is_snowflake(Key) when is_binary(Key) -> + snowflake_id:is_valid(Key); +reference_key_is_snowflake(_) -> + false. + +-spec reference_is_scalar_snowflake_list(term()) -> boolean(). +reference_is_scalar_snowflake_list([]) -> + true; +reference_is_scalar_snowflake_list(Values) when is_list(Values) -> + lists:all(fun reference_is_snowflake_scalar/1, Values); +reference_is_scalar_snowflake_list(_) -> + false. + +-spec reference_is_snowflake_scalar(term()) -> boolean(). +reference_is_snowflake_scalar(Value) when is_integer(Value), Value > 0 -> + true; +reference_is_snowflake_scalar(Value) when is_binary(Value) -> + snowflake_id:is_valid(Value); +reference_is_snowflake_scalar(_) -> + false. + +-spec reference_key_binary(term()) -> binary() | undefined. +reference_key_binary(Key) when is_binary(Key) -> + Key; +reference_key_binary(Key) when is_atom(Key) -> + atom_to_binary(Key, utf8); +reference_key_binary(_) -> undefined. --spec path_push(binary() | undefined, path()) -> path(). -path_push(undefined, Path) -> - Path; -path_push(Binary, Path) -> - [Binary | Path]. +-spec reference_path_push(term(), path()) -> path(). +reference_path_push(Key, Path) -> + case reference_key_binary(Key) of + undefined -> Path; + Binary -> [Binary | Path] + end. --spec has_any_path([binary()], path()) -> boolean(). -has_any_path(Keys, Path) -> +-spec reference_has_any_path([binary()], path()) -> boolean(). +reference_has_any_path(Keys, Path) -> lists:any(fun(Key) -> lists:member(Key, Path) end, Keys). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl b/fluxer_gateway/src/guild/guild_dispatch_member_list.erl index 88441d289..08a2b14fe 100644 --- a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl +++ b/fluxer_gateway/src/guild/guild_dispatch_member_list.erl @@ -43,12 +43,12 @@ dispatch_member_list_event(channel_create, _EventData, _OldState, UpdatedState) broadcast_all_updates(UpdatedState); dispatch_member_list_event(channel_delete, _EventData, _OldState, UpdatedState) -> broadcast_all_updates(UpdatedState); -dispatch_member_list_event(channel_update, EventData, _OldState, UpdatedState) -> - broadcast_channel_update(EventData, UpdatedState); -dispatch_member_list_event(channel_update_bulk, EventData, _OldState, UpdatedState) -> +dispatch_member_list_event(channel_update, EventData, OldState, UpdatedState) -> + broadcast_channel_update(EventData, OldState, UpdatedState); +dispatch_member_list_event(channel_update_bulk, EventData, OldState, UpdatedState) -> Channels = maps:get(<<"channels">>, EventData, []), lists:foldl( - fun broadcast_channel_update/2, + fun(Channel, AccState) -> broadcast_channel_update(Channel, OldState, AccState) end, UpdatedState, Channels ); @@ -73,19 +73,59 @@ broadcast_all_updates(UpdatedState) -> {ok, NewState} = guild_member_list:broadcast_all_member_list_updates(UpdatedState), NewState. --spec broadcast_channel_update(event_data(), guild_state()) -> guild_state(). -broadcast_channel_update(EventData, UpdatedState) -> +-spec broadcast_channel_update(event_data(), guild_state(), guild_state()) -> guild_state(). +broadcast_channel_update(EventData, OldState, UpdatedState) -> ChannelIdBin = maps:get(<<"id">>, EventData, undefined), case guild_dispatch_decorate:parse_snowflake(<<"id">>, ChannelIdBin) of undefined -> UpdatedState; ChannelId -> + broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState) + end. + +%% guild_state:post_update_channel/2 has already rebuilt this channel's engine +%% earlier in the same dispatch, through +%% guild_member_list_write:rebuild_channels_for_permission_change/2. Rebuilding +%% again here is a second full O(members) build over the same state, so when the +%% engine already exists and nothing in the window can have changed what a +%% rebuild would see, queue the sync and skip the rebuild. +-spec broadcast_channel_update_for_id(integer(), guild_state(), guild_state()) -> guild_state(). +broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState) -> + case rebuild_is_redundant(ChannelId, OldState, UpdatedState) of + true -> + sync_channel_list_without_rebuild(ChannelId, UpdatedState); + false -> {ok, NewState} = guild_member_list:broadcast_member_list_updates_for_channel( ChannelId, UpdatedState ), NewState end. +%% The engine must already exist, because the earlier pass only rebuilds +%% ALREADY-LOADED engines while this pass would build a missing one. Virtual +%% channel access is compared across the whole dispatch, which is a conservative +%% superset of the window between the two rebuilds: guild_visibility can grant a +%% user virtual access to a channel they just lost permission for, and that does +%% change what a rebuild sees. +-spec rebuild_is_redundant(integer(), guild_state(), guild_state()) -> boolean(). +rebuild_is_redundant(ChannelId, OldState, UpdatedState) -> + guild_member_list_channel_engine:ref(integer_to_binary(ChannelId), UpdatedState) =/= + undefined andalso + maps:get(virtual_channel_access, OldState, #{}) =:= + maps:get(virtual_channel_access, UpdatedState, #{}). + +%% Mirrors guild_member_list_write:broadcast_list_by_id/4 with the rebuild +%% removed: a list nobody is subscribed to is left alone, otherwise the sync is +%% queued exactly as that function would have queued it. +-spec sync_channel_list_without_rebuild(integer(), guild_state()) -> guild_state(). +sync_channel_list_without_rebuild(ChannelId, State) -> + ListId = integer_to_binary(ChannelId), + SubsTab = maps:get(member_list_subscriptions, State), + case map_size(guild_member_list_subs:get_list_subs(ListId, SubsTab)) of + 0 -> State; + _ -> guild_member_list_sync_batch:queue_list_sync(ListId, State) + end. + -spec extract_user_id_from_event(event_data()) -> user_id() | undefined. extract_user_id_from_event(EventData) -> MUser = maps:get(<<"user">>, EventData, #{}), diff --git a/fluxer_gateway/src/guild/guild_dispatch_push.erl b/fluxer_gateway/src/guild/guild_dispatch_push.erl index 2e72a38b6..700737321 100644 --- a/fluxer_gateway/src/guild/guild_dispatch_push.erl +++ b/fluxer_gateway/src/guild/guild_dispatch_push.erl @@ -8,11 +8,30 @@ collect_and_send_push_notifications/3 ]). +-define(MAX_FORMAT_MEMBERS, 50). +-define(CONCURRENCY_LIMIT_KEY, guild_push_concurrency_limit). +-define(DEFAULT_PUSH_CONCURRENCY, 2). +-define(MAX_PUSH_CONCURRENCY, 8). +-define(PUSH_WORKER_MAX_AGE_MS, 60000). +-define(PUSH_COUNTERS, guild_push_counters). +-define(PUSH_COUNTER_KEYS, [ + worker_started, + worker_completed, + worker_failed, + dropped_at_limit, + spawn_failed, + slot_reclaimed, + slot_deduped, + members_table_missing, + scan_table_unavailable +]). + -type event() :: atom(). -type event_data() :: map(). -type guild_state() :: map(). -type guild_id() :: integer(). -type user_id() :: integer(). +-type push_worker() :: {integer(), pid(), integer()}. -export_type([event/0, event_data/0, guild_state/0, guild_id/0]). -spec maybe_send_push_notifications(event(), event_data(), guild_id(), guild_state()) -> ok. @@ -26,32 +45,496 @@ maybe_send_push_notifications(_Event, _FinalData, _GuildId, _UpdatedState) -> -spec maybe_spawn_push(event_data(), guild_id(), guild_state()) -> ok. maybe_spawn_push(FinalData, GuildId, UpdatedState) -> - case push_inflight_alive() of - true -> ok; - false -> spawn_push(FinalData, GuildId, UpdatedState) + Limit = push_concurrency_limit(), + Workers = live_push_workers(), + put(push_inflight_workers, Workers), + case length(Workers) < Limit of + true -> spawn_push(FinalData, GuildId, UpdatedState); + false -> count_push_event(dropped_at_limit) end. --spec push_inflight_alive() -> boolean(). -push_inflight_alive() -> - case get(push_inflight) of - Pid when is_pid(Pid) -> is_process_alive(Pid); - _ -> false +-spec local_process_alive(pid()) -> boolean(). +local_process_alive(Pid) -> + node(Pid) =:= node() andalso erlang:is_process_alive(Pid). + +-spec push_concurrency_limit() -> pos_integer(). +push_concurrency_limit() -> + case application:get_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, undefined) of + Value when is_integer(Value), Value > 0 -> min(Value, ?MAX_PUSH_CONCURRENCY); + _ -> ?DEFAULT_PUSH_CONCURRENCY + end. + +-spec live_push_workers() -> [push_worker()]. +live_push_workers() -> + Now = erlang:monotonic_time(millisecond), + prune_push_workers(tracked_push_workers(Now), Now, #{}, []). + +-spec tracked_push_workers(integer()) -> [term()]. +tracked_push_workers(Now) -> + merge_inflight_push_worker(get(push_inflight), tracked_worker_list(), Now). + +-spec tracked_worker_list() -> [term()]. +tracked_worker_list() -> + case get(push_inflight_workers) of + Workers when is_list(Workers) -> Workers; + _ -> [] + end. + +-spec merge_inflight_push_worker(term(), [term()], integer()) -> [term()]. +merge_inflight_push_worker(Pid, Workers, Now) when is_pid(Pid) -> + case lists:any(fun(Worker) -> push_worker_pid(Worker) =:= Pid end, Workers) of + true -> Workers; + false -> Workers ++ [{0, Pid, Now}] + end; +merge_inflight_push_worker(_Inflight, Workers, _Now) -> + Workers. + +-spec push_worker_pid(term()) -> pid() | undefined. +push_worker_pid({_Gen, Pid, _StartedMs}) when is_pid(Pid) -> Pid; +push_worker_pid(_Worker) -> undefined. + +-spec prune_push_workers([term()], integer(), map(), [push_worker()]) -> [push_worker()]. +prune_push_workers([], _Now, _Seen, Acc) -> + lists:reverse(Acc); +prune_push_workers([Worker | Rest], Now, Seen, Acc) -> + case keep_push_worker(Worker, Now, Seen) of + {keep, {_Gen, Pid, _StartedMs} = Kept} -> + prune_push_workers(Rest, Now, Seen#{Pid => true}, [Kept | Acc]); + drop -> + prune_push_workers(Rest, Now, Seen, Acc) + end. + +-spec keep_push_worker(term(), integer(), map()) -> {keep, push_worker()} | drop. +keep_push_worker({Gen, Pid, StartedMs} = Worker, Now, Seen) when + is_integer(Gen), is_pid(Pid), is_integer(StartedMs) +-> + case maps:is_key(Pid, Seen) of + true -> drop_duplicate_push_worker(); + false -> keep_live_push_worker(Worker, Now - StartedMs, local_process_alive(Pid)) + end; +keep_push_worker(_Worker, _Now, _Seen) -> + drop. + +-spec drop_duplicate_push_worker() -> drop. +drop_duplicate_push_worker() -> + count_push_event(slot_deduped), + drop. + +-spec keep_live_push_worker(push_worker(), integer(), boolean()) -> + {keep, push_worker()} | drop. +keep_live_push_worker(_Worker, _AgeMs, false) -> + drop; +keep_live_push_worker(Worker, AgeMs, true) when AgeMs < ?PUSH_WORKER_MAX_AGE_MS -> + {keep, Worker}; +keep_live_push_worker(_Worker, _AgeMs, true) -> + count_push_event(slot_reclaimed), + drop. + +-spec note_push_worker(pid()) -> ok. +note_push_worker(Pid) -> + Gen = erlang:unique_integer([monotonic, positive]), + StartedMs = erlang:monotonic_time(millisecond), + put(push_inflight_workers, [{Gen, Pid, StartedMs} | tracked_worker_list()]), + ok. + +-spec count_push_event(atom()) -> ok. +count_push_event(Key) -> + bump_push_counter(Key, true). + +-spec bump_push_counter(atom(), boolean()) -> ok. +bump_push_counter(Key, Retry) -> + try ets:update_counter(?PUSH_COUNTERS, Key, {2, 1}) of + _ -> ok + catch + error:badarg -> retry_push_counter(Key, Retry) + end. + +-spec retry_push_counter(atom(), boolean()) -> ok. +retry_push_counter(Key, true) -> + ok = ensure_push_counters(), + bump_push_counter(Key, false); +retry_push_counter(_Key, false) -> + ok. + +-spec ensure_push_counters() -> ok. +ensure_push_counters() -> + case ets:whereis(?PUSH_COUNTERS) of + undefined -> create_push_counters(); + _ -> ensure_push_counter_keys() + end. + +-spec create_push_counters() -> ok. +create_push_counters() -> + ok = guild_ets_utils:ensure_table(?PUSH_COUNTERS, push_counter_table_options()), + ensure_push_counter_keys(). + +-spec push_counter_table_options() -> [term()]. +push_counter_table_options() -> + [named_table, public, set, {read_concurrency, true}, {write_concurrency, true}]. + +-spec ensure_push_counter_keys() -> ok. +ensure_push_counter_keys() -> + lists:foreach(fun ensure_push_counter_key/1, ?PUSH_COUNTER_KEYS). + +-spec ensure_push_counter_key(atom()) -> ok. +ensure_push_counter_key(Key) -> + try + _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}), + ok + catch + error:badarg -> ok end. -spec spawn_push(event_data(), guild_id(), guild_state()) -> ok. spawn_push(FinalData, GuildId, UpdatedState) -> - PushState = #{ + Data = maps:get(data, UpdatedState, #{}), + case maps:get(members_ets, Data, undefined) of + MembersTab when is_reference(MembersTab) -> + CompactState = compact_push_state(MembersTab, Data, GuildId, UpdatedState), + spawn_compact_push(FinalData, GuildId, CompactState); + _ -> + missing_members_table(FinalData, GuildId, UpdatedState) + end. + +-spec compact_push_state(ets:tid(), map(), guild_id(), guild_state()) -> guild_state(). +compact_push_state(MembersTab, Data, GuildId, UpdatedState) -> + Sessions = maps:get(sessions, UpdatedState, #{}), + #{ + id => maps:get(id, UpdatedState, GuildId), + data => compact_push_data(Data), + virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}), + members_ets => MembersTab, + member_presence => maps:get(member_presence, UpdatedState, undefined), + session_eligibility => build_push_session_eligibility(Sessions, UpdatedState), + member_count => maps:get(member_count, UpdatedState, undefined) + }. + +-spec compact_push_data(map()) -> map(). +compact_push_data(Data) -> + maps:with( + [ + <<"guild">>, + <<"roles">>, + <<"role_index">>, + role_perms_cache, + <<"channels">>, + <<"channel_index">>, + overwrite_perms_cache, + members_ets + ], + Data + ). + +-spec spawn_compact_push(event_data(), guild_id(), guild_state()) -> ok. +spawn_compact_push(FinalData, GuildId, CompactState) -> + spawn_push_worker( + fun() -> + collect_and_send_compact_push_notifications(FinalData, GuildId, CompactState) + end, + GuildId + ). + +-spec missing_members_table(event_data(), guild_id(), guild_state()) -> ok. +missing_members_table(FinalData, GuildId, UpdatedState) -> + count_push_event(members_table_missing), + logger:warning( + "guild_push_members_table_unavailable: guild_id=~p phase=spawn", + [GuildId] + ), + spawn_legacy_push(FinalData, GuildId, UpdatedState). + +-spec spawn_legacy_push(event_data(), guild_id(), guild_state()) -> ok. +spawn_legacy_push(FinalData, GuildId, UpdatedState) -> + LegacyState = legacy_push_state(GuildId, UpdatedState), + spawn_push_worker( + fun() -> + collect_and_send_push_notifications(FinalData, GuildId, LegacyState) + end, + GuildId + ). + +-spec legacy_push_state(guild_id(), guild_state()) -> guild_state(). +legacy_push_state(GuildId, UpdatedState) -> + #{ id => maps:get(id, UpdatedState, GuildId), data => maps:get(data, UpdatedState, #{}), sessions => maps:get(sessions, UpdatedState, #{}), - virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}) - }, - Pid = spawn(fun() -> - collect_and_send_push_notifications(FinalData, GuildId, PushState) - end), - put(push_inflight, Pid), + member_presence => maps:get(member_presence, UpdatedState, undefined), + virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}), + member_count => maps:get(member_count, UpdatedState, undefined) + }. + +-spec spawn_push_worker(fun(() -> ok), guild_id()) -> ok. +spawn_push_worker(Worker, GuildId) -> + Counted = fun() -> run_counted_push_worker(Worker) end, + case try_spawn_push_worker(Counted, GuildId) of + {ok, Pid} -> + put(push_inflight, Pid), + note_push_worker(Pid), + count_push_event(worker_started); + error -> + count_push_event(spawn_failed) + end. + +-spec run_counted_push_worker(fun(() -> ok)) -> ok. +run_counted_push_worker(Worker) -> + try Worker() of + _ -> count_push_event(worker_completed) + catch + _Class:_Reason -> count_push_event(worker_failed) + end. + +-spec try_spawn_push_worker(fun(() -> ok), guild_id()) -> {ok, pid()} | error. +try_spawn_push_worker(Worker, GuildId) -> + try + {ok, spawn(fun() -> run_deprioritised_push_worker(Worker) end)} + catch + error:system_limit -> + logger:warning( + "guild_push_worker_spawn_failed: guild_id=~p reason=system_limit", + [GuildId] + ), + error + end. + +-spec run_deprioritised_push_worker(fun(() -> ok)) -> ok. +run_deprioritised_push_worker(Worker) -> + ok = apply_push_worker_priority(), + Worker(). + +-spec apply_push_worker_priority() -> ok. +apply_push_worker_priority() -> + _ = erlang:process_flag(priority, low), ok. +-spec large_guild_meta(guild_state()) -> map() | undefined. +large_guild_meta(State) -> + case maps:get(member_count, State, undefined) of + MemberCount when is_integer(MemberCount), MemberCount >= 0 -> + #{member_count => MemberCount, features => guild_features(State)}; + _ -> + undefined + end. + +-spec guild_features(guild_state()) -> [binary()]. +guild_features(State) -> + Data = maps:get(data, State, #{}), + Guild = maps:get(<<"guild">>, Data, #{}), + case maps:get(<<"features">>, Guild, []) of + Features when is_list(Features) -> Features; + _ -> [] + end. + +-spec collect_and_send_compact_push_notifications(event_data(), guild_id(), guild_state()) -> + ok. +collect_and_send_compact_push_notifications(MessageData, GuildId, State) -> + case guild_dispatch_config:should_send_push_notifications(State) of + false -> ok; + true -> send_compact_push_notifications(MessageData, GuildId, State) + end. + +-spec send_compact_push_notifications(event_data(), guild_id(), guild_state()) -> ok. +send_compact_push_notifications(MessageData, GuildId, State) -> + ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined), + case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of + undefined -> ok; + ChannelId -> scan_and_send_compact_push(MessageData, GuildId, ChannelId, State) + end. + +-spec scan_and_send_compact_push(event_data(), guild_id(), integer(), guild_state()) -> ok. +scan_and_send_compact_push(MessageData, GuildId, ChannelId, State) -> + Context = compact_scan_context(MessageData, ChannelId, State), + MembersTab = maps:get(members_ets, State), + case scan_push_members(MembersTab, Context, State) of + {ok, #{eligible_user_ids := []}} -> + ok; + {ok, Scan} -> + send_compact_scanned_push(MessageData, GuildId, Scan, State); + {error, table_unavailable} -> + scan_table_unavailable(GuildId) + end. + +-spec scan_table_unavailable(guild_id()) -> ok. +scan_table_unavailable(GuildId) -> + count_push_event(scan_table_unavailable), + logger:warning( + "guild_push_members_table_unavailable: guild_id=~p phase=scan", + [GuildId] + ), + ok. + +-spec compact_scan_context(event_data(), integer(), guild_state()) -> map(). +compact_scan_context(MessageData, ChannelId, State) -> + MentionRoles = maps:get(<<"mention_roles">>, MessageData, []), + #{ + mention_everyone => maps:get(<<"mention_everyone">>, MessageData, false) =:= true, + direct_mentions => direct_mention_id_set(MessageData), + mention_roles => mention_role_id_set(MentionRoles), + format_members => format_member_id_set(MessageData), + channel_id => ChannelId, + session_eligibility => maps:get(session_eligibility, State) + }. + +-spec scan_push_members(ets:tid(), map(), guild_state()) -> + {ok, map()} | {error, table_unavailable}. +scan_push_members(MembersTab, Context, State) -> + try + {ok, fold_push_members(member_id_snapshot(MembersTab), MembersTab, Context, State)} + catch + error:badarg -> {error, table_unavailable} + end. + +-spec initial_scan_acc() -> map(). +initial_scan_acc() -> + #{eligible_user_ids => [], user_roles => #{}, format_members => #{}}. + +-spec member_id_snapshot(ets:tid()) -> [user_id()]. +member_id_snapshot(MembersTab) -> + ets:select(MembersTab, [{{'$1', '_'}, [], ['$1']}]). + +-spec fold_push_members([user_id()], ets:tid(), map(), guild_state()) -> map(). +fold_push_members(UserIds, MembersTab, Context, State) -> + lists:foldl( + fun(UserId, Acc) -> scan_push_member_id(UserId, MembersTab, Context, State, Acc) end, + initial_scan_acc(), + UserIds + ). + +-spec scan_push_member_id(user_id(), ets:tid(), map(), guild_state(), map()) -> map(). +scan_push_member_id(UserId, MembersTab, Context, State, Acc) -> + case ets:lookup(MembersTab, UserId) of + [Row] -> scan_push_member(Row, Context, State, Acc); + _ -> Acc + end. + +-spec scan_push_member(term(), map(), guild_state(), map()) -> map(). +scan_push_member({UserId, Member}, Context, State, Acc) when + is_integer(UserId), UserId > 0, is_map(Member) +-> + Acc1 = maybe_collect_format_member(UserId, Member, Context, Acc), + case scanned_member_is_eligible(UserId, Member, Context, State) of + true -> add_scanned_member(UserId, Member, Acc1); + false -> Acc1 + end; +scan_push_member(_Row, _Context, _State, Acc) -> + Acc. + +-spec scanned_member_is_eligible(user_id(), map(), map(), guild_state()) -> boolean(). +scanned_member_is_eligible(UserId, Member, Context, State) -> + is_push_candidate(UserId, Member, Context) andalso + guild_permissions:can_view_channel( + UserId, maps:get(channel_id, Context), Member, State + ). + +-spec add_scanned_member(user_id(), map(), map()) -> map(). +add_scanned_member(UserId, Member, Acc) -> + Eligible = maps:get(eligible_user_ids, Acc), + UserRoles = maps:get(user_roles, Acc), + Acc#{ + eligible_user_ids := [UserId | Eligible], + user_roles := UserRoles#{UserId => extract_role_ids(Member)} + }. + +-spec maybe_collect_format_member(user_id(), map(), map(), map()) -> map(). +maybe_collect_format_member(UserId, Member, Context, Acc) -> + case maps:is_key(UserId, maps:get(format_members, Context)) of + true -> + Members = maps:get(format_members, Acc), + Acc#{format_members := Members#{UserId => Member}}; + false -> + Acc + end. + +-spec is_push_candidate(user_id(), map(), map()) -> boolean(). +is_push_candidate(UserId, Member, Context) -> + maps:get(UserId, maps:get(session_eligibility, Context), true) orelse + (not mentions_respect_active_session() andalso + is_mention_candidate(UserId, Member, Context)). + +-spec is_mention_candidate(user_id(), map(), map()) -> boolean(). +is_mention_candidate(UserId, Member, Context) -> + maps:get(mention_everyone, Context) orelse + maps:is_key(UserId, maps:get(direct_mentions, Context)) orelse + member_has_mentioned_role(Member, maps:get(mention_roles, Context)). + +-spec mentions_respect_active_session() -> boolean(). +mentions_respect_active_session() -> + application:get_env(fluxer_gateway, push_mentions_respect_active_session, true) =:= true. + +-spec send_compact_scanned_push(event_data(), guild_id(), map(), guild_state()) -> ok. +send_compact_scanned_push(MessageData, GuildId, Scan, State) -> + FormatData = compact_format_data(maps:get(data, State), maps:get(format_members, Scan)), + send_push_to_eligible_users( + MessageData, + GuildId, + lists:reverse(maps:get(eligible_user_ids, Scan)), + maps:get(user_roles, Scan), + maps:get(session_eligibility, State), + FormatData, + large_guild_meta(State) + ). + +-spec compact_format_data(map(), map()) -> map(). +compact_format_data(Data, FormatMembers) -> + Data#{ + <<"members">> => FormatMembers, + members_normalized => FormatMembers, + members_sorted_ids => lists:sort(maps:keys(FormatMembers)), + <<"member_role_index">> => + guild_data_index_members:build_member_role_index(FormatMembers) + }. + +-spec direct_mention_id_set(event_data()) -> map(). +direct_mention_id_set(MessageData) -> + lists:foldl( + fun add_direct_mention_id/2, + #{}, + maps:get(<<"mentions">>, MessageData, []) + ). + +-spec add_direct_mention_id(term(), map()) -> map(). +add_direct_mention_id(Mention, Acc) -> + case direct_mention_user_id(Mention) of + {true, UserId} -> Acc#{UserId => true}; + false -> Acc + end. + +-spec format_member_id_set(event_data()) -> map(). +format_member_id_set(MessageData) -> + AuthorSet = author_format_id_set(maps:get(<<"author">>, MessageData, undefined)), + collect_format_mention_ids( + maps:get(<<"mentions">>, MessageData, []), + ?MAX_FORMAT_MEMBERS, + AuthorSet + ). + +-spec author_format_id_set(term()) -> map(). +author_format_id_set(Author) when is_map(Author) -> + case direct_mention_user_id(Author) of + {true, UserId} -> #{UserId => true}; + false -> #{} + end; +author_format_id_set(_Author) -> + #{}. + +-spec collect_format_mention_ids([term()], non_neg_integer(), map()) -> map(). +collect_format_mention_ids(_Mentions, 0, Acc) -> + Acc; +collect_format_mention_ids([], _Remaining, Acc) -> + Acc; +collect_format_mention_ids([Mention | Rest], Remaining, Acc) -> + case direct_mention_user_id(Mention) of + {true, UserId} -> collect_format_mention_id(UserId, Rest, Remaining, Acc); + false -> collect_format_mention_ids(Rest, Remaining, Acc) + end. + +-spec collect_format_mention_id(user_id(), [term()], non_neg_integer(), map()) -> map(). +collect_format_mention_id(UserId, Rest, Remaining, Acc) -> + case maps:is_key(UserId, Acc) of + true -> collect_format_mention_ids(Rest, Remaining, Acc); + false -> collect_format_mention_ids(Rest, Remaining - 1, Acc#{UserId => true}) + end. + -spec collect_and_send_push_notifications(event_data(), guild_id(), guild_state()) -> ok. collect_and_send_push_notifications(MessageData, GuildId, State) -> case guild_dispatch_config:should_send_push_notifications(State) of @@ -64,7 +547,7 @@ send_push_notifications(MessageData, GuildId, State) -> Data = maps:get(data, State), Members = guild_data_index:member_map(Data), Sessions = maps:get(sessions, State, #{}), - SessionEligibility = build_push_session_eligibility(Sessions), + SessionEligibility = build_push_session_eligibility(Sessions, State), CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData), ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined), case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of @@ -114,7 +597,8 @@ send_to_eligible( EligibleUserIds, UserRolesMap, SessionEligibility, - Data + Data, + large_guild_meta(State) ) end. @@ -128,8 +612,9 @@ push_candidate_user_ids(Members, SessionEligibility, MessageData) -> ). -spec push_candidate_user_ids(term(), map(), map(), event_data()) -> [user_id()]. -push_candidate_user_ids(true, Members, _SessionEligibility, _MessageData) -> - maps:keys(Members); +push_candidate_user_ids(true, Members, SessionEligibility, _MessageData) -> + {CandidateUserIds, _Seen} = base_candidate_acc(Members, SessionEligibility), + lists:reverse(CandidateUserIds); push_candidate_user_ids(_MentionEveryone, Members, SessionEligibility, MessageData) -> BaseAcc = base_candidate_acc(Members, SessionEligibility), {CandidateUserIds, _Seen} = @@ -276,6 +761,72 @@ view_to_filtermap(UserId, ChannelId, Member, State) -> false -> false end. +-spec build_push_session_eligibility(map(), guild_state()) -> #{user_id() => boolean()}. +build_push_session_eligibility(Sessions, State) -> + case presence_eligibility_enabled() of + true -> build_push_presence_eligibility(Sessions, State); + false -> build_push_session_eligibility(Sessions) + end. + +-spec presence_eligibility_enabled() -> boolean(). +presence_eligibility_enabled() -> + application:get_env(fluxer_gateway, push_presence_eligibility, true) =:= true. + +-spec build_push_presence_eligibility(map(), guild_state()) -> #{user_id() => boolean()}. +build_push_presence_eligibility(Sessions, State) -> + Presences = maps:get(member_presence, State, undefined), + maps:fold( + fun(_Sid, Session, Acc) -> + accumulate_presence_eligibility(Session, Presences, Acc) + end, + #{}, + Sessions + ). + +-spec accumulate_presence_eligibility(map(), term(), #{user_id() => boolean()}) -> + #{user_id() => boolean()}. +accumulate_presence_eligibility(Session, Presences, Acc) -> + case maps:get(user_id, Session, undefined) of + UserId when is_integer(UserId) -> + Acc#{ + UserId => + maps:get(UserId, Acc, true) andalso + not actively_engaged(UserId, Presences) + }; + _ -> + Acc + end. + +-spec actively_engaged(user_id(), term()) -> boolean(). +actively_engaged(_UserId, undefined) -> + false; +actively_engaged(UserId, Presences) -> + case lookup_presence_safe(UserId, Presences) of + undefined -> false; + Presence -> presence_is_active(Presence) + end. + +-spec lookup_presence_safe(user_id(), term()) -> map() | undefined. +lookup_presence_safe(UserId, Presences) -> + try guild_state_member:lookup_presence(Presences, UserId) of + Presence when is_map(Presence) -> Presence; + _ -> undefined + catch + error:badarg -> undefined + end. + +-spec presence_is_active(map()) -> boolean(). +presence_is_active(Presence) -> + Status = maps:get(<<"status">>, Presence, <<"offline">>), + Afk = maps:get(<<"afk">>, Presence, false) =:= true, + Mobile = maps:get(<<"mobile">>, Presence, false) =:= true, + is_engaged_status(Status) andalso not Afk andalso not Mobile. + +-spec is_engaged_status(term()) -> boolean(). +is_engaged_status(<<"online">>) -> true; +is_engaged_status(<<"dnd">>) -> true; +is_engaged_status(_Status) -> false. + -spec build_push_session_eligibility(map()) -> #{user_id() => boolean()}. build_push_session_eligibility(Sessions) -> maps:fold( @@ -299,22 +850,23 @@ accumulate_session_eligibility(Session, Acc) -> Acc end. --spec send_push_to_eligible_users(event_data(), guild_id(), [user_id()], map(), map(), map()) -> - ok. +-spec send_push_to_eligible_users( + event_data(), guild_id(), [user_id()], map(), map(), map(), map() | undefined +) -> ok. send_push_to_eligible_users( MessageData, GuildId, EligibleUserIds, UserRolesMap, ConnectedUsers, - Data + Data, + LargeGuildMeta ) -> AuthorIdBin = maps:get(<<"id">>, maps:get(<<"author">>, MessageData, #{}), undefined), case guild_dispatch_decorate:parse_snowflake(<<"author.id">>, AuthorIdBin) of undefined -> ok; AuthorId -> - Guild = maps:get(<<"guild">>, Data), ChannelIdBin = maps:get(<<"channel_id">>, MessageData), ChannelName = find_channel_name(ChannelIdBin, Data), RoleNames = build_role_names_map(Data), @@ -324,11 +876,11 @@ send_push_to_eligible_users( EligibleUserIds, UserRolesMap, ConnectedUsers, - Guild, ChannelName, RoleNames, Data, - AuthorId + AuthorId, + LargeGuildMeta ) end. @@ -338,11 +890,11 @@ send_push_to_eligible_users( [user_id()], map(), map(), - map(), binary(), map(), map(), - integer() + integer(), + map() | undefined ) -> ok. do_send_push( MessageData, @@ -350,12 +902,13 @@ do_send_push( EligibleUserIds, UserRolesMap, ConnectedUsers, - Guild, ChannelName, RoleNames, Data, - AuthorId + AuthorId, + LargeGuildMeta ) -> + Guild = maps:get(<<"guild">>, Data), DefaultMessageNotifications = maps:get(<<"default_message_notifications">>, Guild, 0), GuildName = maps:get(<<"name">>, Guild, <<"Unknown">>), push:handle_message_create(#{ @@ -372,9 +925,19 @@ do_send_push( MessageData, GuildId, RoleNames, Data ), user_roles => UserRolesMap, - connected_users => ConnectedUsers + connected_users => ConnectedUsers, + guild_member_count => meta_member_count(LargeGuildMeta), + guild_features => meta_features(LargeGuildMeta) }). +-spec meta_member_count(map() | undefined) -> non_neg_integer() | undefined. +meta_member_count(#{member_count := MemberCount}) -> MemberCount; +meta_member_count(_Meta) -> undefined. + +-spec meta_features(map() | undefined) -> [binary()] | undefined. +meta_features(#{features := Features}) -> Features; +meta_features(_Meta) -> undefined. + -spec find_channel_name(binary(), map()) -> binary(). find_channel_name(ChannelIdBin, Data) -> case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of @@ -444,6 +1007,134 @@ collect_role_id(Role, Acc) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +presence_eligibility_suppresses_active_desktop_test() -> + Tab = ets:new(test_member_presence, [set, public]), + try + ets:insert( + Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => false}} + ), + ?assertEqual( + #{1 => false}, + build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{ + member_presence => Tab + }) + ) + after + ets:delete(Tab) + end. + +presence_eligibility_allows_mobile_only_session_test() -> + Tab = ets:new(test_member_presence, [set, public]), + try + ets:insert( + Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => true, <<"afk">> => false}} + ), + ?assertEqual( + #{1 => true}, + build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{ + member_presence => Tab + }) + ) + after + ets:delete(Tab) + end. + +presence_eligibility_allows_idle_and_afk_test() -> + Tab = ets:new(test_member_presence, [set, public]), + try + ets:insert( + Tab, {1, #{<<"status">> => <<"idle">>, <<"mobile">> => false, <<"afk">> => false}} + ), + ets:insert( + Tab, {2, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => true}} + ), + ?assertEqual( + #{1 => true, 2 => true}, + build_push_presence_eligibility( + #{<<"s1">> => #{user_id => 1}, <<"s2">> => #{user_id => 2}}, + #{member_presence => Tab} + ) + ) + after + ets:delete(Tab) + end. + +presence_eligibility_allows_unknown_presence_test() -> + Tab = ets:new(test_member_presence, [set, public]), + try + ?assertEqual( + #{7 => true}, + build_push_presence_eligibility(#{<<"s1">> => #{user_id => 7}}, #{ + member_presence => Tab + }) + ) + after + ets:delete(Tab) + end. + +presence_eligibility_allows_when_presence_table_missing_test() -> + ?assertEqual( + #{1 => true}, + build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{}) + ). + +legacy_session_eligibility_suppresses_every_real_session_test() -> + RealSession = #{ + session_id => <<"s1">>, + user_id => 1, + pid => self(), + active_guilds => sets:new(), + bot => false, + is_staff => false, + pending_connect => false, + viewable_channels => #{} + }, + ?assertEqual(#{1 => false}, build_push_session_eligibility(#{<<"s1">> => RealSession})). + +is_push_candidate_suppresses_mention_for_active_user_test() -> + Context = #{ + session_eligibility => #{1 => false}, + mention_everyone => true, + direct_mentions => #{1 => true}, + mention_roles => #{} + }, + ?assertEqual(false, is_push_candidate(1, #{}, Context)). + +is_push_candidate_allows_eligible_user_test() -> + Context = #{ + session_eligibility => #{1 => true}, + mention_everyone => false, + direct_mentions => #{}, + mention_roles => #{} + }, + ?assertEqual(true, is_push_candidate(1, #{}, Context)). + +is_push_candidate_allows_sessionless_user_test() -> + Context = #{ + session_eligibility => #{}, + mention_everyone => false, + direct_mentions => #{}, + mention_roles => #{} + }, + ?assertEqual(true, is_push_candidate(9, #{}, Context)). + +push_candidate_user_ids_everyone_respects_active_sessions_test() -> + Members = #{1 => #{}, 2 => #{}, 3 => #{}}, + ?assertEqual( + [], + push_candidate_user_ids( + Members, #{1 => false, 2 => false, 3 => false}, #{<<"mention_everyone">> => true} + ) + ), + ?assertEqual( + [1, 3], + lists:sort( + push_candidate_user_ids( + Members, #{1 => true, 2 => false}, #{<<"mention_everyone">> => true} + ) + ) + ). + build_push_session_eligibility_test() -> Sessions = #{ <<"s1">> => #{user_id => 1, mobile => false, afk => true}, @@ -477,14 +1168,6 @@ push_candidate_user_ids_includes_connected_mentioned_users_test() -> CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData), ?assertEqual([1, 3], lists:sort(CandidateUserIds)). -push_candidate_user_ids_includes_everyone_even_when_connected_test() -> - Members = #{1 => #{}, 2 => #{}, 3 => #{}}, - SessionEligibility = #{1 => false, 2 => false, 3 => false}, - CandidateUserIds = push_candidate_user_ids( - Members, SessionEligibility, #{<<"mention_everyone">> => true} - ), - ?assertEqual([1, 2, 3], lists:sort(CandidateUserIds)). - push_candidate_user_ids_deduplicates_mentions_test() -> Members = #{1 => #{<<"roles">> => [<<"10">>]}, 2 => #{<<"roles">> => []}}, SessionEligibility = #{1 => false, 2 => false}, @@ -557,13 +1240,18 @@ send_push_to_eligible_users_uses_full_data_for_channel_name_test() -> } }, ?assertEqual( - ok, send_push_to_eligible_users(MessageData, 10, [1], #{1 => []}, #{}, Data) + ok, + send_push_to_eligible_users( + MessageData, 10, [1], #{1 => []}, #{}, Data, undefined + ) ), receive {push_params, Params} -> ?assertEqual(<<"general">>, maps:get(channel_name, Params)), ?assertEqual(<<"Test Guild">>, maps:get(guild_name, Params)), - ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params)) + ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params)), + ?assertEqual(undefined, maps:get(guild_member_count, Params)), + ?assertEqual(undefined, maps:get(guild_features, Params)) after 1000 -> ?assert(false) end, @@ -602,4 +1290,518 @@ extract_role_ids_missing_key_test() -> Member = #{}, ?assertEqual([], extract_role_ids(Member)). +meta_accessors_default_to_undefined_test() -> + ?assertEqual(undefined, meta_member_count(undefined)), + ?assertEqual(undefined, meta_features(undefined)), + ?assertEqual(5, meta_member_count(#{member_count => 5, features => []})), + ?assertEqual([], meta_features(#{member_count => 5, features => []})). + +compact_push_state_drops_members_and_keeps_member_count_test() -> + Tab = ets:new(test_members, [set, public]), + try + Data = compact_test_data(Tab), + UpdatedState = #{ + id => 7, + data => Data, + sessions => #{<<"s1">> => #{user_id => 1}}, + virtual_channel_access => #{}, + member_count => 49435 + }, + Compact = compact_push_state(Tab, Data, 7, UpdatedState), + CompactData = maps:get(data, Compact), + ?assertEqual(49435, maps:get(member_count, Compact)), + ?assertEqual(Tab, maps:get(members_ets, Compact)), + ?assertNot(maps:is_key(sessions, Compact)), + ?assertEqual(#{1 => true}, maps:get(session_eligibility, Compact)), + ?assertNot(maps:is_key(<<"members">>, CompactData)), + ?assertNot(maps:is_key(members_normalized, CompactData)), + ?assertNot(maps:is_key(<<"member_role_index">>, CompactData)), + ?assertEqual( + #{member_count => 49435, features => [<<"COMMUNITY">>]}, + large_guild_meta(Compact) + ) + after + ets:delete(Tab) + end. + +legacy_push_state_carries_member_count_test() -> + UpdatedState = #{id => 7, data => #{}, sessions => #{}, member_count => 1234}, + Legacy = legacy_push_state(7, UpdatedState), + ?assertEqual(1234, maps:get(member_count, Legacy)), + ?assertEqual(#{}, maps:get(sessions, Legacy)), + ?assertEqual(#{member_count => 1234, features => []}, large_guild_meta(Legacy)). + +compact_scan_collects_eligible_users_and_format_members_test() -> + Tab = ets:new(test_members, [set, public]), + try + true = ets:insert(Tab, {1, #{<<"roles">> => []}}), + true = ets:insert(Tab, {2, #{<<"roles">> => []}}), + true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}), + State = compact_scan_state(Tab), + MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]}, + Context = compact_scan_context(MessageData, 10, State), + {ok, Scan} = scan_push_members(Tab, Context, State), + ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))), + ?assertEqual(#{3 => #{<<"roles">> => [<<"200">>]}}, maps:get(format_members, Scan)), + ?assertNot(maps:is_key(3, maps:get(user_roles, Scan))), + ?assertEqual([], maps:get(1, maps:get(user_roles, Scan))) + after + ets:delete(Tab) + end. + +compact_scan_includes_mentioned_user_when_bypass_is_re_enabled_test() -> + Tab = ets:new(test_members, [set, public]), + application:set_env(fluxer_gateway, push_mentions_respect_active_session, false), + try + true = ets:insert(Tab, {1, #{<<"roles">> => []}}), + true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}), + State = compact_scan_state(Tab), + MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]}, + Context = compact_scan_context(MessageData, 10, State), + {ok, Scan} = scan_push_members(Tab, Context, State), + ?assertEqual([1, 3], lists:sort(maps:get(eligible_user_ids, Scan))) + after + application:unset_env(fluxer_gateway, push_mentions_respect_active_session), + ets:delete(Tab) + end. + +scan_push_members_reports_unavailable_table_test() -> + Tab = ets:new(test_members, [set, public]), + State = compact_scan_state(Tab), + Context = compact_scan_context(#{}, 10, State), + true = ets:delete(Tab), + ?assertEqual({error, table_unavailable}, scan_push_members(Tab, Context, State)). + +collect_format_mention_ids_is_bounded_test() -> + Mentions = [#{<<"id">> => integer_to_binary(Id)} || Id <- lists:seq(1, 80)], + Result = collect_format_mention_ids(Mentions, ?MAX_FORMAT_MEMBERS, #{}), + ?assertEqual(?MAX_FORMAT_MEMBERS, map_size(Result)). + +format_member_id_set_includes_author_test() -> + MessageData = #{ + <<"author">> => #{<<"id">> => <<"99">>}, + <<"mentions">> => [#{<<"id">> => <<"1">>}, #{<<"id">> => <<"1">>}] + }, + ?assertEqual(#{1 => true, 99 => true}, format_member_id_set(MessageData)). + +spawn_push_without_members_table_falls_back_to_legacy_push_test() -> + reset_push_worker_state(), + try + ?assertEqual(ok, spawn_push(#{}, 7, #{id => 7, data => #{}, sessions => #{}})), + ?assert(is_pid(get(push_inflight))) + after + reset_push_worker_state() + end. + +compact_push_carries_large_guild_metadata_test() -> + Self = self(), + Tab = ets:new(test_members, [set, public]), + ok = meck:new(push, [passthrough, no_link]), + try + ok = meck:expect(push, handle_message_create, fun(Params) -> + Self ! {push_params, Params}, + ok + end), + State = compact_scan_state(Tab), + Scan = #{ + eligible_user_ids => [1], + user_roles => #{1 => []}, + format_members => #{1 => #{<<"nick">> => <<"one">>}} + }, + MessageData = #{ + <<"channel_id">> => <<"10">>, + <<"author">> => #{<<"id">> => <<"42">>} + }, + ?assertEqual(ok, send_compact_scanned_push(MessageData, 7, Scan, State)), + receive + {push_params, Params} -> + ?assertEqual(49435, maps:get(guild_member_count, Params)), + ?assertEqual([<<"COMMUNITY">>], maps:get(guild_features, Params)), + ?assertEqual([1], maps:get(user_ids, Params)), + ?assertEqual(<<"general">>, maps:get(channel_name, Params)) + after 1000 -> + ?assert(false) + end, + ?assert(meck:validate(push)) + after + meck:unload(push), + ets:delete(Tab) + end. + +compact_format_data_restricts_member_map_test() -> + Data = #{<<"guild">> => #{}, <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}}}, + FormatMembers = #{2 => #{<<"roles">> => [<<"200">>]}}, + Result = compact_format_data(Data, FormatMembers), + ?assertEqual(FormatMembers, maps:get(<<"members">>, Result)), + ?assertEqual(FormatMembers, maps:get(members_normalized, Result)), + ?assertEqual([2], maps:get(members_sorted_ids, Result)), + ?assertEqual(#{200 => #{2 => true}}, maps:get(<<"member_role_index">>, Result)). + +inflight_pid_without_a_slot_still_counts_against_the_limit_test() -> + reset_push_worker_state(), + Blocker = blocking_push_worker(), + put(push_inflight, Blocker), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1), + try + ?assertEqual([Blocker], worker_pids(live_push_workers())), + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Blocker, get(push_inflight)), + ?assertEqual([Blocker], worker_pids(get(push_inflight_workers))) + after + stop_push_worker(Blocker), + reset_push_worker_state() + end. + +unknown_worker_entries_are_dropped_test() -> + reset_push_worker_state(), + Blocker = blocking_push_worker(), + put(push_inflight_workers, [Blocker, {Blocker, 0}, not_a_worker]), + try + ?assertEqual([], live_push_workers()) + after + stop_push_worker(Blocker), + reset_push_worker_state() + end. + +remote_worker_pid_is_dropped_without_badarg_test() -> + reset_push_worker_state(), + Remote = remote_test_pid(), + put(push_inflight_workers, [{1, Remote, erlang:monotonic_time(millisecond)}]), + try + ?assertNotEqual(node(), node(Remote)), + ?assertError(badarg, erlang:is_process_alive(Remote)), + ?assertEqual(false, local_process_alive(Remote)), + ?assertEqual([], live_push_workers()) + after + reset_push_worker_state() + end. + +reused_worker_pid_consumes_only_one_slot_test() -> + reset_push_worker_state(), + Blocker = blocking_push_worker(), + Now = erlang:monotonic_time(millisecond), + put(push_inflight_workers, [{2, Blocker, Now}, {1, Blocker, Now - 10}]), + Before = read_push_counter(slot_deduped), + try + ?assertEqual([{2, Blocker, Now}], live_push_workers()), + ?assertEqual(Before + 1, read_push_counter(slot_deduped)) + after + stop_push_worker(Blocker), + reset_push_worker_state() + end. + +stale_push_worker_slot_is_reclaimed_and_counted_test() -> + reset_push_worker_state(), + Stale = blocking_push_worker(), + Aged = erlang:monotonic_time(millisecond) - ?PUSH_WORKER_MAX_AGE_MS - 1, + put(push_inflight_workers, [{1, Stale, Aged}]), + Before = read_push_counter(slot_reclaimed), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Before + 1, read_push_counter(slot_reclaimed)), + ?assertEqual(1, length(get(push_inflight_workers))), + ?assertNot(lists:member(Stale, worker_pids(get(push_inflight_workers)))) + after + stop_push_worker(Stale), + reset_push_worker_state() + end. + +push_drop_is_readable_from_named_ets_table_test() -> + reset_push_worker_state(), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1), + Blocker = blocking_push_worker(), + put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]), + Before = read_push_counter(dropped_at_limit), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)), + ?assertMatch([{dropped_at_limit, _}], ets:lookup(?PUSH_COUNTERS, dropped_at_limit)), + ?assertEqual( + lists:sort(?PUSH_COUNTER_KEYS), + lists:sort([Key || {Key, _} <- ets:tab2list(?PUSH_COUNTERS)]) + ) + after + stop_push_worker(Blocker), + reset_push_worker_state() + end. + +started_push_worker_is_counted_test() -> + reset_push_worker_state(), + Before = read_push_counter(worker_started), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Before + 1, read_push_counter(worker_started)), + ?assert(is_pid(get(push_inflight))), + ?assertEqual(1, length(get(push_inflight_workers))) + after + reset_push_worker_state() + end. + +worker_completion_is_counted_test() -> + reset_push_worker_state(), + Before = read_push_counter(worker_completed), + BeforeFailed = read_push_counter(worker_failed), + try + ?assertEqual(ok, run_counted_push_worker(fun() -> ok end)), + ?assertEqual(Before + 1, read_push_counter(worker_completed)), + ?assertEqual(BeforeFailed, read_push_counter(worker_failed)) + after + reset_push_worker_state() + end. + +worker_crash_is_counted_as_failure_not_completion_test() -> + reset_push_worker_state(), + Before = read_push_counter(worker_failed), + BeforeCompleted = read_push_counter(worker_completed), + try + ?assertEqual(ok, run_counted_push_worker(fun() -> error(boom) end)), + ?assertEqual(Before + 1, read_push_counter(worker_failed)), + ?assertEqual(BeforeCompleted, read_push_counter(worker_completed)) + after + reset_push_worker_state() + end. + +spawned_worker_reports_started_and_completed_test() -> + reset_push_worker_state(), + Self = self(), + Started = read_push_counter(worker_started), + Completed = read_push_counter(worker_completed), + try + ?assertEqual( + ok, + spawn_push_worker( + fun() -> + Self ! {ran, self()}, + ok + end, + 7 + ) + ), + Pid = get(push_inflight), + receive + {ran, Pid} -> ok + after 1000 -> ?assert(false) + end, + wait_for_counter(worker_completed, Completed + 1), + ?assertEqual(Started + 1, read_push_counter(worker_started)), + ?assertEqual(Completed + 1, read_push_counter(worker_completed)), + Messages = element(2, process_info(self(), messages)), + ?assertEqual([], [M || {'DOWN', _, process, P, _} = M <- Messages, P =:= Pid]) + after + reset_push_worker_state() + end. + +missing_members_table_is_counted_test() -> + reset_push_worker_state(), + Before = read_push_counter(members_table_missing), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Before + 1, read_push_counter(members_table_missing)) + after + reset_push_worker_state() + end. + +scan_table_unavailable_is_counted_test() -> + reset_push_worker_state(), + Tab = ets:new(test_members, [set, public]), + State = compact_scan_state(Tab), + true = ets:delete(Tab), + Before = read_push_counter(scan_table_unavailable), + try + MessageData = #{<<"channel_id">> => <<"10">>}, + ?assertEqual(ok, send_compact_push_notifications(MessageData, 7, State)), + ?assertEqual(Before + 1, read_push_counter(scan_table_unavailable)) + after + reset_push_worker_state() + end. + +bounded_scan_leaves_the_members_table_unfixed_test() -> + reset_push_worker_state(), + Tab = ets:new(test_members, [set, public]), + try + true = ets:insert(Tab, {1, #{<<"roles">> => []}}), + true = ets:insert(Tab, {2, #{<<"roles">> => []}}), + State = compact_scan_state(Tab), + Context = compact_scan_context(#{}, 10, State), + ?assertEqual(false, ets:info(Tab, fixed)), + {ok, Scan} = scan_push_members(Tab, Context, State), + ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))), + ?assertEqual(false, ets:info(Tab, fixed)), + ?assertEqual(false, ets:info(Tab, safe_fixed_monotonic_time)) + after + ets:delete(Tab), + reset_push_worker_state() + end. + +bounded_push_concurrency_spawns_second_worker_under_limit_test() -> + reset_push_worker_state(), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 3), + Blocker = blocking_push_worker(), + put(push_inflight, Blocker), + put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertNotEqual(Blocker, get(push_inflight)), + ?assertEqual(2, length(get(push_inflight_workers))), + ?assert(lists:member(Blocker, worker_pids(get(push_inflight_workers)))) + after + stop_push_worker(Blocker), + reset_push_worker_state() + end. + +bounded_push_concurrency_counts_drops_at_limit_test() -> + reset_push_worker_state(), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2), + First = blocking_push_worker(), + Second = blocking_push_worker(), + Now = erlang:monotonic_time(millisecond), + put(push_inflight, Second), + put(push_inflight_workers, [{2, Second, Now}, {1, First, Now}]), + Before = read_push_counter(dropped_at_limit), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Second, get(push_inflight)), + ?assertEqual([Second, First], worker_pids(get(push_inflight_workers))), + ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)), + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertEqual(Before + 2, read_push_counter(dropped_at_limit)), + ?assertEqual([Second, First], worker_pids(get(push_inflight_workers))) + after + stop_push_worker(First), + stop_push_worker(Second), + reset_push_worker_state() + end. + +bounded_push_concurrency_reaps_finished_workers_test() -> + reset_push_worker_state(), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1), + Finished = blocking_push_worker(), + stop_push_worker(Finished), + put(push_inflight, Finished), + put(push_inflight_workers, [{1, Finished, erlang:monotonic_time(millisecond)}]), + try + ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())), + ?assertNot(lists:member(Finished, worker_pids(get(push_inflight_workers)))) + after + reset_push_worker_state() + end. + +bounded_worker_list_stays_bounded_by_the_limit_test() -> + reset_push_worker_state(), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2), + Blockers = [blocking_push_worker() || _ <- lists:seq(1, 6)], + Now = erlang:monotonic_time(millisecond), + put(push_inflight_workers, [ + {Gen, Pid, Now} + || {Gen, Pid} <- lists:zip(lists:seq(1, 6), Blockers) + ]), + try + lists:foreach( + fun(_) -> ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())) end, + lists:seq(1, 20) + ), + ?assert(length(get(push_inflight_workers)) =< 6) + after + lists:foreach(fun stop_push_worker/1, Blockers), + reset_push_worker_state() + end. + +push_concurrency_limit_defaults_and_clamps_test() -> + reset_push_worker_state(), + try + ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 0), + ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, not_a_number), + ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1000), + ?assertEqual(?MAX_PUSH_CONCURRENCY, push_concurrency_limit()), + ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 6), + ?assertEqual(6, push_concurrency_limit()) + after + reset_push_worker_state() + end. + +legacy_test_state() -> + #{id => 7, data => #{}, sessions => #{}}. + +worker_pids(Workers) -> + [Pid || {_Gen, Pid, _StartedMs} <- Workers]. + +wait_for_counter(Key, Target) -> + wait_for_counter(Key, Target, 200). + +wait_for_counter(Key, Target, 0) -> + ?assertEqual(Target, read_push_counter(Key)); +wait_for_counter(Key, Target, Retries) -> + case read_push_counter(Key) >= Target of + true -> + ok; + false -> + timer:sleep(5), + wait_for_counter(Key, Target, Retries - 1) + end. + +remote_test_pid() -> + binary_to_term(<<131, 88, 119, 12, "fake@nowhere", 1:32, 0:32, 1:32>>). + +read_push_counter(Key) -> + try ets:lookup(?PUSH_COUNTERS, Key) of + [{Key, Value}] when is_integer(Value) -> Value; + _ -> 0 + catch + error:badarg -> 0 + end. + +blocking_push_worker() -> + spawn(fun() -> + receive + stop -> ok + end + end). + +stop_push_worker(Pid) -> + Ref = monitor(process, Pid), + Pid ! stop, + receive + {'DOWN', Ref, process, Pid, _Reason} -> ok + after 5000 -> error(push_worker_did_not_stop) + end. + +reset_push_worker_state() -> + erase(push_inflight), + erase(push_inflight_workers), + ok = application:unset_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY). + +compact_test_data(Tab) -> + #{ + <<"guild">> => #{ + <<"id">> => <<"7">>, + <<"owner_id">> => <<"999">>, + <<"features">> => [<<"COMMUNITY">>], + <<"default_message_notifications">> => 0 + }, + <<"roles">> => [], + <<"role_index">> => #{}, + <<"channels">> => [#{<<"id">> => <<"10">>, <<"name">> => <<"general">>}], + <<"channel_index">> => #{10 => #{<<"id">> => <<"10">>, <<"name">> => <<"general">>}}, + <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}}, + members_normalized => #{1 => #{}, 2 => #{}, 3 => #{}}, + <<"member_role_index">> => #{}, + members_ets => Tab + }. + +compact_scan_state(Tab) -> + #{ + id => 7, + data => compact_push_data(compact_test_data(Tab)), + virtual_channel_access => #{ + 1 => sets:from_list([10]), + 3 => sets:from_list([10]) + }, + members_ets => Tab, + session_eligibility => #{1 => true, 2 => true, 3 => false}, + member_count => 49435 + }. + -endif. diff --git a/fluxer_gateway/src/guild/guild_dispatch_send.erl b/fluxer_gateway/src/guild/guild_dispatch_send.erl index c3b535513..bf3344ebc 100644 --- a/fluxer_gateway/src/guild/guild_dispatch_send.erl +++ b/fluxer_gateway/src/guild/guild_dispatch_send.erl @@ -10,6 +10,8 @@ filter_visible_channels/4 ]). +-define(MAX_BULK_ENCODE_GROUPS, 1024). + -type event() :: atom(). -type event_data() :: map(). -type guild_state() :: map(). @@ -17,6 +19,9 @@ -type guild_id() :: integer(). -type user_id() :: integer(). -type session_pair() :: {session_id(), map()}. +-type bulk_group_key() :: [term()]. +-type bulk_groups() :: #{bulk_group_key() => {[map()], [pid()]}}. +-type bulk_acc() :: {bulk_groups(), non_neg_integer()}. -export_type([event/0, event_data/0, guild_state/0, user_id/0, session_pair/0]). -spec dispatch_to_sessions([session_pair()], event(), event_data(), guild_state()) -> @@ -45,44 +50,76 @@ dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) -> } || Ch <- BulkChannels ], - SuccessCount = lists:foldl( + {Groups, Dispatched} = lists:foldl( fun({_Sid, SessionData}, Acc) -> - dispatch_bulk_to_one_session_indexed( + collect_bulk_recipient( SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc ) end, - 0, + {#{}, 0}, FilteredSessions ), + SuccessCount = dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched), normalize_success(SuccessCount). --spec dispatch_bulk_to_one_session_indexed( +-spec collect_bulk_recipient( map(), event(), event_data(), [{integer() | undefined, map()}], guild_id(), guild_state(), - non_neg_integer() -) -> non_neg_integer(). -dispatch_bulk_to_one_session_indexed( - SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc -) -> + bulk_acc() +) -> bulk_acc(). +collect_bulk_recipient(SessionData, Event, FinalData, IndexedChannels, GuildId, State, Acc) -> Pid = maps:get(pid, SessionData), - case - session_passive:should_receive_event( - Event, FinalData, GuildId, SessionData, UpdatedState - ) - of + Eligible = + is_pid(Pid) andalso + session_passive:should_receive_event(Event, FinalData, GuildId, SessionData, State), + case Eligible of false -> Acc; true -> - FilteredChannels = filter_indexed_for_session( - SessionData, IndexedChannels, UpdatedState - ), - dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) + Filtered = filter_indexed_for_session(SessionData, IndexedChannels, State), + add_bulk_recipient(Pid, Filtered, Event, FinalData, GuildId, Acc) end. +-spec add_bulk_recipient(pid(), [map()], event(), event_data(), guild_id(), bulk_acc()) -> + bulk_acc(). +add_bulk_recipient(_Pid, [], _Event, _FinalData, _GuildId, Acc) -> + Acc; +add_bulk_recipient(Pid, FilteredChannels, Event, FinalData, GuildId, {Groups, Dispatched}) -> + Key = bulk_group_key(FilteredChannels), + case Groups of + #{Key := {Channels, Pids}} -> + {Groups#{Key := {Channels, [Pid | Pids]}}, Dispatched}; + _ when map_size(Groups) >= ?MAX_BULK_ENCODE_GROUPS -> + Sent = dispatch_bulk_to_pid( + Pid, Event, FinalData, FilteredChannels, GuildId, Dispatched + ), + {Groups, Sent}; + _ -> + {Groups#{Key => {FilteredChannels, [Pid]}}, Dispatched} + end. + +-spec bulk_group_key([map()]) -> bulk_group_key(). +bulk_group_key(FilteredChannels) -> + [maps:get(<<"id">>, Ch, undefined) || Ch <- FilteredChannels]. + +-spec dispatch_bulk_groups( + bulk_groups(), event(), event_data(), guild_id(), non_neg_integer() +) -> non_neg_integer(). +dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched) -> + maps:fold( + fun(_Key, {FilteredChannels, Pids}, Acc) -> + dispatch_bulk_to_pids( + lists:reverse(Pids), Event, FinalData, FilteredChannels, GuildId, Acc + ) + end, + Dispatched, + Groups + ). + -spec filter_indexed_for_session(map(), [{integer() | undefined, map()}], guild_state()) -> [map()]. filter_indexed_for_session(SessionData, IndexedChannels, UpdatedState) -> @@ -128,12 +165,7 @@ is_channel_visible(Channel, UserId, Member, State) -> dispatch_bulk_to_pid(_, _, _, [], _GuildId, Acc) -> Acc; dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) when is_pid(Pid) -> - CustomData = FinalData#{<<"channels">> => FilteredChannels}, - EncodedData = - {pre_encoded, - iolist_to_binary( - json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2) - )}, + EncodedData = encode_bulk_payload(FinalData, FilteredChannels), try gateway_dispatch_relay:dispatch(Pid, Event, EncodedData, GuildId), Acc + 1 @@ -143,6 +175,28 @@ dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) when dispatch_bulk_to_pid(_, _, _, _, _GuildId, Acc) -> Acc. +-spec dispatch_bulk_to_pids( + [pid()], event(), event_data(), [map()], guild_id(), non_neg_integer() +) -> non_neg_integer(). +dispatch_bulk_to_pids([], _Event, _FinalData, _FilteredChannels, _GuildId, Acc) -> + Acc; +dispatch_bulk_to_pids(Pids, Event, FinalData, FilteredChannels, GuildId, Acc) -> + EncodedData = encode_bulk_payload(FinalData, FilteredChannels), + try + gateway_dispatch_relay:dispatch_many(Pids, Event, EncodedData, GuildId), + Acc + length(Pids) + catch + _:_ -> Acc + end. + +-spec encode_bulk_payload(event_data(), [map()]) -> {pre_encoded, binary()}. +encode_bulk_payload(FinalData, FilteredChannels) -> + CustomData = FinalData#{<<"channels">> => FilteredChannels}, + {pre_encoded, + iolist_to_binary( + json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2) + )}. + -spec dispatch_standard([session_pair()], event(), event_data(), guild_id(), guild_state()) -> non_neg_integer(). dispatch_standard(FilteredSessions, Event, FinalData, GuildId, State) -> @@ -205,6 +259,8 @@ normalize_dispatched([_ | _]) -> 1. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-define(OVERFLOW_CHANNEL_BITS, 11). + normalize_success_test() -> ?assertEqual(1, normalize_success(5)), ?assertEqual(1, normalize_success(1)), @@ -287,6 +343,150 @@ passive_channel_update_bulk_dispatches_visible_channels_test() -> maps:get(<<"channels">>, Payload) ). +grouped_bulk_encoding_matches_ungrouped_test() -> + flush_dispatches(), + Data = bulk_channel_data(), + State = passive_dispatch_state(), + Sessions = [bulk_session_pair(<<"a">>, #{100 => true})], + ?assertEqual(1, reference_dispatch_bulk_update(Sessions, channel_update_bulk, Data, State)), + Ungrouped = receive_pre_encoded_binary(channel_update_bulk), + ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)), + ?assertEqual(Ungrouped, receive_pre_encoded_binary(channel_update_bulk)). + +reference_dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) -> + GuildId = maps:get(id, UpdatedState), + BulkChannels = maps:get(<<"channels">>, FinalData, []), + IndexedChannels = [ + { + guild_dispatch_decorate:parse_snowflake( + <<"id">>, + maps:get(<<"id">>, Ch, undefined) + ), + Ch + } + || Ch <- BulkChannels + ], + SuccessCount = lists:foldl( + fun({_Sid, SessionData}, Acc) -> + reference_dispatch_bulk_to_one_session_indexed( + SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc + ) + end, + 0, + FilteredSessions + ), + normalize_success(SuccessCount). + +reference_dispatch_bulk_to_one_session_indexed( + SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc +) -> + Pid = maps:get(pid, SessionData), + case + session_passive:should_receive_event( + Event, FinalData, GuildId, SessionData, UpdatedState + ) + of + false -> + Acc; + true -> + FilteredChannels = filter_indexed_for_session( + SessionData, IndexedChannels, UpdatedState + ), + dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) + end. + +grouped_bulk_encodes_once_per_visible_set_test() -> + flush_dispatches(), + Data = bulk_channel_data(), + State = passive_dispatch_state(), + Sessions = [ + bulk_session_pair(<<"a">>, #{100 => true}), + bulk_session_pair(<<"b">>, #{100 => true}), + bulk_session_pair(<<"c">>, #{200 => true}) + ], + ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)), + Bins = [receive_pre_encoded_binary(channel_update_bulk) || _ <- lists:seq(1, 3)], + ?assertEqual(2, length(lists:usort(Bins))), + Sets = [maps:get(<<"channels">>, json:decode(Bin)) || Bin <- Bins], + ?assertEqual(2, length([S || S <- Sets, S =:= [bulk_channel(<<"100">>, <<"first">>)]])), + ?assertEqual(1, length([S || S <- Sets, S =:= [bulk_channel(<<"200">>, <<"second">>)]])). + +grouped_bulk_skips_sessions_without_visible_channels_test() -> + flush_dispatches(), + Data = bulk_channel_data(), + State = passive_dispatch_state(), + Sessions = [ + bulk_session_pair(<<"a">>, #{}), + bulk_session_pair(<<"b">>, #{200 => true}) + ], + ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)), + ?assertEqual( + [bulk_channel(<<"200">>, <<"second">>)], + maps:get(<<"channels">>, receive_pre_encoded_payload(channel_update_bulk)) + ), + ?assertEqual(ok, assert_no_further_dispatch()). + +grouped_bulk_dispatches_overflow_recipients_without_retaining_them_test() -> + flush_dispatches(), + Total = ?MAX_BULK_ENCODE_GROUPS + 1, + State = passive_dispatch_state(), + Data = #{ + <<"guild_id">> => <<"42">>, + <<"channels">> => [ + bulk_channel(integer_to_binary(Id), <<"c">>) + || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS) + ] + }, + Sessions = [overflow_session_pair(N) || N <- lists:seq(1, Total)], + ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)), + Received = collect_bulk_channel_ids(Total, []), + Expected = [overflow_channel_ids(N) || N <- lists:seq(1, Total)], + ?assertEqual(lists:sort(Expected), lists:sort(Received)), + ?assertEqual(ok, assert_no_further_dispatch()). + +overflow_session_pair(N) -> + Base = passive_session_data(), + Sid = integer_to_binary(N), + Viewable = maps:from_list([{Id, true} || Id <- overflow_visible_ids(N)]), + {Sid, Base#{session_id => Sid, viewable_channels => Viewable}}. + +overflow_visible_ids(N) -> + [Id || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS), (N bsr (Id - 1)) band 1 =:= 1]. + +overflow_channel_ids(N) -> + [integer_to_binary(Id) || Id <- overflow_visible_ids(N)]. + +collect_bulk_channel_ids(0, Acc) -> + Acc; +collect_bulk_channel_ids(N, Acc) -> + Payload = receive_pre_encoded_payload(channel_update_bulk), + Ids = [maps:get(<<"id">>, Ch) || Ch <- maps:get(<<"channels">>, Payload)], + collect_bulk_channel_ids(N - 1, [Ids | Acc]). + +bulk_channel_data() -> + #{ + <<"guild_id">> => <<"42">>, + <<"channels">> => [ + bulk_channel(<<"100">>, <<"first">>), + bulk_channel(<<"200">>, <<"second">>) + ] + }. + +bulk_channel(Id, Name) -> + #{<<"id">> => Id, <<"name">> => Name}. + +bulk_session_pair(Sid, ViewableChannels) -> + Base = passive_session_data(), + {Sid, Base#{session_id => Sid, viewable_channels => ViewableChannels}}. + +assert_no_further_dispatch() -> + receive + {'$gen_cast', {dispatch, Event, _Payload}} -> + ?assert(false, {unexpected_dispatch, Event}) + after 0 -> + ok + end. + standard_dispatch_without_eligible_sessions_test() -> flush_dispatches(), Session = (passive_session_data())#{pid => undefined}, @@ -364,9 +564,12 @@ passive_dispatch_state() -> }. receive_pre_encoded_payload(Event) -> + json:decode(receive_pre_encoded_binary(Event)). + +receive_pre_encoded_binary(Event) -> receive {'$gen_cast', {dispatch, Event, {pre_encoded, Bin}}} -> - json:decode(Bin) + Bin after 1000 -> ?assert(false, {dispatch_not_received, Event}) end. diff --git a/fluxer_gateway/src/guild/guild_init.erl b/fluxer_gateway/src/guild/guild_init.erl index d1612c538..6640624ec 100644 --- a/fluxer_gateway/src/guild/guild_init.erl +++ b/fluxer_gateway/src/guild/guild_init.erl @@ -13,7 +13,11 @@ handle_reload/2 ]). +-define(MAX_RELOAD_CARRY_WARN_MEMBERS, 100000). + -type guild_state() :: map(). +-type user_id() :: integer(). +-type member() :: map(). -export_type([guild_state/0]). @@ -122,7 +126,8 @@ handle_reload(NewData, State) -> {ReloadVoiceStates, ReloadData} = extract_voice_states_from_data( NewData, ExistingVoiceStates ), - NormalizedNewData = guild_data_index:normalize_map(ReloadData), + NormalizedNewData0 = guild_data_index:normalize_map(ReloadData), + NormalizedNewData = carry_members_table(OldData, NormalizedNewData0), NewState0 = State#{voice_states => ReloadVoiceStates, data => NormalizedNewData}, NewState1 = guild_availability:handle_unavailability_transition(State, NewState0), NewState2 = guild_sessions:refresh_all_viewable_channels(NewState1), @@ -139,6 +144,89 @@ handle_reload(NewData, State) -> ok = guild_maintenance:maybe_put_permission_cache(NewState), {reply, ok, NewState}. +-spec carry_members_table(term(), term()) -> term(). +carry_members_table(OldData, NewData) when is_map(OldData), is_map(NewData) -> + carry_healthy_members_table(OldData, NewData); +carry_members_table(_OldData, NewData) -> + NewData. + +-spec carry_healthy_members_table(map(), map()) -> map(). +carry_healthy_members_table(OldData, NewData) -> + case members_ets_table(OldData) of + Tab when is_reference(Tab) -> carry_live_members_table(Tab, OldData, NewData); + undefined -> maps:remove(members_ets, NewData) + end. + +-spec members_ets_table(map()) -> ets:tid() | undefined. +members_ets_table(#{members_ets := Tab}) -> + Tab; +members_ets_table(_Data) -> + undefined. + +-spec carry_live_members_table(ets:tid(), map(), map()) -> map(). +carry_live_members_table(Tab, OldData, NewData) -> + case guild_members_table_repair:members_table_healthy(Tab) of + true -> apply_members_table_delta(Tab, OldData, NewData); + false -> maps:remove(members_ets, NewData) + end. + +-spec apply_members_table_delta(ets:tid(), map(), map()) -> map(). +apply_members_table_delta(Tab, OldData, NewData) -> + OldMap = guild_data_index_members:member_map(OldData), + NewMap = guild_data_index_members:member_map(NewData), + maybe_warn_large_carry(map_size(NewMap)), + try + ok = insert_changed_members(Tab, OldMap, NewMap), + ok = delete_removed_members(Tab, OldMap, NewMap), + NewData#{members_ets => Tab} + catch + error:badarg -> + logger:warning( + "guild_reload_members_table_carry_failed: members=~p", [map_size(NewMap)] + ), + maps:remove(members_ets, NewData) + end. + +-spec insert_changed_members(ets:tid(), map(), map()) -> ok. +insert_changed_members(Tab, OldMap, NewMap) -> + maps:foreach( + fun(UserId, Member) -> insert_changed_member(Tab, OldMap, UserId, Member) end, + NewMap + ). + +-spec insert_changed_member(ets:tid(), map(), user_id(), member()) -> ok. +insert_changed_member(Tab, OldMap, UserId, Member) -> + case maps:get(UserId, OldMap, undefined) of + Member -> ok; + _ -> insert_member_row(Tab, UserId, Member) + end. + +-spec insert_member_row(ets:tid(), user_id(), member()) -> ok. +insert_member_row(Tab, UserId, Member) -> + true = ets:insert(Tab, {UserId, Member}), + ok. + +-spec delete_removed_members(ets:tid(), map(), map()) -> ok. +delete_removed_members(Tab, OldMap, NewMap) -> + OldIds = sets:from_list(maps:keys(OldMap)), + NewIds = sets:from_list(maps:keys(NewMap)), + RemovedIds = sets:to_list(sets:subtract(OldIds, NewIds)), + lists:foreach(fun(UserId) -> delete_member_row(Tab, UserId) end, RemovedIds). + +-spec delete_member_row(ets:tid(), user_id()) -> ok. +delete_member_row(Tab, UserId) -> + true = ets:delete(Tab, UserId), + ok. + +-spec maybe_warn_large_carry(non_neg_integer()) -> ok. +maybe_warn_large_carry(Size) when Size > ?MAX_RELOAD_CARRY_WARN_MEMBERS -> + logger:warning( + "guild_reload_members_table_carry_large: members=~p threshold=~p", + [Size, ?MAX_RELOAD_CARRY_WARN_MEMBERS] + ); +maybe_warn_large_carry(_Size) -> + ok. + -spec collect_active_pid(term(), map(), [pid()]) -> [pid()]. collect_active_pid(_Sid, S, Acc) -> case maps:get(pending_connect, S, false) of @@ -168,3 +256,92 @@ maybe_index_voice_state(_, Acc) -> -spec guild_id(guild_state()) -> integer() | undefined. guild_id(State) -> snowflake_id:parse_optional(maps:get(id, State, undefined)). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +carry_members_table_carries_healthy_tid_test() -> + Tab = ets:new(carry_members, [set, public]), + try + OldMap = #{1 => carry_member(1), 2 => carry_member(2)}, + NewMap = #{1 => carry_member(1), 3 => carry_member(3)}, + seed_carry_table(Tab, OldMap), + OldData = carry_data(OldMap, Tab), + Carried = carry_members_table(OldData, carry_data(NewMap, undefined)), + ?assertEqual(Tab, maps:get(members_ets, Carried)), + ?assertEqual([1, 3], carry_table_ids(Tab)) + after + ets:delete(Tab) + end. + +carry_members_table_skips_dead_tid_test() -> + Tab = ets:new(dead_carry_members, [set, public]), + true = ets:delete(Tab), + OldData = carry_data(#{1 => carry_member(1)}, Tab), + NewData = carry_data(#{1 => carry_member(1)}, undefined), + ?assertEqual(NewData, carry_members_table(OldData, NewData)). + +carry_members_table_without_old_tid_leaves_new_data_test() -> + OldData = carry_data(#{1 => carry_member(1)}, undefined), + NewData = carry_data(#{2 => carry_member(2)}, undefined), + ?assertEqual(NewData, carry_members_table(OldData, NewData)). + +carry_members_table_delta_failure_falls_back_to_new_data_test() -> + Tab = ets:new(failing_members, [set, public]), + true = ets:delete(Tab), + OldData = carry_data(#{}, undefined), + NewData = carry_data(#{1 => carry_member(1)}, undefined), + ?assertEqual(NewData, apply_members_table_delta(Tab, OldData, NewData)). + +carry_delta_never_empties_table_test() -> + Tab = ets:new(delta_members, [set, public]), + try + OldMap = #{1 => carry_member(1), 2 => carry_member(2), 3 => carry_member(3)}, + NewMap = #{ + 1 => carry_member(1), + 2 => carry_member(2, <<"changed">>), + 4 => carry_member(4) + }, + seed_carry_table(Tab, OldMap), + ?assertEqual(3, ets:info(Tab, size)), + ok = insert_changed_members(Tab, OldMap, NewMap), + ?assertEqual(4, ets:info(Tab, size)), + ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)), + ok = delete_removed_members(Tab, OldMap, NewMap), + ?assertEqual(3, ets:info(Tab, size)), + ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)), + ?assertEqual([1, 2, 4], carry_table_ids(Tab)), + ?assertEqual([{2, carry_member(2, <<"changed">>)}], ets:lookup(Tab, 2)) + after + ets:delete(Tab) + end. + +carry_delta_skips_unchanged_members_test() -> + Tab = ets:new(unchanged_members, [set, public]), + try + MemberMap = #{1 => carry_member(1)}, + true = ets:insert(Tab, {1, sentinel}), + ok = insert_changed_members(Tab, MemberMap, MemberMap), + ?assertEqual([{1, sentinel}], ets:tab2list(Tab)) + after + ets:delete(Tab) + end. + +carry_data(MemberMap, undefined) -> + #{<<"members">> => MemberMap, members_normalized => MemberMap}; +carry_data(MemberMap, Tab) -> + #{<<"members">> => MemberMap, members_normalized => MemberMap, members_ets => Tab}. + +seed_carry_table(Tab, MemberMap) -> + maps:foreach(fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end, MemberMap). + +carry_member(UserId) -> + #{<<"user">> => #{<<"id">> => UserId}}. + +carry_table_ids(Tab) -> + lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]). + +carry_member(UserId, Nick) -> + #{<<"user">> => #{<<"id">> => UserId}, <<"nick">> => Nick}. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_maintenance.erl b/fluxer_gateway/src/guild/guild_maintenance.erl index e654c44e0..67e36d156 100644 --- a/fluxer_gateway/src/guild/guild_maintenance.erl +++ b/fluxer_gateway/src/guild/guild_maintenance.erl @@ -22,15 +22,18 @@ -type session_id() :: binary(). -type subscription_state() :: guild_subscriptions:subscription_state(). -type guild_state() :: map(). +-type viewable_memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => map()}}. +-type prune_acc() :: {subscription_state(), #{user_id() => non_neg_integer()}, viewable_memo()}. -spec update_counts(guild_state()) -> guild_state(). update_counts(State) -> - Data = maps:get(data, State, #{}), + State0 = guild_members_table_repair:maybe_repair(State), + Data = maps:get(data, State0, #{}), MemberCount = guild_data_index:member_count(Data), - OnlineCount = guild_member_list:get_online_count(State), - PublicOnlineCount = guild_public_online:compute_count(State), - ok = maybe_put_guild_count_cache(State, MemberCount, PublicOnlineCount), - State#{ + OnlineCount = guild_member_list:get_online_count(State0), + PublicOnlineCount = guild_public_online:compute_count(State0), + ok = maybe_put_guild_count_cache(State0, MemberCount, PublicOnlineCount), + State0#{ member_count => MemberCount, online_count => OnlineCount, public_online_count => PublicOnlineCount @@ -160,30 +163,36 @@ require_subscription_state(MemberSubs) when is_map(MemberSubs) -> -spec prune_member_subscription_map(subscription_state(), map(), guild_state()) -> {subscription_state(), #{user_id() => non_neg_integer()}}. +prune_member_subscription_map(MemberSubs, _Sessions, _State) when map_size(MemberSubs) =:= 0 -> + {MemberSubs, #{}}; prune_member_subscription_map(MemberSubs, Sessions, State) -> - maps:fold( + {NewMemberSubs, Counts, _Memo} = maps:fold( fun(MemberId, Subscribers, Acc) -> prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc) end, - {MemberSubs, #{}}, + {MemberSubs, #{}, new_viewable_memo(State)}, MemberSubs - ). + ), + {NewMemberSubs, Counts}. -spec prune_member_subscribers( term(), sets:set(session_id()), map(), guild_state(), - {subscription_state(), #{user_id() => non_neg_integer()}} -) -> {subscription_state(), #{user_id() => non_neg_integer()}}. -prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc) when + prune_acc() +) -> prune_acc(). +prune_member_subscribers(MemberId, Subscribers, Sessions, State, {Subs, Counts, Memo}) when is_integer(MemberId) -> - MemberViewable = member_viewable_channel_map(MemberId, State), + {MemberViewable, Memo1} = memoised_member_viewable_channel_map(MemberId, State, Memo), {KeptSubscribers, RemovedCount} = prune_subscriber_set( Subscribers, Sessions, MemberViewable, State ), - update_pruned_member_subscription(MemberId, KeptSubscribers, RemovedCount, Acc); + {NewSubs, NewCounts} = update_pruned_member_subscription( + MemberId, KeptSubscribers, RemovedCount, {Subs, Counts} + ), + {NewSubs, NewCounts, Memo1}; prune_member_subscribers(_MemberId, _Subscribers, _Sessions, _State, Acc) -> Acc. @@ -204,24 +213,152 @@ member_viewable_channel_map(MemberId, State) -> guild_visibility:get_user_viewable_channels(MemberId, State) ). +%% One pass reads a fixed state, and get_user_viewable_channels/2 reads nothing from a +%% member except its raw roles term, so two members holding the same term see the same +%% channels. The user-specific inputs are the guild owner, virtual channel access and +%% user-type permission overwrites, and every user named by one of those bypasses the +%% memo. A term is still computed the first time the pass meets it, so a term whose +%% computation raises raises on the same member it does without the memo. +-spec new_viewable_memo(guild_state()) -> viewable_memo(). +new_viewable_memo(State) -> + #{exceptions => viewable_exceptions(State), cache => #{}}. + +-spec memoised_member_viewable_channel_map(user_id(), guild_state(), viewable_memo()) -> + {map(), viewable_memo()}. +memoised_member_viewable_channel_map(MemberId, State, Memo) -> + #{exceptions := Exceptions, cache := Cache} = Memo, + case memo_key(MemberId, Exceptions, State) of + bypass -> + {member_viewable_channel_map(MemberId, State), Memo}; + {ok, RawRoles} -> + cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo) + end. + +-spec cached_member_viewable_channel_map( + user_id(), term(), guild_state(), #{term() => map()}, viewable_memo() +) -> {map(), viewable_memo()}. +cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo) -> + case maps:find(RawRoles, Cache) of + {ok, Viewable} -> + {Viewable, Memo}; + error -> + Viewable = member_viewable_channel_map(MemberId, State), + {Viewable, Memo#{cache := Cache#{RawRoles => Viewable}}} + end. + +-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass. +memo_key(MemberId, Exceptions, State) -> + case sets:is_element(MemberId, Exceptions) of + true -> bypass; + false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State)) + end. + +-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass. +member_roles_key(Member) when is_map(Member) -> + {ok, maps:get(<<"roles">>, Member, [])}; +member_roles_key(_Member) -> + bypass. + +-spec viewable_exceptions(guild_state()) -> sets:set(user_id()). +viewable_exceptions(State) -> + Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})), + Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})), + Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)), + Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))), + sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)). + +%% parse_maybe/1, not parse_optional/1: an id this rejects makes the permission check +%% itself raise before it can match a user, so leaving it out keeps the memo exact. +-spec owner_ids(term()) -> [user_id()]. +owner_ids(OwnerIdRaw) -> + case snowflake_id:parse_maybe(OwnerIdRaw) of + OwnerId when is_integer(OwnerId) -> [OwnerId]; + _ -> [] + end. + +%% Both places a permission check can take overwrites from, and role targets alongside +%% user targets. Role ids can never collide with user ids, so the superset only costs a +%% few members their memo entry. +-spec overwrite_target_ids(map()) -> [user_id()]. +overwrite_target_ids(Data) -> + Cached = maps:fold( + fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end, + [], + map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{})) + ), + maps:fold( + fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end, + Cached, + guild_data_index:channel_index(Data) + ). + +-spec cached_target_ids(term(), [user_id()]) -> [user_id()]. +cached_target_ids(Entries, Acc) when is_list(Entries) -> + [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc; +cached_target_ids(_Entries, Acc) -> + Acc. + +-spec channel_target_ids(term(), [user_id()]) -> [user_id()]. +channel_target_ids(Channel, Acc) when is_map(Channel) -> + lists:foldl( + fun overwrite_target_id/2, + Acc, + map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, [])) + ); +channel_target_ids(_Channel, Acc) -> + Acc. + +-spec overwrite_target_id(term(), [user_id()]) -> [user_id()]. +overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of + Id when is_integer(Id) -> [Id | Acc]; + _ -> Acc + end; +overwrite_target_id(_Overwrite, Acc) -> + Acc. + +%% update_pruned_member_subscription/4 drops the whole result when nothing was removed, +%% so the kept set is only ever built when Removed is non-empty, and then by the same +%% fold over the same subscribers as before. -spec prune_subscriber_set(sets:set(session_id()), map(), map(), guild_state()) -> {sets:set(session_id()), non_neg_integer()}. prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) -> - sets:fold( + Removed = sets:fold( fun(SessionId, Acc) -> - prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc) + collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Acc) end, - {sets:new(), 0}, + #{}, Subscribers - ). + ), + keep_remaining_subscribers(Subscribers, Removed). --spec prune_subscriber( - session_id(), map(), map(), guild_state(), {sets:set(session_id()), non_neg_integer()} -) -> {sets:set(session_id()), non_neg_integer()}. -prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) -> +-spec collect_removed_subscriber( + session_id(), map(), map(), guild_state(), #{session_id() => true} +) -> #{session_id() => true}. +collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Removed) -> case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of - true -> {sets:add_element(SessionId, Kept), RemovedCount}; - false -> {Kept, RemovedCount + 1} + true -> Removed; + false -> Removed#{SessionId => true} + end. + +-spec keep_remaining_subscribers(sets:set(session_id()), #{session_id() => true}) -> + {sets:set(session_id()), non_neg_integer()}. +keep_remaining_subscribers(Subscribers, Removed) when map_size(Removed) =:= 0 -> + {Subscribers, 0}; +keep_remaining_subscribers(Subscribers, Removed) -> + Kept = sets:fold( + fun(SessionId, Acc) -> add_kept_subscriber(SessionId, Removed, Acc) end, + sets:new(), + Subscribers + ), + {Kept, map_size(Removed)}. + +-spec add_kept_subscriber(session_id(), #{session_id() => true}, sets:set(session_id())) -> + sets:set(session_id()). +add_kept_subscriber(SessionId, Removed, Kept) -> + case maps:is_key(SessionId, Removed) of + true -> Kept; + false -> sets:add_element(SessionId, Kept) end. -spec subscriber_can_still_view_member(session_id(), map(), map(), guild_state()) -> @@ -399,8 +536,8 @@ prune_test_state() -> 30 => prune_member(30, [OtherRole]) }, <<"channels">> => [ - prune_channel(ChannelA, ViewerRole), - prune_channel(ChannelB, OtherRole) + prune_channel(ChannelA, [prune_overwrite(ViewerRole, 0)]), + prune_channel(ChannelB, [prune_overwrite(OtherRole, 0)]) ] } }. @@ -417,20 +554,186 @@ prune_member(UserId, RoleIds) -> <<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds] }. -prune_channel(ChannelId, RoleId) -> - ViewPerm = constants:view_channel_permission(), +update_counts_repairs_missing_members_table_test() -> + ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true), + try + Result = update_counts(repair_counts_state()), + Tab = maps:get(members_ets, maps:get(data, Result)), + ?assert(is_reference(Tab)), + ?assertNot(maps:is_key(members_ets_repair, Result)), + ?assertEqual([1, 2], lists:sort([Id || {Id, _} <- ets:tab2list(Tab)])), + ?assertEqual(2, maps:get(member_count, Result)), + ets:delete(Tab) + after + application:unset_env(fluxer_gateway, guild_members_table_self_heal) + end. + +repair_counts_state() -> + MemberMap = #{ + 1 => #{<<"user">> => #{<<"id">> => 1}}, + 2 => #{<<"user">> => #{<<"id">> => 2}} + }, #{ - <<"id">> => integer_to_binary(ChannelId), - <<"permission_overwrites">> => [ - #{ - <<"id">> => integer_to_binary(RoleId), - <<"type">> => 0, - <<"allow">> => integer_to_binary(ViewPerm), - <<"deny">> => <<"0">> - } - ] + id => 42, + disable_guild_count_cache_updates => true, + data => #{ + <<"channels">> => [], + <<"members">> => MemberMap, + members_normalized => MemberMap + } }. +prune_channel(ChannelId, Overwrites) -> + #{ + <<"id">> => integer_to_binary(ChannelId), + <<"permission_overwrites">> => Overwrites + }. + +prune_overwrite(TargetId, Type) -> + #{ + <<"id">> => integer_to_binary(TargetId), + <<"type">> => Type, + <<"allow">> => integer_to_binary(constants:view_channel_permission()), + <<"deny">> => <<"0">> + }. + +%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel 600 +%% through it, but 99 holds virtual access to channel 500 and 4242 is a user-overwrite +%% target on 500, so neither may be answered from that term. Member 7 owns the guild, +%% 777 is subscribed without being a member, and session s3 has no cached viewable map. +memo_prune_state() -> + GuildId = 42, + ViewerRole = 1000, + OtherRole = 2000, + Members = maps:from_list( + [{7, prune_member(7, [])}] ++ + [{Id, prune_member(Id, [ViewerRole])} || Id <- [10, 20, 21]] ++ + [{Id, prune_member(Id, [OtherRole])} || Id <- [30, 31, 99, 4242]] + ), + #{ + id => GuildId, + virtual_channel_access => #{99 => sets:from_list([500])}, + sessions => #{ + <<"s1">> => #{user_id => 10, viewable_channels => #{500 => true}}, + <<"s2">> => #{user_id => 30, viewable_channels => #{600 => true}}, + <<"s3">> => #{user_id => 10} + }, + member_subscriptions => memo_prune_subscriptions(), + presence_subscriptions => maps:from_list( + [{Id, 3} || Id <- [7, 20, 21, 30, 31, 99, 4242, 777]] + ), + data => #{ + <<"guild">> => #{<<"owner_id">> => <<"7">>}, + <<"roles">> => [ + prune_role(GuildId, 0), + prune_role(ViewerRole, 0), + prune_role(OtherRole, 0) + ], + <<"members">> => Members, + <<"channels">> => [ + prune_channel(500, [prune_overwrite(ViewerRole, 0), prune_overwrite(4242, 1)]), + prune_channel(600, [prune_overwrite(OtherRole, 0)]) + ] + } + }. + +memo_prune_subscriptions() -> + lists:foldl( + fun({SessionId, MemberIds}, Acc) -> + guild_subscriptions:update_subscriptions(SessionId, MemberIds, Acc) + end, + guild_subscriptions:init_state(), + [ + {<<"s1">>, [20, 21, 30, 31, 7, 99, 4242, 777]}, + {<<"s2">>, [20, 30]}, + {<<"s3">>, [20, 30]}, + {<<"missing">>, [31]} + ] + ). + +%% prune_member_subscription_map/3 as it read before the memo: every subscribed member +%% materialises its own viewable channel map and every subscriber set is rebuilt. +reference_prune_member_subscription_map(MemberSubs, Sessions, State) -> + maps:fold( + fun(MemberId, Subscribers, Acc) -> + reference_prune_member(MemberId, Subscribers, Sessions, State, Acc) + end, + {MemberSubs, #{}}, + MemberSubs + ). + +reference_prune_member(MemberId, Subscribers, Sessions, State, Acc) when is_integer(MemberId) -> + MemberViewable = member_viewable_channel_map(MemberId, State), + {Kept, RemovedCount} = reference_prune_subscriber_set( + Subscribers, Sessions, MemberViewable, State + ), + update_pruned_member_subscription(MemberId, Kept, RemovedCount, Acc); +reference_prune_member(_MemberId, _Subscribers, _Sessions, _State, Acc) -> + Acc. + +reference_prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) -> + sets:fold( + fun(SessionId, Acc) -> + reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc) + end, + {sets:new(), 0}, + Subscribers + ). + +reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) -> + case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of + true -> {sets:add_element(SessionId, Kept), RemovedCount}; + false -> {Kept, RemovedCount + 1} + end. + +prune_member_subscription_map_matches_unmemoised_reference_test() -> + State = memo_prune_state(), + MemberSubs = member_subscriptions(State), + Sessions = maps:get(sessions, State), + ?assertEqual( + reference_prune_member_subscription_map(MemberSubs, Sessions, State), + prune_member_subscription_map(MemberSubs, Sessions, State) + ). + +prune_keeps_exception_members_a_session_can_still_see_test() -> + Result = maybe_prune_invalid_member_subscriptions(guild_role_update, memo_prune_state()), + MemberSubs = maps:get(member_subscriptions, Result), + Presence = #{7 => 3, 20 => 2, 21 => 3, 30 => 1, 31 => 1, 99 => 3, 4242 => 3, 777 => 2}, + ?assertEqual( + [[<<"s1">>], [<<"s1">>], [<<"s1">>], [], []], + [ + lists:sort(guild_subscriptions:get_subscribed_sessions(Id, MemberSubs)) + || Id <- [7, 99, 4242, 31, 777] + ] + ), + ?assertEqual(Presence, maps:get(presence_subscriptions, Result)). + +memo_reuses_one_entry_per_roles_term_test() -> + State = memo_prune_state(), + {First, Memo1} = memoised_member_viewable_channel_map(20, State, new_viewable_memo(State)), + {Second, Memo2} = memoised_member_viewable_channel_map(21, State, Memo1), + ?assertEqual(member_viewable_channel_map(21, State), Second), + ?assertEqual(First, Second), + ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]). + +memo_never_answers_owner_virtual_or_overwrite_targets_from_the_cache_test() -> + State = memo_prune_state(), + {_, Memo} = memoised_member_viewable_channel_map(30, State, new_viewable_memo(State)), + Answers = [memoised_member_viewable_channel_map(Id, State, Memo) || Id <- [7, 99, 4242]], + ?assertEqual( + [member_viewable_channel_map(Id, State) || Id <- [7, 99, 4242]], + [Viewable || {Viewable, _} <- Answers] + ), + ?assertEqual([Memo, Memo, Memo], [M || {_, M} <- Answers]). + +prune_subscriber_set_returns_the_original_set_when_nothing_is_removed_test() -> + State = memo_prune_state(), + Subscribers = sets:from_list([<<"s1">>, <<"s3">>]), + Viewable = member_viewable_channel_map(20, State), + ?assertEqual( + {Subscribers, 0}, + prune_subscriber_set(Subscribers, maps:get(sessions, State), Viewable, State) + ). permission_cache_skips_nick_only_member_update_test() -> UserId = 4242, OldState = permission_cache_state( diff --git a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl index 8fb67b034..f18f3fec0 100644 --- a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl +++ b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl @@ -23,6 +23,8 @@ -type guild_state() :: map(). -type list_id() :: binary(). +-type visibility_memo() :: disabled | #{exceptions := sets:set(integer()), cache := map()}. +-type role_cache() :: #{term() => {[integer()], [integer()]}}. -type engine_ref() :: ets:table(). -export_type([guild_state/0, list_id/0, engine_ref/0]). @@ -244,13 +246,21 @@ build_inputs(ChannelId, State) -> MemberMap = guild_data_index:member_map(Data), ConnectedUserIds = guild_member_list_common:connected_session_user_ids(State), PresenceTab = maps:get(member_presence, State), - Tuples = maps:fold( - fun(UserId, Member, Acc) -> + {Tuples, _Memo, _RoleCache} = maps:fold( + fun(UserId, Member, {Acc, Memo, RoleCache}) -> maybe_prepare_visible_member_tuple( - UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc + UserId, + Member, + ChannelId, + State, + PresenceTab, + ConnectedUserIds, + Acc, + Memo, + RoleCache ) end, - [], + {[], new_visibility_memo(ChannelId, Data, State), #{}}, MemberMap ), Roles = map_utils:ensure_list(maps:get(<<"roles">>, Data, [])), @@ -264,30 +274,165 @@ build_inputs(ChannelId, State) -> {Tuples, HoistedRoleIds}. -spec maybe_prepare_visible_member_tuple( - integer(), map(), pos_integer(), guild_state(), ets:tid() | map(), sets:set(integer()), [ - guild_member_list_store:member_tuple() - ] -) -> [guild_member_list_store:member_tuple()]. + integer(), + map(), + pos_integer(), + guild_state(), + ets:tid() | map(), + sets:set(integer()), + [guild_member_list_store:member_tuple()], + visibility_memo(), + role_cache() +) -> {[guild_member_list_store:member_tuple()], visibility_memo(), role_cache()}. maybe_prepare_visible_member_tuple( - UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc + UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc, Memo, RoleCache ) -> - case can_view(UserId, ChannelId, Member, State) of - true -> prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc); - false -> Acc + {RoleIds, SortedRoleIds, RoleCache1} = member_role_ids(Member, RoleCache), + {Visible, Memo1} = memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo), + case Visible of + true -> + { + prepare_member_tuple( + UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc + ), + Memo1, + RoleCache1 + }; + false -> + {Acc, Memo1, RoleCache1} end. --spec prepare_member_tuple(integer(), map(), ets:tid() | map(), sets:set(integer()), [ - guild_member_list_store:member_tuple() -]) -> [guild_member_list_store:member_tuple()]. -prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc) -> +%% extract_role_ids/1 and the sort that forms the visibility memo key are both +%% pure functions of the member's raw roles term, so caching on that term +%% collapses them to the number of distinct role combinations. +-spec member_role_ids(map(), role_cache()) -> {[integer()], [integer()], role_cache()}. +member_role_ids(Member, RoleCache) -> + RawRoles = maps:get(<<"roles">>, Member, []), + case maps:find(RawRoles, RoleCache) of + {ok, {RoleIds, SortedRoleIds}} -> + {RoleIds, SortedRoleIds, RoleCache}; + error -> + RoleIds = guild_member_list_store:extract_role_ids(Member), + SortedRoleIds = lists:sort(RoleIds), + {RoleIds, SortedRoleIds, RoleCache#{RawRoles => {RoleIds, SortedRoleIds}}} + end. + +%% can_view_channel/4 reads nothing from the member except its role ids, so for a +%% non-category channel its result is a function of the role set. The only +%% user-specific inputs are the guild owner, virtual channel access, and the +%% channel's own permission overwrites, and those users bypass the memo. +-spec new_visibility_memo(pos_integer(), map(), guild_state()) -> visibility_memo(). +new_visibility_memo(ChannelId, Data, State) -> + case memoisable_channel(ChannelId, Data) of + true -> + #{ + exceptions => visibility_exceptions(ChannelId, Data, State), + cache => #{} + }; + false -> + disabled + end. + +-spec memoised_can_view( + integer(), pos_integer(), map(), [integer()], guild_state(), visibility_memo() +) -> {boolean(), visibility_memo()}. +memoised_can_view(UserId, ChannelId, Member, _SortedRoleIds, State, disabled) -> + {can_view(UserId, ChannelId, Member, State), disabled}; +memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo) -> + #{exceptions := Exceptions, cache := Cache} = Memo, + case sets:is_element(UserId, Exceptions) of + true -> + {can_view(UserId, ChannelId, Member, State), Memo}; + false -> + case maps:find(SortedRoleIds, Cache) of + {ok, Visible} -> + {Visible, Memo}; + error -> + Visible = can_view(UserId, ChannelId, Member, State), + {Visible, Memo#{cache := Cache#{SortedRoleIds => Visible}}} + end + end. + +%% Category visibility recurses into child channels, each with its own +%% overwrites, so the exception set is not local to this channel. Do not memoise. +-spec memoisable_channel(pos_integer(), map()) -> boolean(). +memoisable_channel(ChannelId, Data) -> + case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of + Channel when is_map(Channel) -> maps:get(<<"type">>, Channel, undefined) =/= 4; + _ -> false + end. + +-spec visibility_exceptions(pos_integer(), map(), guild_state()) -> sets:set(integer()). +visibility_exceptions(ChannelId, Data, State) -> + Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})), + Owner = + case snowflake_id:parse_optional(maps:get(<<"owner_id">>, Guild, undefined)) of + OwnerId when is_integer(OwnerId) -> [OwnerId]; + _ -> [] + end, + Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))), + sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(ChannelId, Data)). + +%% Every overwrite target on the channel, role and user alike. Role ids can never +%% collide with user ids, so the superset costs only a few direct computations. +-spec overwrite_target_ids(pos_integer(), map()) -> [integer()]. +overwrite_target_ids(ChannelId, Data) -> + Cache = map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{})), + case maps:get(ChannelId, Cache, undefined) of + Cached when is_list(Cached) -> + [Id || {Id, _Type, _Allow, _Deny} <- Cached, is_integer(Id)]; + _ -> + channel_overwrite_target_ids(ChannelId, Data) + end. + +-spec channel_overwrite_target_ids(pos_integer(), map()) -> [integer()]. +channel_overwrite_target_ids(ChannelId, Data) -> + case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of + Channel when is_map(Channel) -> + lists:filtermap( + fun(Overwrite) -> overwrite_target_id(Overwrite) end, + map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, [])) + ); + _ -> + [] + end. + +-spec overwrite_target_id(term()) -> {true, integer()} | false. +overwrite_target_id(Overwrite) when is_map(Overwrite) -> + case snowflake_id:parse_optional(maps:get(<<"id">>, Overwrite, undefined)) of + Id when is_integer(Id) -> {true, Id}; + _ -> false + end; +overwrite_target_id(_Overwrite) -> + false. + +-spec prepare_member_tuple( + integer(), + map(), + ets:tid() | map(), + sets:set(integer()), + [integer()], + [guild_member_list_store:member_tuple()] +) -> [guild_member_list_store:member_tuple()]. +prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc) -> DisplayName = guild_member_list_common:get_member_display_name(Member), SortKey = guild_member_list_common:casefold_binary(DisplayName), - RoleIds = guild_member_list_store:extract_role_ids(Member), + IsOnline = member_is_online(UserId, PresenceTab, ConnectedUserIds), + [{UserId, SortKey, RoleIds, IsOnline} | Acc]. + +%% Presence is consulted only to decide IsOnline, and IsOnline is false for anyone +%% without a connected session, so the ETS read and its default-presence +%% allocation are skipped for every member who is not connected. +-spec member_is_online(integer(), ets:tid() | map(), sets:set(integer())) -> boolean(). +member_is_online(UserId, PresenceTab, ConnectedUserIds) -> + sets:is_element(UserId, ConnectedUserIds) andalso + presence_status_is_online(UserId, PresenceTab). + +-spec presence_status_is_online(integer(), ets:tid() | map()) -> boolean(). +presence_status_is_online(UserId, PresenceTab) -> Presence = guild_state_member:lookup_presence(PresenceTab, UserId), Status = maps:get(<<"status">>, Presence, <<"offline">>), - IsConnected = sets:is_element(UserId, ConnectedUserIds), - IsOnline = IsConnected andalso Status =/= <<"offline">> andalso Status =/= <<"invisible">>, - [{UserId, SortKey, RoleIds, IsOnline} | Acc]. + Status =/= <<"offline">> andalso Status =/= <<"invisible">>. -spec can_view(integer(), pos_integer(), map(), guild_state()) -> boolean(). can_view(UserId, ChannelId, Member, State) when is_integer(UserId), UserId > 0 -> @@ -346,4 +491,91 @@ sync_online_noop_without_engines_test() -> ok, sync_online(1, true, #{data => #{<<"guild">> => #{<<"features">> => []}}}) ). +memo_channel_state() -> + #{ + data => #{ + <<"guild">> => #{<<"features">> => [], <<"owner_id">> => <<"7">>}, + <<"channel_index">> => #{ + 500 => #{ + <<"id">> => <<"500">>, + <<"type">> => 0, + <<"permission_overwrites">> => [#{<<"id">> => <<"42">>, <<"type">> => 1}] + }, + 600 => #{<<"id">> => <<"600">>, <<"type">> => 4} + } + }, + virtual_channel_access => #{99 => sets:from_list([500])} + }. + +memoisable_channel_skips_categories_test() -> + Data = maps:get(data, memo_channel_state()), + ?assert(memoisable_channel(500, Data)), + ?assertNot(memoisable_channel(600, Data)), + ?assertNot(memoisable_channel(777, Data)). + +visibility_exceptions_cover_owner_virtual_and_overwrites_test() -> + State = memo_channel_state(), + Exceptions = visibility_exceptions(500, maps:get(data, State), State), + ?assert(sets:is_element(7, Exceptions)), + ?assert(sets:is_element(99, Exceptions)), + ?assert(sets:is_element(42, Exceptions)), + ?assertNot(sets:is_element(1234, Exceptions)). + +new_visibility_memo_is_disabled_for_categories_test() -> + State = memo_channel_state(), + ?assertEqual(disabled, new_visibility_memo(600, maps:get(data, State), State)), + ?assertNotEqual(disabled, new_visibility_memo(500, maps:get(data, State), State)). + +memoised_can_view_caches_one_entry_per_role_set_test() -> + State = memo_channel_state(), + Memo0 = new_visibility_memo(500, maps:get(data, State), State), + {First, Memo1} = memoised_can_view( + 1000, 500, #{<<"roles">> => [2, 1]}, lists:sort([2, 1]), State, Memo0 + ), + ?assertEqual(1, map_size(maps:get(cache, Memo1))), + {Second, Memo2} = memoised_can_view( + 1001, 500, #{<<"roles">> => [1, 2]}, lists:sort([1, 2]), State, Memo1 + ), + ?assertEqual(First, Second), + ?assertEqual(1, map_size(maps:get(cache, Memo2))). + +memoised_can_view_bypasses_exception_users_test() -> + State = memo_channel_state(), + Memo0 = new_visibility_memo(500, maps:get(data, State), State), + {_, Memo1} = memoised_can_view(7, 500, #{<<"roles">> => []}, [], State, Memo0), + ?assertEqual(0, map_size(maps:get(cache, Memo1))), + {_, Memo2} = memoised_can_view(42, 500, #{<<"roles">> => []}, [], State, Memo1), + ?assertEqual(0, map_size(maps:get(cache, Memo2))). + +memoised_can_view_disabled_never_caches_test() -> + State = memo_channel_state(), + ?assertMatch( + {_, disabled}, + memoised_can_view(1000, 600, #{<<"roles">> => [1]}, [1], State, disabled) + ). + +member_role_ids_caches_and_sorts_test() -> + Member = #{<<"roles">> => [3, 1, 2]}, + {RoleIds, Sorted, Cache1} = member_role_ids(Member, #{}), + ?assertEqual([1, 2, 3], Sorted), + ?assertEqual(lists:sort(RoleIds), Sorted), + ?assertEqual(1, map_size(Cache1)), + {RoleIds2, Sorted2, Cache2} = member_role_ids(#{<<"roles">> => [3, 1, 2]}, Cache1), + ?assertEqual({RoleIds, Sorted}, {RoleIds2, Sorted2}), + ?assertEqual(1, map_size(Cache2)), + {_, _, Cache3} = member_role_ids(#{<<"roles">> => [9]}, Cache2), + ?assertEqual(2, map_size(Cache3)). + +offline_member_skips_the_presence_lookup_test() -> + Presences = #{7 => #{<<"status">> => <<"online">>}}, + Connected = sets:from_list([7]), + ?assert(member_is_online(7, Presences, Connected)), + ?assertNot(member_is_online(8, Presences, Connected)), + ?assertNot(member_is_online(7, Presences, sets:new())). + +invisible_member_is_not_online_test() -> + Connected = sets:from_list([7]), + ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"invisible">>}}, Connected)), + ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"offline">>}}, Connected)). + -endif. diff --git a/fluxer_gateway/src/guild/guild_member_list_common.erl b/fluxer_gateway/src/guild/guild_member_list_common.erl index 6da4efe9f..75800e79e 100644 --- a/fluxer_gateway/src/guild/guild_member_list_common.erl +++ b/fluxer_gateway/src/guild/guild_member_list_common.erl @@ -65,11 +65,45 @@ get_member_sort_key(Member) -> -spec casefold_binary(term()) -> binary(). casefold_binary(Value) -> Bin = normalize_name(Value), - case unicode_chardata_to_binary(string:casefold(Bin)) of - {ok, Folded} -> Folded; - error -> Bin + case ascii_casefold(Bin) of + {ok, Folded} -> + Folded; + not_ascii -> + case unicode_chardata_to_binary(string:casefold(Bin)) of + {ok, Folded} -> Folded; + error -> Bin + end end. +%% Case folding a pure-ASCII binary is exactly A-Z -> a-z, so a byte pass is +%% equivalent to string:casefold/1 and skips two full unicode traversals per +%% member. Anything with a byte above 127 falls through to the unicode path. +-spec ascii_casefold(binary()) -> {ok, binary()} | not_ascii. +ascii_casefold(Bin) -> + case scan_ascii(Bin, false) of + not_ascii -> not_ascii; + false -> {ok, Bin}; + true -> {ok, lower_ascii(Bin, <<>>)} + end. + +-spec scan_ascii(binary(), boolean()) -> boolean() | not_ascii. +scan_ascii(<<>>, HasUpper) -> + HasUpper; +scan_ascii(<>, _HasUpper) when C >= $A, C =< $Z -> + scan_ascii(Rest, true); +scan_ascii(<>, HasUpper) when C < 128 -> + scan_ascii(Rest, HasUpper); +scan_ascii(_Bin, _HasUpper) -> + not_ascii. + +-spec lower_ascii(binary(), binary()) -> binary(). +lower_ascii(<<>>, Acc) -> + Acc; +lower_ascii(<>, Acc) when C >= $A, C =< $Z -> + lower_ascii(Rest, <>); +lower_ascii(<>, Acc) -> + lower_ascii(Rest, <>). + -spec deep_merge_member(map(), map()) -> map(). deep_merge_member(CurrentMember, MemberUpdate) -> User0 = maps:get(<<"user">>, CurrentMember, #{}), @@ -153,3 +187,43 @@ default_presence() -> guild_member_list_connected:default_presence(). -spec connected_session_user_ids(map()) -> sets:set(integer()). connected_session_user_ids(S) -> guild_member_list_connected:connected_session_user_ids(S). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +casefold_reference(Bin) -> + {ok, Folded} = unicode_chardata_to_binary(string:casefold(Bin)), + Folded. + +ascii_casefold_matches_unicode_casefold_test() -> + lists:foreach( + fun(Bin) -> + ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)}) + end, + [ + <<>>, + <<"a">>, + <<"A">>, + <<"Zed">>, + <<"already lower">>, + <<"MiXeD CaSe 123">>, + <<"punct!@#$%^&*()_+-=[]{}">>, + <<"0123456789">>, + <<"~", 127>> + ] + ). + +ascii_casefold_defers_to_unicode_for_non_ascii_test() -> + ?assertEqual(not_ascii, ascii_casefold(<<"Ärger"/utf8>>)), + lists:foreach( + fun(Bin) -> + ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)}) + end, + [<<"Ärger"/utf8>>, <<"ÅSTRÖM"/utf8>>, <<"日本語"/utf8>>, <<"ß"/utf8>>] + ). + +ascii_casefold_returns_input_untouched_when_already_folded_test() -> + Bin = <<"no uppercase here">>, + ?assertEqual({ok, Bin}, ascii_casefold(Bin)). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl index 73a4f6ce6..91503d829 100644 --- a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl +++ b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl @@ -25,7 +25,7 @@ members_cursor(Limit, AfterId, State) -> Data = maps:get(data, State, #{}), MemberMap = guild_data_index:member_map(Data), Total = map_size(MemberMap), - SortedIds = lists:sort(maps:keys(MemberMap)), + SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap), FilteredIds = filter_ids_after(SortedIds, AfterId), ResponseMembers = take_members(FilteredIds, Limit, MemberMap), {reply, #{members => ResponseMembers, total => Total}, State}. @@ -33,8 +33,61 @@ members_cursor(Limit, AfterId, State) -> -spec take_members([integer()], integer(), map()) -> [map()]. take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> []; take_members(Ids, Limit, MemberMap) -> - [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)]. + [Member || Id <- lists:sublist(Ids, Limit), {ok, Member} <- [maps:find(Id, MemberMap)]]. -spec filter_ids_after([integer()], integer() | undefined) -> [integer()]. filter_ids_after(Ids, undefined) -> Ids; filter_ids_after(Ids, AfterId) -> lists:dropwhile(fun(Id) -> Id =< AfterId end, Ids). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +get_members_cursor_matches_full_sort_test() -> + lists:foreach(fun assert_cursor_page_matches/1, cursor_cases()). + +assert_cursor_page_matches({State, Limit, After}) -> + {reply, Reply, _} = get_members_cursor(cursor_request(Limit, After), State), + ?assertEqual(legacy_members_cursor(Limit, After, State), Reply). + +legacy_members_cursor(Limit, After, State) -> + MemberMap = guild_data_index:member_map(maps:get(data, State, #{})), + AfterId = snowflake_id:parse_optional(After), + Ids = filter_ids_after(lists:sort(maps:keys(MemberMap)), AfterId), + #{members => legacy_take_members(Ids, Limit, MemberMap), total => map_size(MemberMap)}. + +legacy_take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> []; +legacy_take_members(Ids, Limit, MemberMap) -> + [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)]. + +cursor_request(Limit, undefined) -> #{<<"limit">> => Limit}; +cursor_request(Limit, After) -> #{<<"limit">> => Limit, <<"after">> => After}. + +cursor_cases() -> + [ + {State, Limit, After} + || State <- cursor_states(), + Limit <- [0, 1, 2, 99], + After <- [undefined, 1, 5, 12, 39, 40, 41, 99] + ]. + +cursor_states() -> + Members = cursor_members(cursor_user_ids(40)), + Cached = guild_data_index:normalize_data(#{<<"members">> => Members}), + [ + #{data => Cached}, + #{data => guild_data_index:put_member(cursor_member(500), Cached)}, + #{data => #{<<"members">> => Members}}, + #{data => guild_data_index:normalize_data(#{<<"members">> => []})}, + #{} + ]. + +cursor_members(UserIds) -> + [cursor_member(UserId) || UserId <- UserIds]. + +cursor_user_ids(Count) -> + lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)). + +cursor_member(UserId) -> + #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_member_list_write.erl b/fluxer_gateway/src/guild/guild_member_list_write.erl index 1dfdc0a46..867e786bf 100644 --- a/fluxer_gateway/src/guild/guild_member_list_write.erl +++ b/fluxer_gateway/src/guild/guild_member_list_write.erl @@ -20,6 +20,10 @@ -type list_id() :: binary(). -type user_id() :: integer(). -type channel_id() :: integer(). +-type engine_ref() :: ets:table(). +-type absence() :: {absent, engine_ref()} | present. + +-define(MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT, 1000000000). -export_type([guild_state/0, list_id/0, user_id/0, channel_id/0]). @@ -180,14 +184,14 @@ broadcast_channel_engine_connection_change(UserId, State, SubsTab) -> guild_state(). fold_connection_change_lists(GuildId, UserId, State, SubsTab) -> Sessions = maps:get(sessions, State, #{}), - guild_member_list_subs:fold_list_ids( + lists:foldl( fun(ListId, AccState) -> - sync_connection_change_for_list( - GuildId, UserId, ListId, SubsTab, Sessions, AccState + sync_connection_change_for_subscribed_list( + GuildId, UserId, ListId, Sessions, AccState ) end, State, - SubsTab + guild_member_list_subs:list_ids(SubsTab) ). -spec dispatch_user_change_to_subscribed_lists( @@ -200,29 +204,38 @@ fold_connection_change_lists(GuildId, UserId, State, SubsTab) -> dispatch_user_change_to_subscribed_lists( UserId, OldMember, NewMember, SubsTab, State ) -> - guild_member_list_subs:fold_list_ids( + lists:foldl( fun(ListId, AccState) -> - dispatch_user_change_to_list( - UserId, OldMember, NewMember, ListId, SubsTab, AccState + dispatch_user_change_to_subscribed_list( + UserId, OldMember, NewMember, ListId, AccState ) end, State, - SubsTab + guild_member_list_subs:list_ids(SubsTab) ). --spec dispatch_user_change_to_list( +-spec dispatch_user_change_to_subscribed_list( user_id(), map() | undefined, map() | undefined, list_id(), - ets:table(), guild_state() ) -> guild_state(). -dispatch_user_change_to_list( - UserId, OldMember, NewMember, ListId, SubsTab, State -) -> +dispatch_user_change_to_subscribed_list(UserId, OldMember, NewMember, ListId, State) -> + Absence = user_change_absence(UserId, ListId, OldMember, NewMember, State), State1 = apply_user_change_to_channel_store(UserId, ListId, OldMember, NewMember, State), - queue_list_sync_if_subscribed(ListId, SubsTab, State1). + case sync_body_unchanged(UserId, ListId, Absence, State1) of + true -> record_member_list_sync_skipped_absent(State1); + false -> guild_member_list_sync_batch:queue_list_sync(ListId, State1) + end. + +-spec user_change_absence( + user_id(), list_id(), map() | undefined, map() | undefined, guild_state() +) -> absence(). +user_change_absence(UserId, ListId, Member, Member, State) when is_map(Member) -> + list_member_absence(UserId, ListId, State); +user_change_absence(_UserId, _ListId, _OldMember, _NewMember, _State) -> + present. -spec apply_user_change_to_channel_store( user_id(), list_id(), map() | undefined, map() | undefined, guild_state() @@ -249,18 +262,13 @@ apply_channel_member_change(UserId, ListId, OldMember, NewMember, State) when apply_channel_member_change(_UserId, _ListId, _OldMember, _NewMember, _State) -> ok. --spec sync_connection_change_for_list( - integer(), - user_id(), - list_id(), - ets:table(), - map(), - guild_state() +-spec sync_connection_change_for_subscribed_list( + integer(), user_id(), list_id(), map(), guild_state() ) -> guild_state(). -sync_connection_change_for_list(_GuildId, _UserId, ListId, SubsTab, _Sessions, State) -> +sync_connection_change_for_subscribed_list(_GuildId, UserId, ListId, _Sessions, State) -> case guild_member_list_channel_engine:is_engine_list(ListId, State) of true -> - queue_list_sync_if_subscribed(ListId, SubsTab, State); + queue_connection_list_sync_unless_absent(UserId, ListId, State); false -> State end. @@ -280,12 +288,10 @@ broadcast_list_by_id(_GuildId, _ChannelId, ListId, State) -> -spec rebuild_subscribed_channel_lists(guild_state(), ets:table()) -> guild_state(). rebuild_subscribed_channel_lists(State, SubsTab) -> - guild_member_list_subs:fold_lists( - fun(ListId, _Subs, AccState) -> - rebuild_channel_store(ListId, AccState) - end, + lists:foldl( + fun rebuild_channel_store/2, State, - SubsTab + guild_member_list_subs:list_ids(SubsTab) ). -spec rebuild_channel_store(list_id(), guild_state()) -> guild_state(). @@ -295,13 +301,104 @@ rebuild_channel_store(ListId, State) -> false -> State end. --spec queue_list_sync_if_subscribed(list_id(), ets:table(), guild_state()) -> guild_state(). -queue_list_sync_if_subscribed(ListId, SubsTab, State) -> - case guild_member_list_subs:has_subs(ListId, SubsTab) of - false -> State; - true -> guild_member_list_sync_batch:queue_list_sync(ListId, State) +-spec queue_connection_list_sync_unless_absent(user_id(), list_id(), guild_state()) -> + guild_state(). +queue_connection_list_sync_unless_absent(UserId, ListId, State) -> + case list_member_absence(UserId, ListId, State) of + {absent, _Ref} -> record_member_list_sync_skipped_absent(State); + present -> queue_connection_list_sync(ListId, State) end. +-spec queue_connection_list_sync(list_id(), guild_state()) -> guild_state(). +queue_connection_list_sync(ListId, State) -> + case maps:get(pending_member_list_sync_batch, State, undefined) of + #{pending_list_ids := PendingListIds} = Batch when is_map(PendingListIds) -> + State#{ + pending_member_list_sync_batch => Batch#{ + pending_list_ids => PendingListIds#{ListId => true} + } + }; + _ -> + TimerRef = erlang:send_after( + connection_sync_delay_ms(State), self(), flush_member_list_sync_batch + ), + State#{ + pending_member_list_sync_batch => #{ + pending_list_ids => #{ListId => true}, + timer_ref => TimerRef + } + } + end. + +-spec connection_sync_delay_ms(guild_state()) -> pos_integer(). +connection_sync_delay_ms(State) -> + case State of + #{member_count := MemberCount} when is_integer(MemberCount), MemberCount >= 5000 -> + positive_env(member_list_large_connection_sync_delay_ms, 1000); + _ -> + positive_env(member_list_connection_sync_delay_ms, 250) + end. + +-spec positive_env(atom(), pos_integer()) -> pos_integer(). +positive_env(Key, Default) -> + case application:get_env(fluxer_gateway, Key, Default) of + Value when is_integer(Value), Value >= 1 -> Value; + _ -> Default + end. + +-spec record_member_list_sync_skipped_absent(guild_state()) -> guild_state(). +record_member_list_sync_skipped_absent(State) -> + case maps:get(member_list_sync_skipped_absent, State, 0) of + Count when + is_integer(Count), Count >= 0, Count < ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT + -> + State#{member_list_sync_skipped_absent => Count + 1}; + ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT -> + State; + Invalid -> + erlang:error({invalid_member_list_sync_skipped_absent, Invalid}) + end. + +-spec sync_body_unchanged(user_id(), list_id(), absence(), guild_state()) -> boolean(). +sync_body_unchanged(_UserId, _ListId, present, _State) -> + false; +sync_body_unchanged(UserId, ListId, {absent, Ref}, State) -> + list_store_ref(ListId, State) =:= Ref andalso member_row_absent(UserId, Ref). + +-spec list_member_absence(user_id(), list_id(), guild_state()) -> absence(). +list_member_absence(UserId, ListId, State) -> + absence_for_store(UserId, list_store_ref(ListId, State)). + +-spec absence_for_store(user_id(), engine_ref() | undefined) -> absence(). +absence_for_store(_UserId, undefined) -> + present; +absence_for_store(UserId, Ref) -> + case member_row_absent(UserId, Ref) of + true -> {absent, Ref}; + false -> present + end. + +-spec member_row_absent(user_id(), engine_ref()) -> boolean(). +member_row_absent(UserId, Ref) when is_integer(UserId), UserId > 0 -> + try guild_member_list_engine:is_member_online(Ref, UserId) of + not_present -> true; + _Online -> false + catch + _:_ -> false + end; +member_row_absent(_UserId, _Ref) -> + false. + +-spec list_store_ref(list_id(), guild_state()) -> engine_ref() | undefined. +list_store_ref(<<"0">>, State) -> + case maps:get(member_list_engine, State, undefined) of + undefined -> undefined; + Ref when is_reference(Ref); is_atom(Ref) -> Ref; + _Other -> undefined + end; +list_store_ref(ListId, State) -> + guild_member_list_channel_engine:ref(ListId, State). + -spec flush_pending_member_list_syncs(guild_state()) -> guild_state(). flush_pending_member_list_syncs(State) -> guild_member_list_sync_batch:flush_pending_syncs(State). @@ -316,12 +413,247 @@ apply_user_change_to_channel_store_updates_engine_test() -> State1 = apply_user_change_to_channel_store(2, <<"123">>, undefined, Member, State), ?assert(maps:is_key(<<"123">>, maps:get(channel_member_list_engines, State1))). -sync_connection_change_ignores_zero_list_test() -> - SubsTab = guild_member_list_subs:new(), - ?assertEqual( - #{}, - sync_connection_change_for_list(1, 2, <<"0">>, SubsTab, #{}, #{}) - ), - guild_member_list_subs:destroy(SubsTab). +absence_is_absent_for_unloaded_member_test() -> + Ref = guild_member_list_engine:new(), + State = engine_state(Ref), + try + ?assertEqual({absent, Ref}, list_member_absence(7, <<"500">>, State)) + after + guild_member_list_engine:destroy(Ref) + end. + +absence_is_present_for_loaded_member_test() -> + Ref = guild_member_list_engine:new(), + State = engine_state(Ref), + try + ok = guild_member_list_engine:bulk_load(Ref, [{7, <<"seven">>, [], true}], []), + ?assertEqual(present, list_member_absence(7, <<"500">>, State)), + ?assertEqual({absent, Ref}, list_member_absence(8, <<"500">>, State)) + after + guild_member_list_engine:destroy(Ref) + end. + +absence_is_present_without_store_test() -> + ?assertEqual(present, list_member_absence(7, <<"500">>, #{})), + ?assertEqual(present, list_member_absence(7, <<"0">>, #{})), + ?assertEqual(present, list_member_absence(0, <<"500">>, #{})). + +default_list_absence_reads_default_engine_test() -> + Ref = guild_member_list_engine:new(), + State = #{member_list_engine => Ref}, + try + ?assertEqual({absent, Ref}, list_member_absence(7, <<"0">>, State)), + ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true), + ?assertEqual(present, list_member_absence(7, <<"0">>, State)) + after + guild_member_list_engine:destroy(Ref) + end. + +sync_body_unchanged_requires_stable_ref_test() -> + Ref = guild_member_list_engine:new(), + Other = guild_member_list_engine:new(), + State = engine_state(Ref), + try + ?assert(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State)), + ?assertNot(sync_body_unchanged(7, <<"500">>, present, State)), + ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Other}, State)), + ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, #{})) + after + guild_member_list_engine:destroy(Ref), + guild_member_list_engine:destroy(Other) + end. + +sync_body_unchanged_false_when_member_reappears_test() -> + Ref = guild_member_list_engine:new(), + State = engine_state(Ref), + try + ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true), + ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State)) + after + guild_member_list_engine:destroy(Ref) + end. + +user_change_absence_forces_sync_on_member_change_test() -> + Ref = guild_member_list_engine:new(), + State = engine_state(Ref), + Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []}, + try + ?assertEqual({absent, Ref}, user_change_absence(7, <<"500">>, Member, Member, State)), + ?assertEqual(present, user_change_absence(7, <<"500">>, Member, undefined, State)), + ?assertEqual(present, user_change_absence(7, <<"500">>, undefined, Member, State)), + ?assertEqual( + present, + user_change_absence(7, <<"500">>, Member, Member#{<<"nick">> => <<"x">>}, State) + ) + after + guild_member_list_engine:destroy(Ref) + end. + +subscribed_list_sync_is_skipped_only_for_absent_member_test() -> + Ref = guild_member_list_engine:new(), + Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []}, + State = (engine_state(Ref))#{pending_member_list_sync_batch => #{pending_list_ids => #{}}}, + try + with_batch_window(false, fun() -> + ?assert( + maps:is_key( + pending_member_list_sync_batch, + dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State) + ) + ), + ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true), + ?assertNot( + maps:is_key( + pending_member_list_sync_batch, + dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State) + ) + ) + end) + after + guild_member_list_engine:destroy(Ref) + end. + +list_id_fold_matches_fold_lists_test() -> + Tab = guild_member_list_subs:new(), + try + guild_member_list_subs:subscribe(<<"s1">>, <<"500">>, [{0, 99}], Tab), + guild_member_list_subs:subscribe(<<"s2">>, <<"500">>, [{0, 9}], Tab), + guild_member_list_subs:subscribe(<<"s1">>, <<"600">>, [{0, 99}], Tab), + Sizes = guild_member_list_subs:fold_lists( + fun(ListId, ListSubs, Acc) -> Acc#{ListId => map_size(ListSubs)} end, + #{}, + Tab + ), + ?assertEqual(lists:sort(maps:keys(Sizes)), guild_member_list_subs:list_ids(Tab)), + ?assertEqual([], [ListId || {ListId, 0} <- maps:to_list(Sizes)]) + after + guild_member_list_subs:destroy(Tab) + end. + +connection_sync_delay_defaults_test() -> + ?assertEqual(250, connection_sync_delay_ms(#{})), + ?assertEqual(250, connection_sync_delay_ms(#{member_count => 4999})), + ?assertEqual(250, connection_sync_delay_ms(#{member_count => undefined})), + ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 5000})), + ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 60000})). + +connection_sync_delay_reads_env_test() -> + with_env(member_list_connection_sync_delay_ms, 40, fun() -> + ?assertEqual(40, connection_sync_delay_ms(#{member_count => 10})) + end), + with_env(member_list_large_connection_sync_delay_ms, 90, fun() -> + ?assertEqual(90, connection_sync_delay_ms(#{member_count => 5000})) + end). + +connection_sync_delay_rejects_invalid_env_test() -> + lists:foreach( + fun(Invalid) -> + with_env(member_list_connection_sync_delay_ms, Invalid, fun() -> + ?assertEqual(250, connection_sync_delay_ms(#{})) + end) + end, + [0, -1, undefined, 250.0, <<"250">>] + ). + +connection_list_sync_arms_one_timer_per_window_test() -> + State1 = queue_connection_list_sync(<<"500">>, #{}), + Batch1 = maps:get(pending_member_list_sync_batch, State1), + TimerRef = maps:get(timer_ref, Batch1), + try + ?assert(is_reference(TimerRef)), + ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch1)), + State2 = queue_connection_list_sync(<<"600">>, State1), + Batch2 = maps:get(pending_member_list_sync_batch, State2), + ?assertEqual(TimerRef, maps:get(timer_ref, Batch2)), + ?assertEqual( + #{<<"500">> => true, <<"600">> => true}, maps:get(pending_list_ids, Batch2) + ), + State3 = queue_connection_list_sync(<<"500">>, State2), + ?assertEqual(Batch2, maps:get(pending_member_list_sync_batch, State3)) + after + _ = erlang:cancel_timer(TimerRef) + end. + +connection_change_sync_is_debounced_test() -> + Ref = guild_member_list_engine:new(), + try + ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true), + Next = sync_connection_change_for_subscribed_list( + 1, 7, <<"500">>, #{}, engine_state(Ref) + ), + Batch = maps:get(pending_member_list_sync_batch, Next), + ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch)), + _ = erlang:cancel_timer(maps:get(timer_ref, Batch)) + after + guild_member_list_engine:destroy(Ref) + end. + +connection_change_skip_absent_arms_no_timer_test() -> + Ref = guild_member_list_engine:new(), + try + Next = sync_connection_change_for_subscribed_list( + 1, 7, <<"500">>, #{}, engine_state(Ref) + ), + ?assertNot(maps:is_key(pending_member_list_sync_batch, Next)), + ?assertEqual(1, maps:get(member_list_sync_skipped_absent, Next)) + after + guild_member_list_engine:destroy(Ref) + end. + +connection_change_skip_absent_keeps_pending_batch_test() -> + Ref = guild_member_list_engine:new(), + TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch), + Batch = #{pending_list_ids => #{<<"600">> => true}, timer_ref => TimerRef}, + State = (engine_state(Ref))#{pending_member_list_sync_batch => Batch}, + try + Next = sync_connection_change_for_subscribed_list(1, 7, <<"500">>, #{}, State), + ?assertEqual(Batch, maps:get(pending_member_list_sync_batch, Next)) + after + _ = erlang:cancel_timer(TimerRef), + guild_member_list_engine:destroy(Ref) + end. + +member_update_sync_stays_immediate_test() -> + Ref = guild_member_list_engine:new(), + TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch), + Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []}, + State = (engine_state(Ref))#{ + pending_member_list_sync_batch => #{ + pending_list_ids => #{<<"600">> => true}, + timer_ref => TimerRef + } + }, + try + ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true), + ?assertNot( + maps:is_key( + pending_member_list_sync_batch, + dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State) + ) + ) + after + _ = erlang:cancel_timer(TimerRef), + guild_member_list_engine:destroy(Ref) + end. + +engine_state(Ref) -> + #{channel_member_list_engines => #{<<"500">> => Ref}, member_presence => #{}}. + +with_batch_window(Value, Fun) -> + with_env(guild_member_list_sync_batch_window, Value, Fun). + +with_env(Key, Value, Fun) -> + Previous = application:get_env(fluxer_gateway, Key), + application:set_env(fluxer_gateway, Key, Value), + try + Fun() + after + restore_env(Key, Previous) + end. + +restore_env(Key, {ok, Previous}) -> + application:set_env(fluxer_gateway, Key, Previous); +restore_env(Key, undefined) -> + application:unset_env(fluxer_gateway, Key). -endif. diff --git a/fluxer_gateway/src/guild/guild_members_table_repair.erl b/fluxer_gateway/src/guild/guild_members_table_repair.erl new file mode 100644 index 000000000..daa508360 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_members_table_repair.erl @@ -0,0 +1,542 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_members_table_repair). +-typing([eqwalizer]). + +-export([ + maybe_repair/1, + members_table_healthy/1 +]). + +-define(REPAIR_KEY, members_ets_repair). +-define(ATTEMPTS_KEY, members_ets_repair_attempts). +-define(MAX_MEMBER_ETS_REPAIR_CHUNK, 5000). +-define(MAX_MEMBER_ETS_REPAIR_MEMBERS, 250000). +-define(MAX_MEMBER_ETS_REPAIR_TICKS, 200). +-define(MAX_MEMBER_ETS_REPAIR_ATTEMPTS, 3). +-define(MAX_MEMBER_ETS_REPAIR_MAILBOX, 1000). + +-type guild_state() :: map(). +-type user_id() :: integer(). +-type member() :: map(). +-type member_map() :: #{user_id() => member()}. +-type pass_result() :: continue | complete | {error, table_unavailable}. + +-export_type([guild_state/0]). + +-spec maybe_repair(guild_state()) -> guild_state(). +maybe_repair(State) -> + case self_heal_enabled() of + true -> repair_enabled(State); + false -> release_pending_repair(State) + end. + +-spec repair_enabled(guild_state()) -> guild_state(). +repair_enabled(State) -> + case members_table_healthy(members_table(State)) of + true -> release_pending_repair(State); + false -> repair_unhealthy(State) + end. + +-spec self_heal_enabled() -> boolean(). +self_heal_enabled() -> + application:get_env(fluxer_gateway, guild_members_table_self_heal, true) =:= true. + +-spec members_table_healthy(term()) -> boolean(). +members_table_healthy(Tab) when is_reference(Tab) -> + table_owner(Tab) =:= self(); +members_table_healthy(_Tab) -> + false. + +-spec repair_unhealthy(guild_state()) -> guild_state(). +repair_unhealthy(State) -> + case mailbox_within_bound(message_queue_len()) of + true -> repair_pass(State); + false -> State + end. + +-spec repair_pass(guild_state()) -> guild_state(). +repair_pass(State) -> + case maps:get(?REPAIR_KEY, State, undefined) of + #{tab := Tab, ticks := Ticks} when is_reference(Tab), is_integer(Ticks) -> + continue_repair(Tab, Ticks, State); + _ -> + start_repair(maps:remove(?REPAIR_KEY, State)) + end. + +-spec start_repair(guild_state()) -> guild_state(). +start_repair(State) -> + case attempts(State) < ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS of + true -> start_repair_within_bounds(State); + false -> State + end. + +-spec start_repair_within_bounds(guild_state()) -> guild_state(). +start_repair_within_bounds(State) -> + Size = map_size(member_map(State)), + case Size =< ?MAX_MEMBER_ETS_REPAIR_MEMBERS of + true -> create_staging_table(State); + false -> refuse_oversized(Size, State) + end. + +-spec refuse_oversized(non_neg_integer(), guild_state()) -> guild_state(). +refuse_oversized(Size, State) -> + logger:warning( + "guild_members_table_repair_refused: guild_id=~p members=~p max=~p", + [guild_id(State), Size, ?MAX_MEMBER_ETS_REPAIR_MEMBERS] + ), + State#{?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS}. + +-spec create_staging_table(guild_state()) -> guild_state(). +create_staging_table(State) -> + case new_staging_table() of + Tab when is_reference(Tab) -> continue_repair(Tab, 0, State); + undefined -> record_start_failure(State) + end. + +-spec continue_repair(ets:tid(), non_neg_integer(), guild_state()) -> guild_state(). +continue_repair(Tab, Ticks, State) when Ticks >= ?MAX_MEMBER_ETS_REPAIR_TICKS -> + abandon_repair(Tab, ticks_exhausted, State); +continue_repair(Tab, Ticks, State) -> + case run_pass(Tab, member_map(State)) of + continue -> State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks + 1}}; + complete -> publish_repair(Tab, State); + {error, table_unavailable} -> abandon_repair(Tab, table_unavailable, State) + end. + +-spec publish_repair(ets:tid(), guild_state()) -> guild_state(). +publish_repair(Tab, State) -> + case maps:get(data, State, #{}) of + Data when is_map(Data) -> publish_into_data(Tab, Data, State); + _ -> abandon_repair(Tab, invalid_guild_data, State) + end. + +-spec publish_into_data(ets:tid(), map(), guild_state()) -> guild_state(). +publish_into_data(Tab, Data, State) -> + logger:info( + "guild_members_table_repair_completed: guild_id=~p members=~p", + [guild_id(State), map_size(member_map(State))] + ), + Published = State#{data => Data#{members_ets => Tab}}, + maps:remove(?ATTEMPTS_KEY, maps:remove(?REPAIR_KEY, Published)). + +-spec abandon_repair(ets:tid(), atom(), guild_state()) -> guild_state(). +abandon_repair(Tab, Reason, State) -> + ok = delete_staging_table(Tab), + logger:warning( + "guild_members_table_repair_abandoned: guild_id=~p reason=~p attempts=~p", + [guild_id(State), Reason, attempts(State) + 1] + ), + bump_attempts(maps:remove(?REPAIR_KEY, State)). + +-spec record_start_failure(guild_state()) -> guild_state(). +record_start_failure(State) -> + logger:warning( + "guild_members_table_repair_start_failed: guild_id=~p attempts=~p", + [guild_id(State), attempts(State) + 1] + ), + bump_attempts(State). + +-spec run_pass(ets:tid(), member_map()) -> pass_result(). +run_pass(Tab, MemberMap) -> + run_pass(Tab, MemberMap, ?MAX_MEMBER_ETS_REPAIR_CHUNK). + +-spec run_pass(ets:tid(), member_map(), pos_integer()) -> pass_result(). +run_pass(Tab, MemberMap, Budget) -> + try + apply_pass(Tab, MemberMap, Budget) + catch + error:badarg -> {error, table_unavailable} + end. + +-spec apply_pass(ets:tid(), member_map(), pos_integer()) -> continue | complete. +apply_pass(Tab, MemberMap, Budget) -> + case collect_missing_rows(Tab, MemberMap, Budget) of + [] -> drop_extra_rows(Tab, MemberMap, Budget); + Rows -> insert_missing_rows(Tab, MemberMap, Rows, Budget) + end. + +-spec insert_missing_rows(ets:tid(), member_map(), [{user_id(), member()}], pos_integer()) -> + continue | complete. +insert_missing_rows(Tab, MemberMap, Rows, Budget) -> + true = ets:insert(Tab, Rows), + case length(Rows) < Budget of + true -> drop_extra_rows(Tab, MemberMap, Budget); + false -> continue + end. + +-spec drop_extra_rows(ets:tid(), member_map(), pos_integer()) -> continue | complete. +drop_extra_rows(Tab, MemberMap, Budget) -> + case collect_extra_ids(Tab, MemberMap, Budget) of + [] -> complete; + Ids -> delete_extra_ids(Tab, Ids, Budget) + end. + +-spec delete_extra_ids(ets:tid(), [user_id()], pos_integer()) -> continue | complete. +delete_extra_ids(Tab, Ids, Budget) -> + lists:foreach(fun(UserId) -> ets:delete(Tab, UserId) end, Ids), + case length(Ids) < Budget of + true -> complete; + false -> continue + end. + +-spec collect_missing_rows(ets:tid(), member_map(), pos_integer()) -> + [{user_id(), member()}]. +collect_missing_rows(Tab, MemberMap, Budget) -> + collect_missing_iter(maps:next(maps:iterator(MemberMap)), Tab, Budget, []). + +-spec collect_missing_iter(term(), ets:tid(), non_neg_integer(), [{user_id(), member()}]) -> + [{user_id(), member()}]. +collect_missing_iter(_Next, _Tab, 0, Acc) -> + Acc; +collect_missing_iter(none, _Tab, _Budget, Acc) -> + Acc; +collect_missing_iter({UserId, Member, Iter}, Tab, Budget, Acc) -> + case ets:member(Tab, UserId) of + true -> + collect_missing_iter(maps:next(Iter), Tab, Budget, Acc); + false -> + collect_missing_iter(maps:next(Iter), Tab, Budget - 1, [{UserId, Member} | Acc]) + end. + +-spec collect_extra_ids(ets:tid(), member_map(), pos_integer()) -> [user_id()]. +collect_extra_ids(Tab, MemberMap, Budget) -> + collect_extra_iter(ets:first(Tab), Tab, MemberMap, Budget, []). + +-spec collect_extra_iter(term(), ets:tid(), member_map(), non_neg_integer(), [user_id()]) -> + [user_id()]. +collect_extra_iter(_Key, _Tab, _MemberMap, 0, Acc) -> + Acc; +collect_extra_iter('$end_of_table', _Tab, _MemberMap, _Budget, Acc) -> + Acc; +collect_extra_iter(Key, Tab, MemberMap, Budget, Acc) -> + Next = ets:next(Tab, Key), + case maps:is_key(Key, MemberMap) of + true -> + collect_extra_iter(Next, Tab, MemberMap, Budget, Acc); + false -> + collect_extra_iter(Next, Tab, MemberMap, Budget - 1, [Key | Acc]) + end. + +-spec release_pending_repair(guild_state()) -> guild_state(). +release_pending_repair(State) -> + case maps:get(?REPAIR_KEY, State, undefined) of + undefined -> State; + #{tab := Tab} when is_reference(Tab) -> discard_staging_table(Tab, State); + _ -> maps:remove(?REPAIR_KEY, State) + end. + +-spec discard_staging_table(ets:tid(), guild_state()) -> guild_state(). +discard_staging_table(Tab, State) -> + ok = delete_staging_table(Tab), + maps:remove(?REPAIR_KEY, State). + +-spec new_staging_table() -> ets:tid() | undefined. +new_staging_table() -> + try ets:new(guild_members_data, [set, public, {read_concurrency, true}]) of + Tab when is_reference(Tab) -> Tab; + _Named -> undefined + catch + error:badarg -> undefined; + error:system_limit -> undefined + end. + +-spec delete_staging_table(ets:tid()) -> ok. +delete_staging_table(Tab) -> + try ets:delete(Tab) of + _ -> ok + catch + error:badarg -> ok + end. + +-spec table_owner(term()) -> pid() | undefined. +table_owner(Tab) -> + try ets:info(eqwalizer:dynamic_cast(Tab), owner) of + Owner when is_pid(Owner) -> Owner; + _ -> undefined + catch + error:badarg -> undefined + end. + +-spec members_table(guild_state()) -> ets:tid() | undefined. +members_table(State) -> + members_ets_table(data_map(State)). + +-spec members_ets_table(map()) -> ets:tid() | undefined. +members_ets_table(#{members_ets := Tab}) -> + Tab; +members_ets_table(_Data) -> + undefined. + +-spec member_map(guild_state()) -> member_map(). +member_map(State) -> + guild_data_index_members:member_map(data_map(State)). + +-spec data_map(guild_state()) -> map(). +data_map(State) -> + case maps:get(data, State, #{}) of + Data when is_map(Data) -> Data; + _ -> #{} + end. + +-spec guild_id(guild_state()) -> term(). +guild_id(State) -> + maps:get(id, State, undefined). + +-spec attempts(guild_state()) -> non_neg_integer(). +attempts(State) -> + case maps:get(?ATTEMPTS_KEY, State, 0) of + N when is_integer(N), N >= 0 -> N; + _ -> 0 + end. + +-spec bump_attempts(guild_state()) -> guild_state(). +bump_attempts(State) -> + State#{?ATTEMPTS_KEY => attempts(State) + 1}. + +-spec message_queue_len() -> non_neg_integer(). +message_queue_len() -> + case erlang:process_info(self(), message_queue_len) of + {message_queue_len, Len} when is_integer(Len), Len >= 0 -> Len; + _ -> 0 + end. + +-spec mailbox_within_bound(non_neg_integer()) -> boolean(). +mailbox_within_bound(Len) -> + Len < ?MAX_MEMBER_ETS_REPAIR_MAILBOX. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +healthy_table_is_left_untouched_test() -> + Tab = ets:new(healthy_members, [set, public]), + try + State = repair_state_with_table(#{1 => test_member(1)}, Tab), + ?assertEqual(State, maybe_repair(State)), + ?assertEqual(0, ets:info(Tab, size)) + after + ets:delete(Tab) + end. + +healthy_table_releases_pending_staging_table_test() -> + Healthy = ets:new(healthy_members, [set, public]), + try + Staging = ets:new(rollback_members, [set, public]), + State0 = with_repair_key( + repair_state_with_table(#{1 => test_member(1)}, Healthy), Staging, 3 + ), + State1 = maybe_repair(State0), + ?assertNot(maps:is_key(?REPAIR_KEY, State1)), + ?assertEqual(undefined, ets:info(Staging, size)) + after + ets:delete(Healthy) + end. + +repair_publishes_table_matching_member_map_test() -> + ok = drain_mailbox(), + with_self_heal_enabled(fun() -> + MemberMap = #{1 => test_member(1), 2 => test_member(2)}, + State1 = maybe_repair(repair_state(MemberMap)), + Tab = published_table(State1), + ?assert(is_reference(Tab)), + ?assertNot(maps:is_key(?REPAIR_KEY, State1)), + ?assertEqual(lists:sort(maps:to_list(MemberMap)), lists:sort(ets:tab2list(Tab))), + ?assertEqual(State1, maybe_repair(State1)), + ets:delete(Tab) + end). + +repair_replaces_dead_table_tid_test() -> + ok = drain_mailbox(), + with_self_heal_enabled(fun() -> + Dead = ets:new(dead_members, [set, public]), + true = ets:delete(Dead), + State0 = repair_state_with_table(#{1 => test_member(1)}, Dead), + State1 = maybe_repair(State0), + Tab = published_table(State1), + ?assert(is_reference(Tab)), + ?assertNotEqual(Dead, Tab), + ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab)), + ets:delete(Tab) + end). + +repair_replaces_table_owned_by_another_process_test() -> + ok = drain_mailbox(), + with_self_heal_enabled(fun() -> + {Owner, Foreign} = foreign_members_table(), + State1 = maybe_repair(repair_state_with_table(#{1 => test_member(1)}, Foreign)), + Tab = published_table(State1), + ?assert(is_reference(Tab)), + ?assertNotEqual(Foreign, Tab), + ?assertEqual(Owner, ets:info(Foreign, owner)), + Owner ! stop, + ets:delete(Tab) + end). + +repair_pass_is_bounded_by_chunk_test() -> + Tab = ets:new(bounded_members, [set, public]), + try + MemberMap = maps:from_list([{N, test_member(N)} || N <- lists:seq(1, 5)]), + ?assertEqual(continue, run_pass(Tab, MemberMap, 2)), + ?assertEqual(2, ets:info(Tab, size)), + ?assertEqual(continue, run_pass(Tab, MemberMap, 2)), + ?assertEqual(4, ets:info(Tab, size)), + ?assertEqual(complete, run_pass(Tab, MemberMap, 2)), + ?assertEqual(5, ets:info(Tab, size)) + after + ets:delete(Tab) + end. + +repair_picks_up_member_added_mid_build_test() -> + Tab = ets:new(midbuild_members, [set, public]), + try + Map1 = #{1 => test_member(1), 2 => test_member(2)}, + ?assertEqual(continue, run_pass(Tab, Map1, 1)), + Map2 = Map1#{3 => test_member(3)}, + ?assertEqual(continue, run_pass(Tab, Map2, 1)), + ?assertEqual(continue, run_pass(Tab, Map2, 1)), + ?assertEqual(complete, run_pass(Tab, Map2, 1)), + ?assertEqual([1, 2, 3], table_ids(Tab)) + after + ets:delete(Tab) + end. + +repair_deletes_member_removed_mid_build_test() -> + Tab = ets:new(midremove_members, [set, public]), + try + Map1 = #{1 => test_member(1), 2 => test_member(2)}, + ?assertEqual(continue, run_pass(Tab, Map1, 2)), + ?assertEqual(2, ets:info(Tab, size)), + ?assertEqual(complete, run_pass(Tab, #{1 => test_member(1)}, 2)), + ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab)) + after + ets:delete(Tab) + end. + +repair_abandons_after_tick_cap_test() -> + ok = drain_mailbox(), + with_self_heal_enabled(fun() -> + Tab = ets:new(ticks_members, [set, public]), + State0 = with_repair_key( + repair_state(#{1 => test_member(1)}), Tab, ?MAX_MEMBER_ETS_REPAIR_TICKS + ), + State1 = maybe_repair(State0), + ?assertNot(maps:is_key(?REPAIR_KEY, State1)), + ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)), + ?assertEqual(undefined, ets:info(Tab, size)), + ?assertEqual(undefined, published_table(State1)) + end). + +repair_abandons_when_staging_table_is_dead_test() -> + ok = drain_mailbox(), + with_self_heal_enabled(fun() -> + Tab = ets:new(dying_members, [set, public]), + true = ets:delete(Tab), + State0 = with_repair_key(repair_state(#{1 => test_member(1)}), Tab, 1), + State1 = maybe_repair(State0), + ?assertNot(maps:is_key(?REPAIR_KEY, State1)), + ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)), + ?assertEqual(undefined, published_table(State1)) + end). + +repair_stops_after_attempt_cap_test() -> + State0 = (repair_state(#{1 => test_member(1)}))#{ + ?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS + }, + ?assertEqual(State0, maybe_repair(State0)). + +oversized_member_map_is_refused_and_counted_test() -> + State0 = repair_state(#{1 => test_member(1)}), + State1 = refuse_oversized(?MAX_MEMBER_ETS_REPAIR_MEMBERS + 1, State0), + ?assertEqual(?MAX_MEMBER_ETS_REPAIR_ATTEMPTS, maps:get(?ATTEMPTS_KEY, State1)), + ?assertEqual(State1, start_repair(State1)). + +mailbox_guard_bound_test() -> + ?assert(mailbox_within_bound(0)), + ?assert(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX - 1)), + ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX)), + ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX + 1)). + +repair_skips_pass_when_mailbox_is_deep_test() -> + with_self_heal_enabled(fun() -> + Self = self(), + Sent = lists:seq(1, ?MAX_MEMBER_ETS_REPAIR_MAILBOX), + lists:foreach(fun(N) -> Self ! {mailbox_guard, N} end, Sent), + State = repair_state(#{1 => test_member(1)}), + ?assertEqual(State, maybe_repair(State)), + drain_mailbox_guard(?MAX_MEMBER_ETS_REPAIR_MAILBOX) + end). + +members_table_healthy_rejects_non_references_test() -> + ?assertNot(members_table_healthy(undefined)), + ?assertNot(members_table_healthy(not_a_table)), + ?assertNot(members_table_healthy(make_ref())). + +%% repair_unhealthy/1 bails out when the CURRENT process mailbox exceeds +%% ?MAX_MEMBER_ETS_REPAIR_MAILBOX, so a test that leaves the eunit process's +%% mailbox polluted by an earlier test silently exercises the bail-out branch +%% instead of the repair it means to assert. +drain_mailbox() -> + receive + _Any -> drain_mailbox() + after 0 -> ok + end. + +%% Self-heal defaults to off (production enables it via config), so a test that +%% exercises the actual repair pass must arm it and always disarm it afterwards - +%% left on, it would make every other test's update_counts/1 call in this shared +%% eunit VM attempt a real repair too. +with_self_heal_enabled(Fun) -> + ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true), + try + Fun() + after + application:unset_env(fluxer_gateway, guild_members_table_self_heal) + end. + +repair_state(MemberMap) -> + #{ + id => 42, + data => #{<<"members">> => MemberMap, members_normalized => MemberMap} + }. + +repair_state_with_table(MemberMap, Tab) -> + State = repair_state(MemberMap), + Data = maps:get(data, State), + State#{data => Data#{members_ets => Tab}}. + +with_repair_key(State, Tab, Ticks) -> + State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks}}. + +published_table(State) -> + maps:get(members_ets, maps:get(data, State, #{}), undefined). + +test_member(UserId) -> + #{<<"user">> => #{<<"id">> => UserId}}. + +table_ids(Tab) -> + lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]). + +foreign_members_table() -> + Self = self(), + Owner = spawn(fun() -> foreign_table_owner(Self) end), + receive + {foreign_members_table, Tab} -> {Owner, Tab} + after 5000 -> error(foreign_table_timeout) + end. + +foreign_table_owner(Parent) -> + Parent ! {foreign_members_table, ets:new(foreign_members, [set, public])}, + receive + stop -> ok + after 30000 -> ok + end. + +drain_mailbox_guard(0) -> + ok; +drain_mailbox_guard(N) -> + receive + {mailbox_guard, _} -> drain_mailbox_guard(N - 1) + after 0 -> ok + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_mutual_online.erl b/fluxer_gateway/src/guild/guild_mutual_online.erl index e8855f13f..4c4350755 100644 --- a/fluxer_gateway/src/guild/guild_mutual_online.erl +++ b/fluxer_gateway/src/guild/guild_mutual_online.erl @@ -13,6 +13,8 @@ -type user_id() :: integer(). -type guild_state() :: map(). +-type viewable_index() :: #{user_id() => map()}. +-type index_ctx() :: {map(), viewable_index(), guild_state()}. -spec compute_count(user_id() | term(), guild_state()) -> non_neg_integer(). compute_count(UserId, State) when is_integer(UserId), UserId > 0 -> @@ -37,7 +39,7 @@ slow_count(UserId, State) -> true -> self_online_count(UserId, State); false -> - count_mutually_visible(UserId, ViewerSet, State) + count_mutually_visible_indexed(UserId, ViewerSet, State) end. -spec self_online_count(user_id(), guild_state()) -> non_neg_integer(). @@ -47,38 +49,120 @@ self_online_count(UserId, State) -> false -> 0 end. --spec count_mutually_visible(user_id(), sets:set(), guild_state()) -> non_neg_integer(). -count_mutually_visible(UserId, ViewerSet, State) -> +-spec count_mutually_visible_indexed(user_id(), sets:set(), guild_state()) -> + non_neg_integer(). +count_mutually_visible_indexed(UserId, ViewerSet, State) -> Tab = maps:get(member_presence, State), + Ctx = {viewer_channel_map(ViewerSet), build_viewable_index(State), State}, ets:foldl( fun({OtherUserId, Presence}, Acc) -> - count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) + count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc) end, 0, Tab ). --spec count_online_member( - user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer() +-spec count_online_member_indexed( + user_id(), index_ctx(), term(), term(), non_neg_integer() ) -> non_neg_integer(). -count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when +count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc) when is_integer(OtherUserId), is_map(Presence), OtherUserId > 0 -> case is_online(Presence) of false -> Acc; true when OtherUserId =:= UserId -> Acc + 1; - true -> count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) + true -> count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc) end; -count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) -> +count_online_member_indexed(_UserId, _Ctx, _OtherUserId, _Presence, Acc) -> Acc. --spec count_if_mutually_visible(user_id(), sets:set(), guild_state(), non_neg_integer()) -> +-spec count_if_mutually_visible_indexed(user_id(), index_ctx(), non_neg_integer()) -> non_neg_integer(). -count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) -> - OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State), - case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of +count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc) -> + case shares_viewable_channel(OtherUserId, Ctx) of + true -> Acc + 1; + false -> Acc + end. + +-spec shares_viewable_channel(user_id(), index_ctx()) -> boolean(). +shares_viewable_channel(OtherUserId, {ViewerMap, Index, State}) -> + case maps:find(OtherUserId, Index) of + {ok, OtherMap} -> maps_share_any_key(OtherMap, ViewerMap); + error -> channel_list_shares_any(OtherUserId, ViewerMap, State) + end. + +-spec channel_list_shares_any(user_id(), map(), guild_state()) -> boolean(). +channel_list_shares_any(OtherUserId, ViewerMap, State) -> + Channels = guild_visibility:get_user_viewable_channels(OtherUserId, State), + lists:any(fun(ChannelId) -> maps:is_key(ChannelId, ViewerMap) end, Channels). + +-spec viewer_channel_map(sets:set()) -> map(). +viewer_channel_map(ViewerSet) -> + sets:fold(fun(ChannelId, Acc) -> Acc#{ChannelId => true} end, #{}, ViewerSet). + +-spec build_viewable_index(guild_state()) -> viewable_index(). +build_viewable_index(State) -> + build_index_from_sessions(maps:get(sessions, State, #{})). + +-spec build_index_from_sessions(term()) -> viewable_index(). +build_index_from_sessions(Sessions) when is_map(Sessions) -> + build_index_iter(maps:iterator(Sessions), #{}); +build_index_from_sessions(_) -> + #{}. + +-spec build_index_iter(maps:iterator(), viewable_index()) -> viewable_index(). +build_index_iter(Iterator, Acc) -> + case maps:next(Iterator) of + none -> + Acc; + {_, SessionData, Next} when is_map(SessionData) -> + build_index_iter(Next, index_session(SessionData, Acc)); + {_, _, Next} -> + build_index_iter(Next, Acc) + end. + +-spec index_session(map(), viewable_index()) -> viewable_index(). +index_session(SessionData, Acc) -> + SessionUserId = maps:get(user_id, SessionData, undefined), + ViewableChannels = maps:get(viewable_channels, SessionData, undefined), + index_session_entry(SessionUserId, ViewableChannels, Acc). + +-spec index_session_entry(term(), term(), viewable_index()) -> viewable_index(). +index_session_entry(UserId, ViewableChannels, Acc) when + is_integer(UserId), is_map(ViewableChannels) +-> + put_first_session_map(UserId, ViewableChannels, Acc); +index_session_entry(_, _, Acc) -> + Acc. + +-spec put_first_session_map(user_id(), map(), viewable_index()) -> viewable_index(). +put_first_session_map(UserId, ViewableChannels, Acc) -> + case maps:is_key(UserId, Acc) of true -> Acc; - false -> Acc + 1 + false -> Acc#{UserId => ViewableChannels} + end. + +-spec maps_share_any_key(map(), map()) -> boolean(). +maps_share_any_key(MapA, MapB) -> + {Smaller, Larger} = + case map_size(MapA) =< map_size(MapB) of + true -> {MapA, MapB}; + false -> {MapB, MapA} + end, + maps_share_any_key_iter(maps:iterator(Smaller), Larger). + +-spec maps_share_any_key_iter(maps:iterator(), map()) -> boolean(). +maps_share_any_key_iter(Iterator, LargerMap) -> + case maps:next(Iterator) of + none -> false; + {Key, _, NextIterator} -> key_matches_or_continue(Key, NextIterator, LargerMap) + end. + +-spec key_matches_or_continue(term(), maps:iterator(), map()) -> boolean(). +key_matches_or_continue(Key, NextIterator, LargerMap) -> + case maps:is_key(Key, LargerMap) of + true -> true; + false -> maps_share_any_key_iter(NextIterator, LargerMap) end. -spec is_self_online(user_id(), guild_state()) -> boolean(). @@ -258,6 +342,69 @@ slow_path_returns_zero_when_viewer_offline_and_no_channels_test() -> }, ?assertEqual(0, compute_count(10, State)). +index_matches_scan_without_sessions_test() -> + State = mutual_visibility_state(1, 5000), + ViewerSet = guild_visibility:viewable_channel_set(10, State), + Expected = reference_count_mutually_visible(10, ViewerSet, State), + ?assertEqual(2, Expected), + ?assertEqual(Expected, compute_count(10, State)). + +index_matches_scan_with_cached_session_channels_test() -> + Base = mutual_visibility_state(1, 5000), + State = Base#{sessions => cached_viewable_sessions()}, + ViewerSet = guild_visibility:viewable_channel_set(10, State), + Expected = reference_count_mutually_visible(10, ViewerSet, State), + ?assertEqual(3, Expected), + ?assertEqual(Expected, compute_count(10, State)). + +index_counts_with_cached_session_channels_test() -> + Base = mutual_visibility_state(1, 5000), + State = Base#{sessions => cached_viewable_sessions()}, + ?assertEqual(3, compute_count(10, State)). + +cached_viewable_sessions() -> + #{ + <<"s20a">> => #{user_id => 20, viewable_channels => undefined}, + <<"s20b">> => #{user_id => 20, viewable_channels => #{100 => true}}, + <<"s40">> => #{user_id => 40, viewable_channels => #{101 => true}} + }. + +-spec reference_count_mutually_visible(user_id(), sets:set(), guild_state()) -> + non_neg_integer(). +reference_count_mutually_visible(UserId, ViewerSet, State) -> + Tab = maps:get(member_presence, State), + ets:foldl( + fun({OtherUserId, Presence}, Acc) -> + reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) + end, + 0, + Tab + ). + +-spec reference_count_online_member( + user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer() +) -> non_neg_integer(). +reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when + is_integer(OtherUserId), is_map(Presence), OtherUserId > 0 +-> + case is_online(Presence) of + false -> Acc; + true when OtherUserId =:= UserId -> Acc + 1; + true -> reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) + end; +reference_count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) -> + Acc. + +-spec reference_count_if_mutually_visible( + user_id(), sets:set(), guild_state(), non_neg_integer() +) -> non_neg_integer(). +reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) -> + OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State), + case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of + true -> Acc; + false -> Acc + 1 + end. + make_presence_tab(Map) -> Tab = ets:new(test_member_presence, [set, public]), maps:foreach(fun(K, V) -> ets:insert(Tab, {K, V}) end, Map), diff --git a/fluxer_gateway/src/guild/guild_passive_sync.erl b/fluxer_gateway/src/guild/guild_passive_sync.erl index 1de624378..1275b261b 100644 --- a/fluxer_gateway/src/guild/guild_passive_sync.erl +++ b/fluxer_gateway/src/guild/guild_passive_sync.erl @@ -15,6 +15,7 @@ -export_type([guild_state/0, channel_id/0, last_message_id/0, version/0, voice_state/0]). -define(PASSIVE_SYNC_INTERVAL, 30000). +-define(LARGE_GUILD_MEMBER_COUNT, 250). -type guild_state() :: map(). -type channel_id() :: binary(). @@ -30,40 +31,53 @@ schedule_passive_sync(State) -> -spec handle_passive_sync(guild_state()) -> {noreply, guild_state()}. handle_passive_sync(State) -> GuildId = maps:get(id, State), - Sessions = maps:get(sessions, State, #{}), - Data = maps:get(data, State, #{}), - MemberCount = maps:get(member_count, State, undefined), - VoiceStates = maps:get(voice_states, State, #{}), - _ = spawn(fun() -> - send_passive_updates( - GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount - ) - end), + ok = maybe_spawn_passive_updates(GuildId, State), _ = schedule_passive_sync(State), {noreply, State}. +%% send_passive_updates/4 keeps no session unless is_large_guild/1 holds, so for any other +%% guild the spawned child returns without touching a session, a payload or the registry. +-spec maybe_spawn_passive_updates(integer(), guild_state()) -> ok. +maybe_spawn_passive_updates(GuildId, State) -> + case is_large_guild(maps:get(member_count, State, undefined)) of + false -> ok; + true -> spawn_passive_updates(GuildId, State) + end. + +-spec spawn_passive_updates(integer(), guild_state()) -> ok. +spawn_passive_updates(GuildId, State) -> + _ = spawn(fun() -> send_passive_updates_for_state(GuildId, State) end), + ok. + -spec send_passive_updates_to_sessions(guild_state()) -> guild_state(). send_passive_updates_to_sessions(State) -> - GuildId = maps:get(id, State), + ok = send_passive_updates_for_state(maps:get(id, State), State), + State. + +-spec send_passive_updates_for_state(integer(), guild_state()) -> ok. +send_passive_updates_for_state(GuildId, State) -> Sessions = maps:get(sessions, State, #{}), Data = maps:get(data, State, #{}), MemberCount = maps:get(member_count, State, undefined), VoiceStates = maps:get(voice_states, State, #{}), send_passive_updates( GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount - ), - State. + ). -spec passive_sync_state(guild_state(), map(), map()) -> guild_state(). passive_sync_state(State, Data, VoiceStates) -> State#{data => Data, voice_states => VoiceStates}. +-spec is_large_guild(term()) -> boolean(). +is_large_guild(MemberCount) -> + is_integer(MemberCount) andalso MemberCount > ?LARGE_GUILD_MEMBER_COUNT. + -spec send_passive_updates(integer(), map(), guild_state(), non_neg_integer() | undefined) -> ok. send_passive_updates(GuildId, Sessions, State, MemberCount) -> Data = maps:get(data, State, #{}), Channels = guild_data_index:channel_list(Data), - IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250, + IsLargeGuild = is_large_guild(MemberCount), PassiveSessions = maps:filter( fun(_SessionId, SessionData) -> IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData) @@ -327,3 +341,129 @@ snowflake_binary(FieldName, Value) -> {ok, Id} -> integer_to_binary(Id); {error, _, _} -> undefined end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +is_large_guild_matches_previous_inline_threshold_test() -> + lists:foreach( + fun(MemberCount) -> + Expected = is_integer(MemberCount) andalso MemberCount > 250, + ?assertEqual(Expected, is_large_guild(MemberCount)) + end, + [undefined, 0, 1, 249, 250, 251, 1000000] + ). + +skipped_spawn_drops_no_session_the_old_filter_kept_test() -> + GuildId = 4242, + lists:foreach( + fun({MemberCount, Sessions}) -> + Kept = reference_passive_sessions(GuildId, Sessions, MemberCount), + ?assert(is_large_guild(MemberCount) orelse map_size(Kept) =:= 0) + end, + passive_gate_cases() + ), + Large = reference_passive_sessions(GuildId, passive_test_sessions(), 251), + ?assertEqual(1, map_size(Large)). + +reference_passive_sessions(GuildId, Sessions, MemberCount) -> + IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250, + maps:filter( + fun(_SessionId, SessionData) -> + IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData) + end, + Sessions + ). + +passive_gate_cases() -> + Sessions = passive_test_sessions(), + [ + {undefined, #{}}, + {undefined, Sessions}, + {0, Sessions}, + {250, Sessions}, + {251, Sessions}, + {251, #{}}, + {not_a_count, Sessions} + ]. + +passive_test_sessions() -> + passive_test_sessions(<<"passive-session">>). + +passive_test_sessions(SessionId) -> + #{ + SessionId => #{ + session_id => SessionId, + user_id => 1130650140672000000, + pid => self(), + active_guilds => sets:new(), + bot => false + } + }. + +handle_passive_sync_stays_silent_below_threshold_test() -> + flush_passive_dispatches(), + ok = passive_sync_registry:init(), + GuildId = 1427764661718740995, + SessionId = <<"small-guild-session">>, + State = passive_sync_test_state(GuildId, SessionId, 250), + ?assertMatch({noreply, State}, handle_passive_sync(State)), + ?assertEqual(no_dispatch, receive_passive_dispatch(200)), + RegState = passive_sync_registry:lookup(SessionId, GuildId), + ?assertEqual(#{}, maps:get(previous_passive_updates, RegState)). + +handle_passive_sync_still_dispatches_above_threshold_test() -> + flush_passive_dispatches(), + ok = passive_sync_registry:init(), + GuildId = 1427764661718740996, + SessionId = <<"large-guild-session">>, + ChannelId = passive_test_channel_id(), + MessageId = passive_test_message_id(), + State = passive_sync_test_state(GuildId, SessionId, 251), + ?assertMatch({noreply, State}, handle_passive_sync(State)), + Payload = receive_passive_dispatch(5000), + ?assertEqual( + #{integer_to_binary(ChannelId) => integer_to_binary(MessageId)}, + maps:get(<<"channels">>, Payload) + ). + +passive_sync_test_state(GuildId, SessionId, MemberCount) -> + UserId = 1130650140672000000, + Channel = #{ + <<"id">> => passive_test_channel_id(), + <<"last_message_id">> => passive_test_message_id() + }, + #{ + id => GuildId, + member_count => MemberCount, + sessions => passive_test_sessions(SessionId), + data => #{ + <<"guild">> => #{<<"id">> => GuildId, <<"owner_id">> => UserId}, + <<"channels">> => [Channel], + <<"members">> => [#{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => []}], + <<"roles">> => [] + }, + voice_states => #{} + }. + +passive_test_channel_id() -> + 1497639278555484216. + +passive_test_message_id() -> + 1500000000000000000. + +receive_passive_dispatch(Timeout) -> + receive + {'$gen_cast', {dispatch, passive_updates, Payload}} when is_map(Payload) -> + Payload + after Timeout -> + no_dispatch + end. + +flush_passive_dispatches() -> + case receive_passive_dispatch(0) of + no_dispatch -> ok; + _ -> flush_passive_dispatches() + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_permission_cache.erl b/fluxer_gateway/src/guild/guild_permission_cache.erl index a19a3a7ab..52ee39735 100644 --- a/fluxer_gateway/src/guild/guild_permission_cache.erl +++ b/fluxer_gateway/src/guild/guild_permission_cache.erl @@ -30,6 +30,7 @@ -export_type([guild_id/0, user_id/0, channel_id/0, guild_state/0, guild_data/0]). -define(TABLE, guild_permission_cache). +-define(STRIPPED_MEMBERS_MEMO, guild_permission_cache_stripped_members). -spec put_state(guild_state()) -> ok. put_state(State) when is_map(State) -> @@ -134,7 +135,7 @@ ensure_table() -> -spec strip_data(guild_data()) -> guild_data(). strip_data(Data) when is_map(Data) -> Guild = strip_guild(maps:get(<<"guild">>, Data, #{})), - Members = strip_members(maps:get(<<"members">>, Data, #{})), + Members = memoised_strip_members(maps:get(<<"members">>, Data, #{})), Roles = strip_roles(maps:get(<<"roles">>, Data, [])), Channels = strip_channels(maps:get(<<"channels">>, Data, [])), ChannelIndex = strip_channel_index(maps:get(<<"channel_index">>, Data, #{})), @@ -170,6 +171,54 @@ strip_guild(Guild) when is_map(Guild) -> strip_guild(_) -> #{}. +%% Keyed on the IDENTITY of the source term, never on its value: a hit means this process +%% already stripped that exact term, and strip_members/1 is pure, so the stored result is +%% that term's result. Every write to a member map yields a new term and so cannot hit, and +%% only a completed strip is stored, so a term whose strip raises still raises. +-spec memoised_strip_members(term()) -> map(). +memoised_strip_members(Members) -> + case strip_members_memo_enabled() of + true -> strip_members_memoised(Members); + false -> strip_members_unmemoised(Members) + end. + +-spec strip_members_unmemoised(term()) -> map(). +strip_members_unmemoised(Members) -> + _ = erlang:erase(?STRIPPED_MEMBERS_MEMO), + strip_members(Members). + +-spec strip_members_memoised(term()) -> map(). +strip_members_memoised(Members) -> + case erlang:get(?STRIPPED_MEMBERS_MEMO) of + {Source, Stripped} when is_map(Stripped) -> + reuse_stripped_members(Members, Source, Stripped); + _ -> + store_stripped_members(Members) + end. + +-spec reuse_stripped_members(term(), term(), map()) -> map(). +reuse_stripped_members(Members, Source, Stripped) -> + case erts_debug:same(Members, Source) of + true -> Stripped; + false -> store_stripped_members(Members) + end. + +%% Dropped before the new one is built so a member map that shares nothing with the +%% previous one, a wholesale rebuild, is never held twice on the guild heap at once. +-spec store_stripped_members(term()) -> map(). +store_stripped_members(Members) -> + _ = erlang:erase(?STRIPPED_MEMBERS_MEMO), + Stripped = strip_members(Members), + _ = erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, Stripped}), + Stripped. + +-spec strip_members_memo_enabled() -> boolean(). +strip_members_memo_enabled() -> + case application:get_env(fluxer_gateway, permission_cache_strip_memo_enabled, true) of + false -> false; + _ -> true + end. + -spec strip_members(map() | list() | term()) -> map(). strip_members(Members) when is_map(Members) -> maps:map(fun(_UserId, Member) -> strip_member(Member) end, Members); @@ -191,22 +240,30 @@ strip_member_entry(_, Acc) -> strip_member(Member) when is_map(Member) -> User = maps:get(<<"user">>, Member, #{}), StrippedUser = strip_user(User), - Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])), + Roles = strip_role_ids(maps:get(<<"roles">>, Member, [])), Base = #{ <<"user">> => StrippedUser, <<"roles">> => Roles }, - copy_optional_member_fields([<<"communication_disabled_until">>], Member, Base); + copy_if_present(<<"communication_disabled_until">>, Member, Base); strip_member(_) -> #{}. --spec copy_optional_member_fields([binary()], map(), map()) -> map(). -copy_optional_member_fields(Keys, Source, Acc) -> - lists:foldl( - fun(Key, Current) -> copy_if_present(Key, Source, Current) end, - Acc, - Keys - ). +%% snowflake_id:parse/1 returns a positive integer unchanged, so a roles list that already +%% holds only those is its own parse_list/1 result and does not need to be rebuilt. +-spec strip_role_ids(term()) -> list(). +strip_role_ids(Roles) when is_list(Roles) -> + case all_role_ids_parsed(Roles) of + true -> Roles; + false -> snowflake_id:parse_list(Roles) + end; +strip_role_ids(Roles) -> + snowflake_id:parse_list(Roles). + +-spec all_role_ids_parsed(term()) -> boolean(). +all_role_ids_parsed([Id | Rest]) when is_integer(Id), Id > 0 -> all_role_ids_parsed(Rest); +all_role_ids_parsed([]) -> true; +all_role_ids_parsed(_) -> false. -spec copy_if_present(binary(), map(), map()) -> map(). copy_if_present(Key, Source, Current) -> @@ -327,4 +384,263 @@ strip_member_preserves_communication_disabled_until_test() -> ok = delete(GuildId) end. +strip_member_matches_reference_implementation_test() -> + lists:foreach( + fun(Member) -> + ?assertEqual(reference_strip_member(Member), strip_member(Member)) + end, + strip_member_cases() + ). + +strip_member_cases() -> + Until = <<"2026-05-09T22:00:00.000Z">>, + User = #{<<"id">> => 5, <<"username">> => <<"dropped">>}, + [ + #{<<"user">> => User, <<"roles">> => [7, 9]}, + #{<<"user">> => User, <<"roles">> => [<<"7">>, 9]}, + #{<<"user">> => User, <<"roles">> => [<<"7">>, <<"9">>]}, + #{<<"user">> => User, <<"roles">> => []}, + #{<<"user">> => User}, + #{<<"roles">> => [7]}, + #{<<"user">> => User, <<"roles">> => [7], <<"communication_disabled_until">> => Until}, + #{<<"user">> => User, <<"roles">> => [7], <<"nick">> => <<"dropped">>}, + #{<<"user">> => #{}, <<"roles">> => undefined}, + #{<<"user">> => not_a_map, <<"roles">> => null}, + not_a_member + ]. + +reference_strip_member(Member) when is_map(Member) -> + StrippedUser = strip_user(maps:get(<<"user">>, Member, #{})), + Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])), + Base = #{<<"user">> => StrippedUser, <<"roles">> => Roles}, + lists:foldl( + fun(Key, Current) -> copy_if_present(Key, Member, Current) end, + Base, + [<<"communication_disabled_until">>] + ); +reference_strip_member(_) -> + #{}. + +strip_member_rejects_unparsed_role_ids_test() -> + ?assertError({invalid_snowflake, 0}, strip_member(#{<<"roles">> => [0]})), + ?assertError({invalid_snowflake, -3}, strip_member(#{<<"roles">> => [-3]})), + ?assertError({invalid_snowflake, <<"0">>}, strip_member(#{<<"roles">> => [<<"0">>]})), + ?assertError({invalid_snowflake, bad}, strip_member(#{<<"roles">> => [7, bad]})), + ?assertError({invalid_snowflake_list, oops}, strip_member(#{<<"roles">> => oops})). + +strip_members_keeps_already_parsed_role_ids_test() -> + GuildId = 905, + UserId = 906, + Member = #{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => [42, 43]}, + Data = #{ + <<"guild">> => #{<<"owner_id">> => <<"1">>}, + <<"roles">> => [], + <<"members">> => #{UserId => Member}, + <<"channels">> => [] + }, + ok = put_data(GuildId, Data), + try + {ok, #{} = MemberData} = get_member(GuildId, UserId), + ?assertEqual([42, 43], maps:get(<<"roles">>, MemberData)), + ?assertEqual(#{<<"id">> => UserId}, maps:get(<<"user">>, MemberData)) + after + ok = delete(GuildId) + end. + +memo_reset() -> + erlang:erase(?STRIPPED_MEMBERS_MEMO). + +memo_member(Id, Roles) -> + #{ + <<"user">> => #{<<"id">> => integer_to_binary(Id), <<"username">> => <<"dropped">>}, + <<"roles">> => Roles, + <<"nick">> => <<"dropped">> + }. + +memo_members(Ids) -> + maps:from_list([{Id, memo_member(Id, [<<"42">>])} || Id <- Ids]). + +%% strip_members/1 as it read before the memo, over the pre-shortcut strip_member/1. +reference_strip_members(Members) when is_map(Members) -> + maps:map(fun(_UserId, Member) -> reference_strip_member(Member) end, Members); +reference_strip_members(Members) when is_list(Members) -> + lists:foldl(fun reference_strip_member_entry/2, #{}, Members); +reference_strip_members(_) -> + #{}. + +reference_strip_member_entry(Member, Acc) when is_map(Member) -> + case get_member_user_id(Member) of + undefined -> Acc; + UserId -> Acc#{UserId => reference_strip_member(Member)} + end; +reference_strip_member_entry(_, Acc) -> + Acc. + +assert_memo_matches_reference(Members) -> + Expected = reference_strip_members(Members), + ?assertEqual(Expected, memoised_strip_members(Members)), + ?assertEqual(Expected, memoised_strip_members(Members)). + +memo_members_cases() -> + [ + #{}, + memo_members([1]), + memo_members([1, 2, 3]), + #{7 => memo_member(7, [])}, + [memo_member(4, [<<"42">>]), memo_member(5, [7, 9])], + [memo_member(6, []), not_a_member], + [], + not_a_member_map, + undefined + ]. + +memoised_strip_members_matches_reference_test() -> + memo_reset(), + try + lists:foreach(fun assert_memo_matches_reference/1, memo_members_cases()) + after + memo_reset() + end. + +memo_reuses_the_stripped_term_for_an_identical_members_term_test() -> + memo_reset(), + try + Members = memo_members([1, 2]), + First = memoised_strip_members(Members), + Second = memoised_strip_members(Members), + ?assertEqual(reference_strip_members(Members), First), + ?assert(erts_debug:same(First, Second)), + ?assertEqual({Members, First}, erlang:get(?STRIPPED_MEMBERS_MEMO)) + after + memo_reset() + end. + +memo_recomputes_for_an_equal_but_distinct_members_term_test() -> + memo_reset(), + try + Members = memo_members([1, 2]), + Copy = binary_to_term(term_to_binary(Members)), + Stripped = memoised_strip_members(Members), + ?assertNot(erts_debug:same(Members, Copy)), + ?assertEqual(Stripped, memoised_strip_members(Copy)), + {Source, _Stripped} = erlang:get(?STRIPPED_MEMBERS_MEMO), + ?assert(erts_debug:same(Copy, Source)) + after + memo_reset() + end. + +memo_changed_member_maps(Members) -> + [ + Members#{2 => memo_member(2, [<<"7">>])}, + Members#{3 => memo_member(3, [<<"42">>])}, + maps:remove(1, Members), + maps:map(fun(Id, _Member) -> memo_member(Id, []) end, Members) + ]. + +memo_follows_every_change_to_the_member_map_test() -> + memo_reset(), + try + Members = memo_members([1, 2]), + _Stripped = memoised_strip_members(Members), + lists:foreach( + fun assert_memo_matches_reference/1, + memo_changed_member_maps(Members) + ) + after + memo_reset() + end. + +memo_never_suppresses_an_invalid_role_id_test() -> + memo_reset(), + try + Good = memo_members([1]), + Stripped = memoised_strip_members(Good), + Bad = #{2 => memo_member(2, [bad])}, + ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)), + ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO)), + ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)), + ?assertEqual(Stripped, memoised_strip_members(Good)), + ?assertEqual({Good, Stripped}, erlang:get(?STRIPPED_MEMBERS_MEMO)) + after + memo_reset() + end. + +memo_ignores_an_unrecognised_entry_test() -> + memo_reset(), + try + Members = memo_members([1]), + Expected = reference_strip_members(Members), + erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, junk}), + ?assertEqual(Expected, memoised_strip_members(Members)), + erlang:put(?STRIPPED_MEMBERS_MEMO, stale_shape), + ?assertEqual(Expected, memoised_strip_members(Members)), + ?assertEqual({Members, Expected}, erlang:get(?STRIPPED_MEMBERS_MEMO)) + after + memo_reset() + end. + +memo_disabled_matches_reference_and_releases_the_memo_test() -> + memo_reset(), + Members = memo_members([1, 2]), + Expected = reference_strip_members(Members), + ?assertEqual(Expected, memoised_strip_members(Members)), + application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false), + try + ?assertEqual(Expected, memoised_strip_members(Members)), + ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO)) + after + application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled), + memo_reset() + end. + +%% strip_data/1 with the memo off is the computation this module ran before the memo. +unmemoised_strip_data(Data) -> + application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false), + try + strip_data(Data) + after + application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled) + end. + +memo_test_data(OwnerId) -> + guild_data_index:normalize_data(#{ + <<"guild">> => #{<<"owner_id">> => OwnerId}, + <<"roles">> => [#{<<"id">> => <<"42">>, <<"permissions">> => <<"0">>}], + <<"members">> => [memo_member(1, [<<"42">>]), memo_member(2, [])], + <<"channels">> => [#{<<"id">> => <<"500">>, <<"permission_overwrites">> => []}] + }). + +strip_data_reuses_the_stripped_members_across_a_non_member_change_test() -> + memo_reset(), + Data = memo_test_data(<<"1">>), + Updated = Data#{<<"guild">> => #{<<"owner_id">> => <<"9">>}}, + Expected = unmemoised_strip_data(Data), + ExpectedUpdated = unmemoised_strip_data(Updated), + memo_reset(), + try + ?assertEqual(Expected, strip_data(Data)), + ?assertEqual(ExpectedUpdated, strip_data(Updated)), + MembersA = maps:get(<<"members">>, strip_data(Data)), + MembersB = maps:get(<<"members">>, strip_data(Updated)), + ?assert(erts_debug:same(MembersA, MembersB)) + after + memo_reset() + end. + +put_normalized_data_writes_the_same_snapshot_with_a_warm_memo_test() -> + GuildId = 907, + memo_reset(), + Data = memo_test_data(<<"1">>), + try + ok = put_normalized_data(GuildId, Data), + {ok, First} = get_snapshot(GuildId), + ok = put_normalized_data(GuildId, Data), + {ok, Second} = get_snapshot(GuildId), + ?assertEqual(First, Second), + ?assertEqual(#{id => GuildId, data => unmemoised_strip_data(Data)}, Second) + after + memo_reset(), + ok = delete(GuildId) + end. + -endif. diff --git a/fluxer_gateway/src/guild/guild_presence.erl b/fluxer_gateway/src/guild/guild_presence.erl index 07719adab..dd06a85c0 100644 --- a/fluxer_gateway/src/guild/guild_presence.erl +++ b/fluxer_gateway/src/guild/guild_presence.erl @@ -14,8 +14,13 @@ -type member() :: map(). -type user_id() :: integer(). --define(HEAVY_MEMBER_DATA_KEYS, [<<"members">>, members_normalized, <<"member_role_index">>]). +%% members_sorted_ids trims with the member map it indexes: a snapshot that kept it would +%% answer sorted_member_ids/2 with ids for members the snapshot no longer carries. +-define(HEAVY_MEMBER_DATA_KEYS, [ + <<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids +]). -define(PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULT, 5000). +-define(SESSIONS_PROJECTION_CACHE, presence_snapshot_sessions_cache). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -112,12 +117,17 @@ spawn_presence_broadcast(UserId, OldPresence, PresenceMap, OldState, NewState) - PresenceMap ), {Pid, NewState2} = guild_broadcaster:ensure(NewState1), - NewSnap = build_broadcast_snapshot(NewState2), + {NewSnap, NewState3} = build_broadcast_snapshot_cached(NewState2), guild_broadcaster:cast_presence(Pid, UserId, PresenceMap, #{}, NewSnap), - NewState2. + NewState3. -spec build_broadcast_snapshot(guild_state()) -> map(). build_broadcast_snapshot(State) -> + {Snapshot, _State} = build_broadcast_snapshot_cached(State), + Snapshot. + +-spec build_broadcast_snapshot_cached(guild_state()) -> {map(), guild_state()}. +build_broadcast_snapshot_cached(State) -> maybe_trim_snapshot(build_base_snapshot(State), State). -spec build_base_snapshot(guild_state()) -> map(). @@ -139,11 +149,11 @@ build_base_snapshot(State) -> Keys ). --spec maybe_trim_snapshot(map(), guild_state()) -> map(). +-spec maybe_trim_snapshot(map(), guild_state()) -> {map(), guild_state()}. maybe_trim_snapshot(Snapshot, State) -> case should_trim_snapshot(State) of - true -> trim_snapshot(Snapshot); - false -> Snapshot + true -> trim_snapshot(Snapshot, State); + false -> {Snapshot, State} end. -spec should_trim_snapshot(guild_state()) -> boolean(). @@ -179,9 +189,9 @@ presence_snapshot_trim_member_threshold() -> _ -> ?PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULT end. --spec trim_snapshot(map()) -> map(). -trim_snapshot(Snapshot) -> - trim_snapshot_sessions(trim_snapshot_data(Snapshot)). +-spec trim_snapshot(map(), guild_state()) -> {map(), guild_state()}. +trim_snapshot(Snapshot, State) -> + trim_snapshot_sessions(trim_snapshot_data(Snapshot), State). -spec trim_snapshot_data(map()) -> map(). trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) -> @@ -189,15 +199,33 @@ trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) -> trim_snapshot_data(Snapshot) -> Snapshot. --spec trim_snapshot_sessions(map()) -> map(). -trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) -> - Projected = maps:map( +-spec trim_snapshot_sessions(map(), guild_state()) -> {map(), guild_state()}. +trim_snapshot_sessions(#{sessions := Sessions} = Snapshot, State) when is_map(Sessions) -> + {Projected, NewState} = cached_projected_sessions(Sessions, State), + {Snapshot#{sessions => Projected}, NewState}; +trim_snapshot_sessions(Snapshot, State) -> + {Snapshot, State}. + +%% Keyed by the sessions map itself, so every writer of that map invalidates the +%% projection without having to know this cache exists. +-spec cached_projected_sessions(map(), guild_state()) -> {map(), guild_state()}. +cached_projected_sessions(Sessions, State) -> + case maps:get(?SESSIONS_PROJECTION_CACHE, State, undefined) of + {Sessions, Projected} when is_map(Projected) -> {Projected, State}; + _ -> store_projected_sessions(Sessions, State) + end. + +-spec store_projected_sessions(map(), guild_state()) -> {map(), guild_state()}. +store_projected_sessions(Sessions, State) -> + Projected = project_sessions(Sessions), + {Projected, State#{?SESSIONS_PROJECTION_CACHE => {Sessions, Projected}}}. + +-spec project_sessions(map()) -> map(). +project_sessions(Sessions) -> + maps:map( fun(_SessionId, SessionData) -> project_session(SessionData) end, Sessions - ), - Snapshot#{sessions => Projected}; -trim_snapshot_sessions(Snapshot) -> - Snapshot. + ). -spec project_session(term()) -> term(). project_session(SessionData) when is_map(SessionData) -> @@ -440,6 +468,7 @@ snapshot_trim_test_state(Tab) -> <<"members">> => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}}, members_normalized => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}}, <<"member_role_index">> => #{1 => []}, + members_sorted_ids => [1], <<"channels">> => [], <<"roles">> => [] }, @@ -463,6 +492,7 @@ build_broadcast_snapshot_trims_by_default_test() -> ?assertNot(maps:is_key(<<"members">>, SnapData)), ?assertNot(maps:is_key(members_normalized, SnapData)), ?assertNot(maps:is_key(<<"member_role_index">>, SnapData)), + ?assertNot(maps:is_key(members_sorted_ids, SnapData)), ?assertEqual(Tab, maps:get(members_ets, SnapData)), ?assert(maps:is_key(<<"channels">>, SnapData)), SnapSession = maps:get(<<"s1">>, maps:get(sessions, Snap)), @@ -499,4 +529,149 @@ build_broadcast_snapshot_no_trim_below_threshold_test() -> ets:delete(Tab) end. +reference_project_session(SessionData) when is_map(SessionData) -> + maps:with([user_id, pid, viewable_channels], SessionData); +reference_project_session(SessionData) -> + SessionData. + +reference_project_sessions(Sessions) -> + maps:map( + fun(_SessionId, SessionData) -> reference_project_session(SessionData) end, + Sessions + ). + +reference_trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) -> + Snapshot#{sessions => reference_project_sessions(Sessions)}; +reference_trim_snapshot_sessions(Snapshot) -> + Snapshot. + +reference_maybe_trim_snapshot(true, Snapshot) -> + reference_trim_snapshot_sessions(trim_snapshot_data(Snapshot)); +reference_maybe_trim_snapshot(false, Snapshot) -> + Snapshot. + +reference_broadcast_snapshot(State) -> + reference_maybe_trim_snapshot(should_trim_snapshot(State), build_base_snapshot(State)). + +cache_test_session(UserId, ViewableChannels) -> + #{ + user_id => UserId, + pid => self(), + viewable_channels => ViewableChannels, + active_guilds => [42], + pending_connect => false + }. + +cache_test_state(Sessions) -> + #{ + id => 42, + member_count => 10, + data => #{<<"channels">> => [], <<"members">> => #{}}, + sessions => Sessions + }. + +assert_cached_snapshot_matches_reference(Sessions, State) -> + NextState = State#{sessions => Sessions}, + {Snapshot, CachedState} = build_broadcast_snapshot_cached(NextState), + ?assertEqual(reference_broadcast_snapshot(NextState), Snapshot), + ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, Snapshot)), + CachedState. + +session_projection_mutation_sequence() -> + A = cache_test_session(1, #{100 => true}), + B = cache_test_session(2, #{100 => true, 200 => true}), + S1 = #{<<"a">> => A}, + S2 = S1#{<<"b">> => B}, + S3 = S2#{<<"a">> => cache_test_session(1, #{100 => true, 300 => true})}, + S4 = maps:remove(<<"b">>, S3), + S5 = S4#{<<"a">> => (maps:get(<<"a">>, S4))#{pending_connect => true}}, + S6 = S5#{<<"c">> => not_a_map}, + [#{}, S1, S1, S2, S3, S4, S5, S6, S1, #{}]. + +cached_projection_matches_reference_across_mutations_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2), + try + lists:foldl( + fun assert_cached_snapshot_matches_reference/2, + cache_test_state(#{}), + session_projection_mutation_sequence() + ) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold) + end. + +cached_projection_is_reused_when_sessions_unchanged_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2), + try + Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})}, + {Snap1, State1} = build_broadcast_snapshot_cached(cache_test_state(Sessions)), + {Snap2, State2} = build_broadcast_snapshot_cached(State1), + Projected1 = maps:get(sessions, Snap1), + Projected2 = maps:get(sessions, Snap2), + ?assertEqual(reference_project_sessions(Sessions), Projected1), + ?assertEqual(Projected1, Projected2), + ?assert(erts_debug:same(Projected1, Projected2)), + ?assertEqual({Sessions, Projected1}, maps:get(?SESSIONS_PROJECTION_CACHE, State2)) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold) + end. + +cached_projection_ignores_foreign_cache_entry_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2), + try + Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})}, + Stale = #{<<"z">> => cache_test_session(9, #{999 => true})}, + Foreign = {Stale, reference_project_sessions(Stale)}, + State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => Foreign}, + {Snapshot, _NewState} = build_broadcast_snapshot_cached(State), + ?assertEqual(reference_broadcast_snapshot(State), Snapshot), + ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot)) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold) + end. + +cached_projection_ignores_corrupt_cache_entry_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2), + try + Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})}, + State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => {Sessions, junk}}, + {Snapshot, _NewState} = build_broadcast_snapshot_cached(State), + ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot)) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold) + end. + +cached_projection_not_stored_when_trim_disabled_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_enabled, false), + try + Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})}, + State = cache_test_state(Sessions), + {Snapshot, NewState} = build_broadcast_snapshot_cached(State), + ?assertEqual(reference_broadcast_snapshot(State), Snapshot), + ?assertEqual(Sessions, maps:get(sessions, Snapshot)), + ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, NewState)) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_enabled) + end. + +handle_bus_presence_threads_projection_cache_test() -> + application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2), + Sessions = #{<<"s1">> => cache_test_session(1, #{100 => true})}, + State = (presence_test_state())#{member_count => 10, sessions => Sessions}, + Payload = #{ + <<"status">> => <<"online">>, + <<"mobile">> => true, + <<"afk">> => false, + <<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"Alpha">>} + }, + try + {noreply, NewState} = handle_bus_presence(1, Payload, State), + ?assertEqual( + {Sessions, reference_project_sessions(Sessions)}, + maps:get(?SESSIONS_PROJECTION_CACHE, NewState) + ) + after + application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold) + end. + -endif. diff --git a/fluxer_gateway/src/guild/guild_public_online.erl b/fluxer_gateway/src/guild/guild_public_online.erl index c118903d3..894975d6c 100644 --- a/fluxer_gateway/src/guild/guild_public_online.erl +++ b/fluxer_gateway/src/guild/guild_public_online.erl @@ -15,6 +15,8 @@ -type channel() :: map(). -type user_id() :: integer(). +-define(ALL_PERMS_VIEWER_KEY, public_online_all_perms_viewer). + -spec compute_count(guild_state()) -> non_neg_integer(). compute_count(State) -> case resolve_data(State) of @@ -57,7 +59,15 @@ classify_everyone_viewable(Channels, State) -> case {OpenAcc, RestrictedAcc} of {[], []} -> empty; {[_ | _], _} -> {open, OpenAcc}; - {[], _} -> {restricted, sets:from_list(RestrictedAcc)} + {[], _} -> classify_restricted(RestrictedAcc) + end. + +-spec classify_restricted([integer()]) -> + {open, [integer()]} | {restricted, sets:set(integer())}. +classify_restricted(RestrictedIds) -> + case application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, false) of + true -> {open, RestrictedIds}; + _ -> {restricted, sets:from_list(RestrictedIds)} end. -spec classify_channel_visibility( @@ -200,9 +210,66 @@ build_state(Channels, Members, Roles, Presences) -> <<"channels">> => Channels }, member_presence => Tab, + member_list_engine => build_engine(Presences), sessions => #{} }. +build_engine(Presences) -> + Online = length([S || {_UserId, S} <- Presences, S =/= <<"offline">>, S =/= <<"invisible">>]), + Ref = ets:new(test_member_list_engine, [set, public]), + ets:insert(Ref, [{total_count, length(Presences)}, {online_count, Online}]), + Ref. + +deny_view_overwrite(Id, Type) -> + #{ + <<"id">> => integer_to_binary(Id), + <<"type">> => Type, + <<"allow">> => <<"0">>, + <<"deny">> => integer_to_binary(view_perm()) + }. + +channel(ChannelId, Overwrites) -> + #{ + <<"id">> => integer_to_binary(ChannelId), + <<"type">> => 0, + <<"permission_overwrites">> => Overwrites + }. + +role(RoleId) -> + #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}. + +with_all_perms_viewer(Fun) -> + Previous = application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY), + application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, true), + try + Fun() + after + restore_all_perms_viewer(Previous) + end. + +restore_all_perms_viewer(undefined) -> + application:unset_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY); +restore_all_perms_viewer({ok, Value}) -> + application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, Value). + +verify_gate_state() -> + VerifiedRoleId = 4242, + Channels = [channel(400, [deny_view_overwrite(VerifiedRoleId, 0)])], + Members = [{1, [VerifiedRoleId]}, {2, [VerifiedRoleId]}, {3, [VerifiedRoleId]}, {4, []}], + Presences = [ + {1, <<"online">>}, + {2, <<"online">>}, + {3, <<"idle">>}, + {4, <<"online">>} + ], + build_state(Channels, Members, [role(VerifiedRoleId)], Presences). + +user_deny_state() -> + Channels = [channel(700, [deny_view_overwrite(42, 1)])], + Members = [{41, []}, {42, []}, {43, []}], + Presences = [{41, <<"online">>}, {42, <<"online">>}, {43, <<"online">>}], + build_state(Channels, Members, [], Presences). + returns_zero_when_no_channels_test() -> State = build_state([], [], [], []), ?assertEqual(0, compute_count(State)). @@ -303,4 +370,36 @@ channel_has_view_restricting_overrides_no_overwrites_test() -> Channel = #{<<"permission_overwrites">> => []}, ?assertNot(channel_has_view_restricting_overrides(Channel, view_perm())). +verify_gate_collapses_count_without_flag_test() -> + ?assertEqual(1, compute_count(verify_gate_state())). + +verify_gate_reports_full_count_with_flag_test() -> + State = verify_gate_state(), + ?assertEqual(4, with_all_perms_viewer(fun() -> compute_count(State) end)). + +everyone_deny_channel_stays_excluded_with_flag_test() -> + GuildId = guild_id(), + Channels = [channel(500, [deny_view_overwrite(GuildId, 0)])], + State = build_state(Channels, [{1, []}, {2, []}], [], [{1, <<"online">>}, {2, <<"idle">>}]), + ?assertEqual(0, compute_count(State)), + ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)). + +no_channels_stays_zero_with_flag_test() -> + State = build_state([], [{1, []}], [], [{1, <<"online">>}]), + ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)). + +plain_open_channel_unchanged_by_flag_test() -> + Members = [{1, []}, {2, []}, {3, []}], + Presences = [{1, <<"online">>}, {2, <<"idle">>}, {3, <<"offline">>}], + State = build_state([channel(600, [])], Members, [], Presences), + ?assertEqual(2, compute_count(State)), + ?assertEqual(2, with_all_perms_viewer(fun() -> compute_count(State) end)). + +user_deny_collapses_count_without_flag_test() -> + ?assertEqual(2, compute_count(user_deny_state())). + +user_deny_reports_full_count_with_flag_test() -> + State = user_deny_state(), + ?assertEqual(3, with_all_perms_viewer(fun() -> compute_count(State) end)). + -endif. diff --git a/fluxer_gateway/src/guild/guild_sessions.erl b/fluxer_gateway/src/guild/guild_sessions.erl index 95a5035e2..63e3ea108 100644 --- a/fluxer_gateway/src/guild/guild_sessions.erl +++ b/fluxer_gateway/src/guild/guild_sessions.erl @@ -28,6 +28,8 @@ build_viewable_channel_map/1 ]). +-define(MAX_PERM_MEMO_ENTRIES, 8192). + -type guild_state() :: map(). -type session_id() :: binary(). -type user_id() :: integer(). @@ -36,6 +38,8 @@ -type session_data() :: map(). -type sessions_map() :: #{session_id() => session_data()}. -type session_pair() :: {session_id(), session_data()}. +-type perm_memo() :: #{user_id() => non_neg_integer()}. +-type message_ctx() :: {channel_id(), binary(), session_id() | undefined, guild_state()}. -export_type([ guild_state/0, session_id/0, @@ -59,7 +63,116 @@ handle_session_connect(Request, Pid, State) -> -spec handle_session_down(reference(), guild_state()) -> {noreply, guild_state()} | {stop, normal, guild_state()}. handle_session_down(Ref, State) -> - guild_sessions_connect:handle_session_down(Ref, State). + case pending_session_by_ref(Ref, State) of + {ok, SessionId, Session, Sessions} -> + handle_pending_ref_down(SessionId, Session, Ref, Sessions, State); + not_found -> + guild_sessions_connect:handle_session_down(Ref, State) + end. + +-spec handle_pending_ref_down( + session_id(), session_data(), reference(), sessions_map(), guild_state() +) -> {noreply, guild_state()} | {stop, normal, guild_state()}. +handle_pending_ref_down(SessionId, Session, Ref, Sessions, State) -> + case pending_session_owns_connected_tracking(Session, Sessions, State) of + true -> guild_sessions_connect:handle_session_down(Ref, State); + false -> handle_pending_session_down(SessionId, Ref, Sessions, State) + end. + +-spec pending_session_by_ref(reference(), guild_state()) -> + {ok, session_id(), session_data(), sessions_map()} | not_found. +pending_session_by_ref(Ref, State) -> + Sessions = maps:get(sessions, State, #{}), + case session_id_by_ref(Ref, Sessions, State) of + SessionId when is_binary(SessionId) -> pending_session_entry(SessionId, Sessions); + _ -> not_found + end. + +-spec pending_session_entry(session_id(), sessions_map()) -> + {ok, session_id(), session_data(), sessions_map()} | not_found. +pending_session_entry(SessionId, Sessions) -> + case maps:get(SessionId, Sessions, undefined) of + #{pending_connect := true} = Session -> {ok, SessionId, Session, Sessions}; + _ -> not_found + end. + +-spec session_id_by_ref(reference(), sessions_map(), guild_state()) -> session_id() | undefined. +session_id_by_ref(Ref, Sessions, State) -> + Refs = maps:get(guild_session_refs, State, #{}), + case maps:get(Ref, Refs, undefined) of + SessionId when is_binary(SessionId) -> SessionId; + _ -> session_id_by_ref_scan(Ref, Sessions) + end. + +-spec session_id_by_ref_scan(reference(), sessions_map()) -> session_id() | undefined. +session_id_by_ref_scan(Ref, Sessions) -> + maps:fold( + fun(SessionId, Session, Found) -> + match_session_ref(Ref, SessionId, Session, Found) + end, + undefined, + Sessions + ). + +-spec match_session_ref(reference(), session_id(), session_data(), session_id() | undefined) -> + session_id() | undefined. +match_session_ref(Ref, SessionId, #{mref := Ref}, _Found) -> SessionId; +match_session_ref(_Ref, _SessionId, _Session, Found) -> Found. + +-spec pending_session_owns_connected_tracking(session_data(), sessions_map(), guild_state()) -> + boolean(). +pending_session_owns_connected_tracking(Session, Sessions, State) -> + UserId = maps:get(user_id, Session, undefined), + Counts = maps:get(user_session_counts, State, #{}), + TrackedCount = non_negative_count(maps:get(UserId, Counts, 0)), + TrackedCount > active_session_count(UserId, Sessions). + +-spec non_negative_count(term()) -> non_neg_integer(). +non_negative_count(Count) when is_integer(Count), Count >= 0 -> Count; +non_negative_count(_) -> 0. + +-spec active_session_count(user_id() | undefined, sessions_map()) -> non_neg_integer(). +active_session_count(UserId, Sessions) -> + maps:fold( + fun(_SessionId, Session, Count) -> + count_active_session(UserId, Session, Count) + end, + 0, + Sessions + ). + +-spec count_active_session(user_id() | undefined, session_data(), non_neg_integer()) -> + non_neg_integer(). +count_active_session(UserId, #{user_id := UserId} = Session, Count) -> + case maps:get(pending_connect, Session, false) of + true -> Count; + false -> Count + 1 + end; +count_active_session(_UserId, _Session, Count) -> + Count. + +-spec handle_pending_session_down(session_id(), reference(), sessions_map(), guild_state()) -> + {noreply, guild_state()}. +handle_pending_session_down(SessionId, Ref, Sessions, State) -> + NewSessions = maps:remove(SessionId, Sessions), + SessionRefs = maps:get(guild_session_refs, State, #{}), + State1 = State#{ + sessions => NewSessions, + guild_session_refs => maps:remove(Ref, SessionRefs) + }, + State2 = guild_sessions_connect_cleanup:cleanup_connect_admission_for_session( + SessionId, State1 + ), + finish_pending_session_down(NewSessions, State2). + +-spec finish_pending_session_down(sessions_map(), guild_state()) -> {noreply, guild_state()}. +finish_pending_session_down(NewSessions, State) -> + case map_size(NewSessions) of + 0 -> + {noreply, guild_sessions_connect_cleanup:maybe_mark_auto_stop_pending(State)}; + _ -> + {noreply, guild_sessions_connect_cleanup:clear_auto_stop_pending(State)} + end. -spec remove_session(session_id(), guild_state()) -> guild_state(). remove_session(SessionId, State) -> @@ -121,21 +234,68 @@ filter_sessions_for_channel(Sessions, ChannelId, SessionIdOpt, State) -> sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state() ) -> [session_pair()]. filter_sessions_for_message(Sessions, ChannelId, MessageId, SessionIdOpt, State) -> - filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) -> - session_can_view_channel(S, ChannelId, State) andalso - session_can_access_message(S, ChannelId, MessageId, State) - end). + filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State). --spec session_can_access_message(map(), channel_id(), binary(), guild_state()) -> boolean(). -session_can_access_message(SessionData, ChannelId, MessageId, State) -> - case maps:get(user_id, SessionData, undefined) of - UserId when is_integer(UserId) -> - Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State), - guild_permissions:can_access_message_by_permissions(Perms, MessageId, State); - _ -> - false +-spec filter_message_memo( + sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state() +) -> [session_pair()]. +filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State) -> + Ctx = {ChannelId, MessageId, SessionIdOpt, State}, + {Acc, _Memo} = maps:fold( + fun(Sid, S, In) -> collect_message_session(Sid, S, Ctx, In) end, + {[], #{}}, + Sessions + ), + Acc. + +-spec collect_message_session( + session_id(), session_data(), message_ctx(), {[session_pair()], perm_memo()} +) -> {[session_pair()], perm_memo()}. +collect_message_session(Sid, S, Ctx, {Acc, Memo}) -> + {ChannelId, _MessageId, SessionIdOpt, State} = Ctx, + Visible = + not is_pending_or_excluded(Sid, S, SessionIdOpt) andalso + session_can_view_channel(S, ChannelId, State), + case Visible of + true -> memo_message_session(Sid, S, Ctx, Acc, Memo); + false -> {Acc, Memo} end. +-spec memo_message_session( + session_id(), session_data(), message_ctx(), [session_pair()], perm_memo() +) -> {[session_pair()], perm_memo()}. +memo_message_session(Sid, S, {ChannelId, MessageId, _SessionIdOpt, State}, Acc, Memo) -> + case maps:get(user_id, S, undefined) of + UserId when is_integer(UserId) -> + {Perms, Memo1} = memo_member_permissions(UserId, ChannelId, State, Memo), + Ok = guild_permissions:can_access_message_by_permissions(Perms, MessageId, State), + {prepend_session(Ok, Sid, S, Acc), Memo1}; + _ -> + {Acc, Memo} + end. + +-spec memo_member_permissions(user_id(), channel_id(), guild_state(), perm_memo()) -> + {non_neg_integer(), perm_memo()}. +memo_member_permissions(UserId, ChannelId, State, Memo) -> + case maps:get(UserId, Memo, undefined) of + Perms when is_integer(Perms) -> + {Perms, Memo}; + _ -> + Computed = guild_permissions:get_member_permissions(UserId, ChannelId, State), + {Computed, store_perm_memo(UserId, Computed, Memo)} + end. + +-spec store_perm_memo(user_id(), non_neg_integer(), perm_memo()) -> perm_memo(). +store_perm_memo(UserId, Perms, Memo) when map_size(Memo) < ?MAX_PERM_MEMO_ENTRIES -> + Memo#{UserId => Perms}; +store_perm_memo(_UserId, _Perms, Memo) -> + Memo. + +-spec prepend_session(boolean(), session_id(), session_data(), [session_pair()]) -> + [session_pair()]. +prepend_session(true, Sid, S, Acc) -> [{Sid, S} | Acc]; +prepend_session(false, _Sid, _S, Acc) -> Acc. + -spec filter_sessions_for_manage_channels( sessions_map(), channel_id(), session_id() | undefined, guild_state() ) -> [session_pair()]. @@ -341,4 +501,182 @@ filter_active_sessions_with_predicate_test() -> ?assertEqual(1, length(Result)), [{<<"b">>, _}] = Result. +store_perm_memo_test() -> + ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{})), + ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{7 => 42})). + +store_perm_memo_bound_test() -> + Full = maps:from_list([{I, 0} || I <- lists:seq(1, ?MAX_PERM_MEMO_ENTRIES)]), + ?assertEqual(Full, store_perm_memo(0, 1, Full)), + ?assertEqual(?MAX_PERM_MEMO_ENTRIES, map_size(store_perm_memo(0, 1, Full))). + +memo_member_permissions_hit_test() -> + Memo = #{9 => 123}, + ?assertEqual({123, Memo}, memo_member_permissions(9, 5, #{}, Memo)). + +memo_member_permissions_miss_test() -> + ?assertEqual({0, #{9 => 0}}, memo_member_permissions(9, 5, #{}, #{})). + +-spec reference_session_can_access_message(map(), channel_id(), binary(), guild_state()) -> + boolean(). +reference_session_can_access_message(SessionData, ChannelId, MessageId, State) -> + case maps:get(user_id, SessionData, undefined) of + UserId when is_integer(UserId) -> + Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State), + guild_permissions:can_access_message_by_permissions(Perms, MessageId, State); + _ -> + false + end. + +-spec reference_filter_message_direct( + sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state() +) -> [session_pair()]. +reference_filter_message_direct(Sessions, ChannelId, MessageId, SessionIdOpt, State) -> + filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) -> + session_can_view_channel(S, ChannelId, State) andalso + reference_session_can_access_message(S, ChannelId, MessageId, State) + end). + +filter_message_memo_matches_direct_test() -> + State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}}, + Owner1 = #{user_id => 1, viewable_channels => #{5 => true}}, + Owner2 = #{user_id => 1, viewable_channels => #{5 => true}}, + Other = #{user_id => 2, viewable_channels => #{5 => true}}, + Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other}, + Direct = reference_filter_message_direct(Sessions, 5, <<"1">>, undefined, State), + Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State), + ?assertEqual(Direct, Memoized), + ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])). + +filter_message_memo_collects_visible_sessions_test() -> + State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}}, + Owner1 = #{user_id => 1, viewable_channels => #{5 => true}}, + Owner2 = #{user_id => 1, viewable_channels => #{5 => true}}, + Other = #{user_id => 2, viewable_channels => #{5 => true}}, + Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other}, + Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State), + ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])). + +filter_message_memo_skips_excluded_test() -> + State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}}, + S1 = #{user_id => 1, viewable_channels => #{5 => true}}, + S2 = #{user_id => 1, viewable_channels => #{5 => true}, pending_connect => true}, + Sessions = #{<<"a">> => S1, <<"b">> => S2}, + ?assertEqual([], filter_message_memo(Sessions, 5, <<"1">>, <<"a">>, State)). + +filter_sessions_for_message_test() -> + State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}}, + Sessions = #{<<"a">> => #{user_id => 1, viewable_channels => #{5 => true}}}, + ?assertEqual( + [{<<"a">>, maps:get(<<"a">>, Sessions)}], + filter_sessions_for_message(Sessions, 5, <<"1">>, undefined, State) + ). + +non_negative_count_test() -> + ?assertEqual(3, non_negative_count(3)), + ?assertEqual(0, non_negative_count(0)), + ?assertEqual(0, non_negative_count(-1)), + ?assertEqual(0, non_negative_count(undefined)). + +active_session_count_test() -> + Sessions = #{ + <<"a">> => #{user_id => 1, pending_connect => true}, + <<"b">> => #{user_id => 1, pending_connect => false}, + <<"c">> => #{user_id => 1}, + <<"d">> => #{user_id => 2} + }, + ?assertEqual(2, active_session_count(1, Sessions)), + ?assertEqual(1, active_session_count(2, Sessions)), + ?assertEqual(0, active_session_count(3, Sessions)). + +session_id_by_ref_uses_index_test() -> + Ref = make_ref(), + Sessions = #{<<"a">> => #{mref => make_ref()}}, + State = #{guild_session_refs => #{Ref => <<"a">>}}, + ?assertEqual(<<"a">>, session_id_by_ref(Ref, Sessions, State)). + +session_id_by_ref_scan_fallback_test() -> + Ref = make_ref(), + Sessions = #{<<"a">> => #{mref => make_ref()}, <<"b">> => #{mref => Ref}}, + ?assertEqual(<<"b">>, session_id_by_ref(Ref, Sessions, #{})), + ?assertEqual(undefined, session_id_by_ref(make_ref(), Sessions, #{})). + +pending_session_by_ref_found_test() -> + Ref = make_ref(), + Session = #{user_id => 1, mref => Ref, pending_connect => true}, + Sessions = #{<<"a">> => Session}, + State = #{sessions => Sessions, guild_session_refs => #{Ref => <<"a">>}}, + ?assertEqual({ok, <<"a">>, Session, Sessions}, pending_session_by_ref(Ref, State)), + ?assertEqual( + {ok, <<"a">>, Session, Sessions}, + pending_session_by_ref(Ref, #{sessions => Sessions}) + ). + +pending_session_by_ref_not_pending_test() -> + Ref = make_ref(), + Session = #{user_id => 1, mref => Ref, pending_connect => false}, + State = #{sessions => #{<<"a">> => Session}, guild_session_refs => #{Ref => <<"a">>}}, + ?assertEqual(not_found, pending_session_by_ref(Ref, State)). + +pending_session_by_ref_unknown_ref_test() -> + Sessions = #{<<"a">> => #{user_id => 1, mref => make_ref(), pending_connect => true}}, + ?assertEqual(not_found, pending_session_by_ref(make_ref(), #{sessions => Sessions})). + +pending_session_by_ref_stale_index_test() -> + Ref = make_ref(), + State = #{sessions => #{}, guild_session_refs => #{Ref => <<"gone">>}}, + ?assertEqual(not_found, pending_session_by_ref(Ref, State)). + +pending_session_owns_connected_tracking_untracked_test() -> + Session = #{user_id => 1, pending_connect => true}, + Sessions = #{<<"a">> => Session}, + ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, #{})). + +pending_session_owns_connected_tracking_other_session_owns_test() -> + Session = #{user_id => 1, pending_connect => true}, + Active = #{user_id => 1, pending_connect => false}, + Sessions = #{<<"a">> => Session, <<"b">> => Active}, + State = #{user_session_counts => #{1 => 1}}, + ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, State)). + +pending_session_owns_connected_tracking_true_test() -> + Session = #{user_id => 1, pending_connect => true}, + Sessions = #{<<"a">> => Session}, + State = #{user_session_counts => #{1 => 1}}, + ?assertEqual(true, pending_session_owns_connected_tracking(Session, Sessions, State)). + +handle_pending_session_down_keeps_tracking_test() -> + Ref = make_ref(), + Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true}, + Active = #{session_id => <<"b">>, user_id => 1, mref => make_ref()}, + Sessions = #{<<"a">> => Pending, <<"b">> => Active}, + State = #{ + sessions => Sessions, + guild_session_refs => #{Ref => <<"a">>}, + session_connect_pending => #{<<"a">> => #{}}, + user_session_counts => #{1 => 1}, + connected_user_ids => sets:from_list([1]) + }, + {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State), + ?assertEqual(#{<<"b">> => Active}, maps:get(sessions, NewState)), + ?assertEqual(#{}, maps:get(guild_session_refs, NewState)), + ?assertEqual(#{}, maps:get(session_connect_pending, NewState)), + ?assertEqual(#{1 => 1}, maps:get(user_session_counts, NewState)), + ?assertEqual([1], sets:to_list(maps:get(connected_user_ids, NewState))), + ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)). + +handle_pending_session_down_last_session_test() -> + Ref = make_ref(), + Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true}, + Sessions = #{<<"a">> => Pending}, + State = #{ + sessions => Sessions, + guild_session_refs => #{Ref => <<"a">>}, + disable_auto_stop_on_empty => true + }, + {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State), + ?assertEqual(#{}, maps:get(sessions, NewState)), + ?assertEqual(#{}, maps:get(guild_session_refs, NewState)), + ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)). + -endif. diff --git a/fluxer_gateway/src/guild/guild_subscription_handler.erl b/fluxer_gateway/src/guild/guild_subscription_handler.erl index 3e0274135..2062a719a 100644 --- a/fluxer_gateway/src/guild/guild_subscription_handler.erl +++ b/fluxer_gateway/src/guild/guild_subscription_handler.erl @@ -22,6 +22,8 @@ -define(LAZY_SUBSCRIBE_COALESCE_MS, 100). -define(MAX_BUFFERED_LAZY_SUBSCRIBE_RANGES, 10). +-define(ENGINES_KEY, channel_member_list_engines). +-define(DISPATCHED_KEY, lazy_subscribe_dispatched). -spec handle_call(term(), gen_server:from(), guild_state()) -> {reply, term(), guild_state()}. @@ -47,6 +49,73 @@ handle_info(flush_lazy_subscribe_buffer, State) -> -spec buffer_lazy_subscribe(map(), guild_state()) -> guild_state(). buffer_lazy_subscribe(Request, State) -> + case dispatch_immediately(Request, State) of + true -> dispatch_lazy_subscribe_now(Request, State); + false -> enqueue_lazy_subscribe(Request, State) + end. + +-spec dispatch_immediately(map(), guild_state()) -> boolean(). +dispatch_immediately(Request, State) -> + maps:get(lazy_subscribe_timer, State, undefined) =:= undefined andalso + map_size(maps:get(lazy_subscribe_buffer, State, #{})) =:= 0 andalso + engine_already_built(Request, State). + +-spec engine_already_built(map(), guild_state()) -> boolean(). +engine_already_built(#{channel_id := ChannelId}, State) -> + case guild_member_list:calculate_list_id(ChannelId, State) of + ListId when is_binary(ListId) -> + maps:is_key(ListId, maps:get(?ENGINES_KEY, State, #{})); + _ -> + false + end; +engine_already_built(_Request, _State) -> + false. + +%% A leading-edge dispatch still buffers its request, marked as dispatched, so a second +%% subscribe for the same key inside the window merges into it exactly as it did before +%% the leading edge existed. The mark only survives while nothing merges in, because every +%% merge_lazy_subscribe_request/2 clause returns a map derived from the incoming request. +-spec dispatch_lazy_subscribe_now(map(), guild_state()) -> guild_state(). +dispatch_lazy_subscribe_now(Request, State) -> + try process_lazy_subscribe(Request, State) of + NewState -> enqueue_lazy_subscribe(mark_lazy_subscribe_dispatched(Request), NewState) + catch + Class:Reason:Stack -> + log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack), + State + end. + +-spec mark_lazy_subscribe_dispatched(map()) -> map(). +mark_lazy_subscribe_dispatched(Request) -> + Request#{?DISPATCHED_KEY => true}. + +-spec log_lazy_subscribe_dispatch_error(map(), atom(), term(), list()) -> ok. +log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack) -> + logger:warning( + "guild_lazy_subscribe_dispatch_failed: session_id=~p channel_id=~p error=~p:~p ~p", + [ + maps:get(session_id, Request, undefined), + maps:get(channel_id, Request, undefined), + Class, + Reason, + Stack + ] + ). + +-spec arm_lazy_subscribe_timer(guild_state()) -> guild_state(). +arm_lazy_subscribe_timer(State) -> + case maps:get(lazy_subscribe_timer, State, undefined) of + undefined -> + Ref = erlang:send_after( + ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer + ), + State#{lazy_subscribe_timer => Ref}; + _ -> + State + end. + +-spec enqueue_lazy_subscribe(map(), guild_state()) -> guild_state(). +enqueue_lazy_subscribe(Request, State) -> #{session_id := SessionId, channel_id := ChannelId} = Request, BufferKey = {SessionId, ChannelId}, Buffer = maps:get(lazy_subscribe_buffer, State, #{}), @@ -59,13 +128,8 @@ buffer_lazy_subscribe(Request, State) -> TimerRef = maps:get(lazy_subscribe_timer, State, undefined), NewState = State#{lazy_subscribe_buffer => NewBuffer, lazy_subscribe_order => NewOrder}, case TimerRef of - undefined -> - Ref = erlang:send_after( - ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer - ), - NewState#{lazy_subscribe_timer => Ref}; - _ -> - NewState + undefined -> arm_lazy_subscribe_timer(NewState); + _ -> NewState end. -spec merge_lazy_subscribe_request(map() | undefined, map()) -> map(). @@ -139,6 +203,8 @@ ordered_lazy_subscribe_keys(State, Buffer) -> guild_state(). process_buffered_lazy_subscribe(BufferKey, Buffer, State) -> case maps:find(BufferKey, Buffer) of + {ok, #{?DISPATCHED_KEY := true}} -> + State; {ok, Request} -> process_lazy_subscribe(Request, State); error -> @@ -294,13 +360,9 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) - FilteredMemberIds = filter_member_ids_for_subscription( GuildId, SessionUserId, MemberIds, State ), - OldSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, MemberSubs), - NewMemberSubs = guild_subscriptions:update_subscriptions( + {NewMemberSubs, Added, Removed} = guild_subscriptions:update_subscriptions_with_delta( SessionId, FilteredMemberIds, MemberSubs ), - NewSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, NewMemberSubs), - Added = sets:to_list(sets:subtract(NewSubscriptions, OldSubscriptions)), - Removed = sets:to_list(sets:subtract(OldSubscriptions, NewSubscriptions)), State1 = State#{member_subscriptions => NewMemberSubs}, State2 = handle_added_subscriptions(Added, SessionId, State1), handle_removed_subscriptions(Removed, State2). @@ -312,7 +374,7 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) - filter_member_ids_for_subscription(_GuildId, undefined, _MemberIds, _State) -> []; filter_member_ids_for_subscription(_GuildId, SessionUserId, MemberIds, State) -> - filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State). + guild_subscription_mutual_channels:filter_member_ids(SessionUserId, MemberIds, State). -spec handle_added_subscriptions([user_id()], session_id(), guild_state()) -> guild_state(). handle_added_subscriptions(Added, SessionId, State) -> @@ -333,58 +395,6 @@ handle_removed_subscriptions(Removed, State) -> Removed ). --spec filter_member_ids_with_mutual_channels( - user_id(), [user_id()], guild_state() -) -> [user_id()]. -filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State) -> - SessionMap = get_session_channel_map(SessionUserId, State), - lists:filtermap( - fun(MemberId) -> - has_mutual_channel( - MemberId, SessionUserId, SessionMap, State - ) - end, - MemberIds - ). - --spec get_session_channel_map(user_id(), guild_state()) -> map(). -get_session_channel_map(SessionUserId, State) -> - case - guild_visibility_channels:get_cached_viewable_channel_map( - SessionUserId, State - ) - of - undefined -> - guild_sessions:build_viewable_channel_map( - guild_visibility:get_user_viewable_channels( - SessionUserId, State - ) - ); - M -> - M - end. - --spec has_mutual_channel( - term(), user_id(), map(), guild_state() -) -> {true, user_id()} | false. -has_mutual_channel(MemberId, SessionUserId, _SessionMap, _State) when - MemberId =:= SessionUserId; not is_integer(MemberId) --> - false; -has_mutual_channel(MemberId, _SessionUserId, SessionMap, State) -> - MemberChannels = guild_visibility:get_user_viewable_channels( - MemberId, State - ), - HasMutual = has_shared_channel(MemberChannels, SessionMap), - case HasMutual of - true -> {true, MemberId}; - false -> false - end. - --spec has_shared_channel([integer()], map()) -> boolean(). -has_shared_channel(MemberChannels, SessionMap) -> - lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels). - -ifdef(TEST). -spec disabled_operations_state(integer() | binary()) -> guild_state(). @@ -497,4 +507,138 @@ flush_lazy_subscribe_buffer_clears_state_test() -> ?assertEqual(error, maps:find(lazy_subscribe_order, NewState)), ?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)). +warm_list_state(ChannelId) -> + ListId = integer_to_binary(ChannelId), + #{ + data => #{ + <<"channel_index">> => #{ + ChannelId => #{<<"id">> => ListId} + } + }, + ?ENGINES_KEY => #{ListId => warm_list_state_engine_ref} + }. + +engine_already_built_detects_built_engine_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + ?assertEqual(true, engine_already_built(Request, warm_list_state(500))). + +engine_already_built_false_without_engine_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)), + ?assertEqual(false, engine_already_built(Request, State)). + +engine_already_built_false_for_unknown_channel_test() -> + Request = #{session_id => <<"s1">>, channel_id => 999, ranges => [{0, 99}]}, + ?assertEqual(false, engine_already_built(Request, warm_list_state(500))). + +engine_already_built_false_on_bare_state_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + ?assertEqual(false, engine_already_built(Request, #{})). + +dispatch_immediately_requires_idle_window_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + Warm = warm_list_state(500), + ?assertEqual(true, dispatch_immediately(Request, Warm)), + ?assertEqual( + false, dispatch_immediately(Request, Warm#{lazy_subscribe_timer => make_ref()}) + ), + ?assertEqual( + false, + dispatch_immediately(Request, Warm#{ + lazy_subscribe_buffer => #{{<<"s2">>, 501} => Request} + }) + ). + +warm_subscribe_records_the_dispatched_request_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + NewState = buffer_lazy_subscribe(Request, warm_list_state(500)), + Buffer = maps:get(lazy_subscribe_buffer, NewState), + ?assertEqual(Request#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)), + ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, NewState)), + ?assertNotEqual(undefined, maps:get(lazy_subscribe_timer, NewState, undefined)). + +subscribe_behind_a_warm_dispatch_is_coalesced_test() -> + Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)), + Request2 = #{session_id => <<"s2">>, channel_id => 500, ranges => [{0, 99}]}, + State2 = buffer_lazy_subscribe(Request2, State1), + Buffer = maps:get(lazy_subscribe_buffer, State2), + ?assertEqual(Request2, maps:get({<<"s2">>, 500}, Buffer)), + ?assertEqual(Request1#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)), + ?assertEqual(2, map_size(Buffer)). + +%% Reference oracle: what enqueue_lazy_subscribe/2 buffered for a repeated +%% {session, channel} before the leading-edge dispatch existed. +reference_coalesced_request(Request1, Request2) -> + merge_lazy_subscribe_request(Request1, Request2). + +repeated_warm_subscribe_merges_like_the_pre_leading_edge_path_test() -> + Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{100, 199}]}, + State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)), + State2 = buffer_lazy_subscribe(Request2, State1), + Buffer = maps:get(lazy_subscribe_buffer, State2), + Merged = maps:get({<<"s1">>, 500}, Buffer), + ?assertEqual(1, map_size(Buffer)), + ?assertEqual(reference_coalesced_request(Request1, Request2), Merged), + ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)), + ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, State2)). + +repeated_warm_subscribe_with_empty_ranges_replaces_test() -> + Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => []}, + State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)), + State2 = buffer_lazy_subscribe(Request2, State1), + Merged = maps:get({<<"s1">>, 500}, maps:get(lazy_subscribe_buffer, State2)), + ?assertEqual(reference_coalesced_request(Request1, Request2), Merged), + ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)). + +dispatched_entry_state() -> + Key = {<<"s1">>, 500}, + Dispatched = #{session_id => <<"s1">>, channel_id => 500, ?DISPATCHED_KEY => true}, + State = #{ + lazy_subscribe_buffer => #{Key => Dispatched}, + lazy_subscribe_order => [Key], + lazy_subscribe_timer => make_ref() + }, + {Key, Dispatched, State}. + +flush_skips_an_already_dispatched_entry_test() -> + {_Key, _Dispatched, State} = dispatched_entry_state(), + NewState = flush_lazy_subscribe_buffer(State), + ?assertEqual(error, maps:find(lazy_subscribe_buffer, NewState)), + ?assertEqual(error, maps:find(lazy_subscribe_order, NewState)), + ?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)). + +flush_processes_the_same_entry_once_unmarked_test() -> + {Key, Dispatched, State} = dispatched_entry_state(), + Unmarked = maps:remove(?DISPATCHED_KEY, Dispatched), + ?assertError( + {badmatch, _}, + flush_lazy_subscribe_buffer(State#{lazy_subscribe_buffer => #{Key => Unmarked}}) + ). + +failed_warm_dispatch_is_not_re_enqueued_test() -> + Warm = warm_list_state(500), + Broken = #{session_id => <<"s1">>, channel_id => 500}, + ?assertEqual(Warm, buffer_lazy_subscribe(Broken, Warm)). + +failed_warm_dispatch_does_not_escape_handle_call_test() -> + Warm = warm_list_state(500), + Broken = #{session_id => <<"s1">>, channel_id => 500}, + From = {self(), make_ref()}, + ?assertEqual({reply, ok, Warm}, handle_call({lazy_subscribe, Broken}, From, Warm)). + +cold_list_still_uses_the_coalesce_buffer_test() -> + Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]}, + State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)), + NewState = buffer_lazy_subscribe(Request, State), + Buffer = maps:get(lazy_subscribe_buffer, NewState), + ?assertEqual(Request, maps:get({<<"s1">>, 500}, Buffer)). + +arm_lazy_subscribe_timer_is_idempotent_test() -> + Ref = make_ref(), + State = #{lazy_subscribe_timer => Ref}, + ?assertEqual(Ref, maps:get(lazy_subscribe_timer, arm_lazy_subscribe_timer(State))). + -endif. diff --git a/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl new file mode 100644 index 000000000..404bdf93f --- /dev/null +++ b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl @@ -0,0 +1,287 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_subscription_mutual_channels). +-typing([eqwalizer]). + +-export([filter_member_ids/3]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-type guild_state() :: map(). +-type user_id() :: integer(). +-type memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => boolean()}}. +-export_type([guild_state/0, user_id/0]). + +-spec filter_member_ids(user_id(), [user_id()], guild_state()) -> [user_id()]. +filter_member_ids(_SessionUserId, [], _State) -> + []; +filter_member_ids(SessionUserId, MemberIds, State) -> + SessionMap = session_channel_map(SessionUserId, State), + {Kept, _Memo} = lists:foldl( + fun(MemberId, Acc) -> + keep_member_with_mutual_channel(MemberId, SessionUserId, SessionMap, State, Acc) + end, + {[], new_memo(State)}, + MemberIds + ), + lists:reverse(Kept). + +-spec session_channel_map(user_id(), guild_state()) -> map(). +session_channel_map(SessionUserId, State) -> + case guild_visibility_channels:get_cached_viewable_channel_map(SessionUserId, State) of + undefined -> + guild_sessions:build_viewable_channel_map( + guild_visibility:get_user_viewable_channels(SessionUserId, State) + ); + Map -> + Map + end. + +-spec keep_member_with_mutual_channel( + term(), user_id(), map(), guild_state(), {[user_id()], memo()} +) -> {[user_id()], memo()}. +keep_member_with_mutual_channel(MemberId, SessionUserId, _SessionMap, _State, Acc) when + MemberId =:= SessionUserId; not is_integer(MemberId) +-> + Acc; +keep_member_with_mutual_channel(MemberId, _SessionUserId, SessionMap, State, {Kept, Memo}) -> + case memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) of + {true, Memo1} -> {[MemberId | Kept], Memo1}; + {false, Memo1} -> {Kept, Memo1} + end. + +%% get_user_viewable_channels/2 reads nothing from a member except its raw roles +%% term, so two members holding the same term see the same channels and give the same +%% answer against a fixed session map. The user-specific inputs are the guild owner, +%% virtual channel access and user-type permission overwrites, and every user named +%% by one of those bypasses the memo. +-spec new_memo(guild_state()) -> memo(). +new_memo(State) -> + #{exceptions => exceptions(State), cache => #{}}. + +-spec memoised_has_mutual_channel(user_id(), map(), guild_state(), memo()) -> + {boolean(), memo()}. +memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) -> + #{exceptions := Exceptions, cache := Cache} = Memo, + case memo_key(MemberId, Exceptions, State) of + bypass -> + {has_mutual_channel(MemberId, SessionMap, State), Memo}; + {ok, RawRoles} -> + cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo) + end. + +-spec cached_has_mutual_channel( + user_id(), term(), map(), guild_state(), #{term() => boolean()}, memo() +) -> {boolean(), memo()}. +cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo) -> + case maps:find(RawRoles, Cache) of + {ok, HasMutual} -> + {HasMutual, Memo}; + error -> + HasMutual = has_mutual_channel(MemberId, SessionMap, State), + {HasMutual, Memo#{cache := Cache#{RawRoles => HasMutual}}} + end. + +-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass. +memo_key(MemberId, Exceptions, State) -> + case sets:is_element(MemberId, Exceptions) of + true -> bypass; + false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State)) + end. + +-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass. +member_roles_key(Member) when is_map(Member) -> + {ok, maps:get(<<"roles">>, Member, [])}; +member_roles_key(_Member) -> + bypass. + +-spec exceptions(guild_state()) -> sets:set(user_id()). +exceptions(State) -> + Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})), + Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})), + Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)), + Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))), + sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)). + +%% parse_maybe/1, not parse_optional/1: an id this rejects can never match a user in +%% a permission check either, so leaving it out of the exception set keeps the memo +%% exact while keeping the set itself total. +-spec owner_ids(term()) -> [user_id()]. +owner_ids(OwnerIdRaw) -> + case snowflake_id:parse_maybe(OwnerIdRaw) of + OwnerId when is_integer(OwnerId) -> [OwnerId]; + _ -> [] + end. + +%% Every overwrite target on every channel, read from both places a permission check +%% can take them from, and role targets alongside user targets. Role ids can never +%% collide with user ids, so the superset only costs a few members their memo entry. +-spec overwrite_target_ids(map()) -> [user_id()]. +overwrite_target_ids(Data) -> + Cached = maps:fold( + fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end, + [], + map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{})) + ), + maps:fold( + fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end, + Cached, + guild_data_index:channel_index(Data) + ). + +-spec cached_target_ids(term(), [user_id()]) -> [user_id()]. +cached_target_ids(Entries, Acc) when is_list(Entries) -> + [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc; +cached_target_ids(_Entries, Acc) -> + Acc. + +-spec channel_target_ids(term(), [user_id()]) -> [user_id()]. +channel_target_ids(Channel, Acc) when is_map(Channel) -> + lists:foldl( + fun overwrite_target_id/2, + Acc, + map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, [])) + ); +channel_target_ids(_Channel, Acc) -> + Acc. + +-spec overwrite_target_id(term(), [user_id()]) -> [user_id()]. +overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of + Id when is_integer(Id) -> [Id | Acc]; + _ -> Acc + end; +overwrite_target_id(_Overwrite, Acc) -> + Acc. + +-spec has_mutual_channel(user_id(), map(), guild_state()) -> boolean(). +has_mutual_channel(MemberId, SessionMap, State) -> + MemberChannels = guild_visibility:get_user_viewable_channels(MemberId, State), + has_shared_channel(MemberChannels, SessionMap). + +-spec has_shared_channel([integer()], map()) -> boolean(). +has_shared_channel(MemberChannels, SessionMap) -> + lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels). + +-ifdef(TEST). + +-define(GUILD_ID, 42). +-define(VIEWER_ROLE, 1000). +-define(OTHER_ROLE, 2000). + +test_role(RoleId) -> + #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}. + +test_member(UserId, RoleIds) -> + #{ + <<"user">> => #{<<"id">> => integer_to_binary(UserId)}, + <<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds] + }. + +test_overwrite(TargetId, Type) -> + #{ + <<"id">> => integer_to_binary(TargetId), + <<"type">> => Type, + <<"allow">> => integer_to_binary(constants:view_channel_permission()), + <<"deny">> => <<"0">> + }. + +test_channel(ChannelId, Overwrites) -> + #{ + <<"id">> => integer_to_binary(ChannelId), + <<"type">> => 0, + <<"permission_overwrites">> => Overwrites + }. + +%% Session user 10 and members 20 and 21 reach channel 500 through the viewer role. +%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel +%% 600 by role, but 99 holds virtual access to 500 and 4242 is a user-overwrite +%% target on 500, so both must still come out true. Member 7 owns the guild. +test_state() -> + Channels = [ + test_channel(500, [test_overwrite(?VIEWER_ROLE, 0), test_overwrite(4242, 1)]), + test_channel(600, [test_overwrite(?OTHER_ROLE, 0)]) + ], + Roles = [test_role(?GUILD_ID), test_role(?VIEWER_ROLE), test_role(?OTHER_ROLE)], + Members = maps:from_list( + [{7, test_member(7, [])}] ++ + [{Id, test_member(Id, [?VIEWER_ROLE])} || Id <- [10, 20, 21]] ++ + [{Id, test_member(Id, [?OTHER_ROLE])} || Id <- [30, 31, 99, 4242]] + ), + #{ + id => ?GUILD_ID, + virtual_channel_access => #{99 => sets:from_list([500])}, + data => #{ + <<"guild">> => #{<<"owner_id">> => <<"7">>}, + <<"roles">> => Roles, + <<"members">> => Members, + <<"channels">> => Channels, + <<"channel_index">> => guild_data_index:build_id_index(Channels) + } + }. + +%% filter_member_ids/3 as it read before the memo: every candidate materialises its +%% own complete viewable channel list. +reference_filter_member_ids(SessionUserId, MemberIds, State) -> + SessionMap = session_channel_map(SessionUserId, State), + lists:filtermap( + fun(MemberId) -> reference_keep(MemberId, SessionUserId, SessionMap, State) end, + MemberIds + ). + +reference_keep(MemberId, SessionUserId, _SessionMap, _State) when + MemberId =:= SessionUserId; not is_integer(MemberId) +-> + false; +reference_keep(MemberId, _SessionUserId, SessionMap, State) -> + case has_mutual_channel(MemberId, SessionMap, State) of + true -> {true, MemberId}; + false -> false + end. + +candidate_ids() -> + [20, 30, 21, 31, 10, 7, 99, 4242, 777, 20]. + +exceptions_cover_owner_virtual_and_overwrite_targets_test() -> + Exceptions = exceptions(test_state()), + ?assertEqual([true, true, true], [ + sets:is_element(Id, Exceptions) + || Id <- [7, 99, 4242] + ]), + ?assertEqual([false, false], [sets:is_element(Id, Exceptions) || Id <- [20, 30]]). + +filter_member_ids_matches_unmemoised_test() -> + State = test_state(), + Ids = candidate_ids(), + ?assertEqual([20, 21, 7, 99, 4242, 20], filter_member_ids(10, Ids, State)), + ?assertEqual( + [reference_filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]], + [filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]] + ). + +memo_holds_one_entry_per_roles_term_test() -> + State = test_state(), + SessionMap = session_channel_map(10, State), + {First, Memo1} = memoised_has_mutual_channel(20, SessionMap, State, new_memo(State)), + {Second, Memo2} = memoised_has_mutual_channel(21, SessionMap, State, Memo1), + ?assert(First), + ?assertEqual(First, Second), + ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]). + +exception_user_is_not_answered_from_its_role_set_test() -> + State = test_state(), + SessionMap = session_channel_map(10, State), + {false, Memo} = memoised_has_mutual_channel(30, SessionMap, State, new_memo(State)), + ?assertEqual( + [{true, Memo}, {true, Memo}, {true, Memo}], + [memoised_has_mutual_channel(Id, SessionMap, State, Memo) || Id <- [7, 99, 4242]] + ). + +non_member_candidate_is_dropped_test() -> + State = test_state(), + ?assertEqual([], filter_member_ids(10, [777], State)), + ?assertEqual([], reference_filter_member_ids(10, [777], State)). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_subscriptions.erl b/fluxer_gateway/src/guild/guild_subscriptions.erl index 50c5b8850..2ca02b260 100644 --- a/fluxer_gateway/src/guild/guild_subscriptions.erl +++ b/fluxer_gateway/src/guild/guild_subscriptions.erl @@ -9,6 +9,7 @@ unsubscribe/3, unsubscribe_session/2, update_subscriptions/3, + update_subscriptions_with_delta/3, get_subscribed_sessions/2, is_subscribed/3, get_user_ids_for_session/2 @@ -52,30 +53,44 @@ remove_session_subscriber(UserId, SessionId, Subscribers, State) -> _ -> State#{UserId => NewSubscribers} end. +%% Every subscriber set in the map is non-empty: this module, guild_maintenance and +%% guild_presence_sync all drop a user key once its last subscriber goes. Entries the +%% session is absent from are therefore already correct and are left as they are +%% instead of being rebuilt into a fresh map. -spec unsubscribe_session(session_id(), subscription_state()) -> subscription_state(). unsubscribe_session(SessionId, State) -> maps:fold( fun(UserId, Subscribers, Acc) -> - remove_session_from_subs(UserId, SessionId, Subscribers, Acc) + drop_session_subscriber(UserId, SessionId, Subscribers, Acc) end, - #{}, + State, State ). --spec remove_session_from_subs( +-spec drop_session_subscriber( user_id(), session_id(), sets:set(session_id()), subscription_state() ) -> subscription_state(). -remove_session_from_subs(UserId, SessionId, Subscribers, Acc) -> - NewSubscribers = sets:del_element(SessionId, Subscribers), - case sets:size(NewSubscribers) of - 0 -> Acc; - _ -> Acc#{UserId => NewSubscribers} +drop_session_subscriber(UserId, SessionId, Subscribers, Acc) -> + case sets:is_element(SessionId, Subscribers) of + true -> remove_session_subscriber(UserId, SessionId, Subscribers, Acc); + false -> Acc end. -spec update_subscriptions(session_id(), [user_id()], subscription_state()) -> subscription_state(). update_subscriptions(SessionId, NewMemberIds, State) -> + {NewState, _Added, _Removed} = update_subscriptions_with_delta( + SessionId, NewMemberIds, State + ), + NewState. + +%% After both folds the session subscribes to exactly NewMemberIds, so the added and +%% removed sets are the ToAdd and ToRemove already computed here. Deriving them from +%% the resulting map instead would cost two more full folds of the subscription map. +-spec update_subscriptions_with_delta(session_id(), [user_id()], subscription_state()) -> + {subscription_state(), [user_id()], [user_id()]}. +update_subscriptions_with_delta(SessionId, NewMemberIds, State) -> CurrentlySubscribed = get_user_ids_for_session(SessionId, State), NewMemberIdSet = sets:from_list(NewMemberIds), ToRemove = sets:subtract(CurrentlySubscribed, NewMemberIdSet), @@ -87,13 +102,14 @@ update_subscriptions(SessionId, NewMemberIds, State) -> State, ToRemove ), - sets:fold( + State2 = sets:fold( fun(UserId, AccState) -> subscribe(SessionId, UserId, AccState) end, State1, ToAdd - ). + ), + {State2, sets:to_list(ToAdd), sets:to_list(ToRemove)}. -spec get_subscribed_sessions(user_id(), subscription_state()) -> [session_id()]. get_subscribed_sessions(UserId, State) -> @@ -193,4 +209,83 @@ get_user_ids_for_session_test() -> UserIds = get_user_ids_for_session(<<"session1">>, State2), ?assertEqual([100, 200], lists:sort(sets:to_list(UserIds))). +%% The subscription map as unsubscribe_session/2 built it before it stopped +%% rebuilding untouched entries. +reference_unsubscribe_session(SessionId, State) -> + maps:fold( + fun(UserId, Subscribers, Acc) -> + reference_drop_subscriber(UserId, sets:del_element(SessionId, Subscribers), Acc) + end, + #{}, + State + ). + +reference_drop_subscriber(UserId, NewSubscribers, Acc) -> + case sets:size(NewSubscribers) of + 0 -> Acc; + _ -> Acc#{UserId => NewSubscribers} + end. + +%% The state and delta handle_update_member_subscriptions_local/4 derived before it +%% stopped re-folding the map to rediscover what changed. +reference_update_with_delta(SessionId, NewMemberIds, State) -> + Old = get_user_ids_for_session(SessionId, State), + NewState = update_subscriptions(SessionId, NewMemberIds, State), + New = get_user_ids_for_session(SessionId, NewState), + { + NewState, + sets:to_list(sets:subtract(New, Old)), + sets:to_list(sets:subtract(Old, New)) + }. + +populate(Pairs) -> + lists:foldl( + fun({SessionId, UserId}, Acc) -> subscribe(SessionId, UserId, Acc) end, + init_state(), + Pairs + ). + +delta_fixture_states() -> + [ + init_state(), + populate([{<<"s1">>, 100}]), + populate([{<<"s1">>, 100}, {<<"s1">>, 200}, {<<"s2">>, 200}, {<<"s2">>, 300}]), + populate([{<<"s2">>, N} || N <- lists:seq(1, 64)]), + populate( + [{<<"s1">>, N} || N <- lists:seq(1, 200)] ++ + [{<<"s2">>, N} || N <- lists:seq(150, 400)] + ) + ]. + +delta_fixture_cases() -> + Inputs = [[], [100], [200, 300], [300, 200], [100, 100, 200], lists:seq(100, 500)], + [{State, Ids} || State <- delta_fixture_states(), Ids <- Inputs]. + +unsubscribe_session_matches_rebuilt_map_test() -> + Cases = [ + {State, SessionId} + || State <- delta_fixture_states(), + SessionId <- [<<"s1">>, <<"s2">>, <<"never_subscribed">>] + ], + ?assertEqual( + [reference_unsubscribe_session(S, M) || {M, S} <- Cases], + [unsubscribe_session(S, M) || {M, S} <- Cases] + ). + +update_subscriptions_with_delta_matches_refold_test() -> + Cases = delta_fixture_cases(), + ?assertEqual( + [reference_update_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases], + [update_subscriptions_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases] + ). + +update_subscriptions_with_delta_leaves_other_sessions_alone_test() -> + State0 = subscribe(<<"s2">>, 100, subscribe(<<"s1">>, 100, init_state())), + {NewState, Added, Removed} = update_subscriptions_with_delta(<<"s1">>, [200], State0), + ?assertEqual([200], Added), + ?assertEqual([100], Removed), + ?assert(is_subscribed(<<"s2">>, 100, NewState)), + ?assertNot(is_subscribed(<<"s1">>, 100, NewState)), + ?assert(is_subscribed(<<"s1">>, 200, NewState)). + -endif. diff --git a/fluxer_gateway/src/guild/voice/guild_voice.erl b/fluxer_gateway/src/guild/voice/guild_voice.erl index 75992f6c5..e080649a8 100644 --- a/fluxer_gateway/src/guild/voice/guild_voice.erl +++ b/fluxer_gateway/src/guild/voice/guild_voice.erl @@ -112,7 +112,7 @@ switch_voice_region(GuildId, ChannelId, GuildPid) -> -spec handle_virtual_channel_access_for_move(integer(), integer(), map(), pid()) -> ok. handle_virtual_channel_access_for_move(UserId, ChannelId, _ConnectionsToMove, GuildPid) -> - case gen_server:call(GuildPid, {get_sessions}, 10000) of + case guild_voice_server_state:guild_state_call(GuildPid, 10000) of State when is_map(State) -> maybe_grant_move_access(UserId, ChannelId, State, GuildPid); _ -> @@ -159,9 +159,9 @@ cleanup_virtual_access_on_disconnect(UserId, GuildPid) -> -spec resolve_guild_id_from_pid(pid()) -> integer() | undefined. resolve_guild_id_from_pid(GuildPid) -> - try gen_server:call(GuildPid, {get_sessions}, 5000) of - State when is_map(State) -> - maps:get(id, State, undefined); + try guild_voice_server_state:guild_id_call(GuildPid, 5000) of + GuildId when is_integer(GuildId) -> + GuildId; _ -> undefined catch diff --git a/fluxer_gateway/src/guild/voice/guild_voice_move.erl b/fluxer_gateway/src/guild/voice/guild_voice_move.erl index a4d620cbf..79b602e46 100644 --- a/fluxer_gateway/src/guild/voice/guild_voice_move.erl +++ b/fluxer_gateway/src/guild/voice/guild_voice_move.erl @@ -176,7 +176,7 @@ send_voice_server_update_for_move( integer(), integer(), integer(), binary(), binary() | null, pid() ) -> ok. do_send_voice_server_update(GuildId, ChannelId, UserId, SessionId, OldConnectionId, GuildPid) -> - try gen_server:call(GuildPid, {get_sessions}, 10000) of + try guild_voice_server_state:guild_state_call(GuildPid, 10000) of State when is_map(State) -> request_and_broadcast_token( GuildId, ChannelId, UserId, SessionId, OldConnectionId, State diff --git a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl index 8789c59d4..a64145252 100644 --- a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl +++ b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl @@ -256,7 +256,7 @@ member_user_id(Member) -> -spec fetch_state_and_send(integer(), integer(), map(), pid(), integer(), map()) -> ok. fetch_state_and_send(GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member) -> - try gen_server:call(GuildPid, {get_sessions}, 10000) of + try guild_voice_server_state:guild_state_call(GuildPid, 10000) of StateData when is_map(StateData) -> request_and_broadcast( GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member, StateData diff --git a/fluxer_gateway/src/guild/voice/guild_voice_region.erl b/fluxer_gateway/src/guild/voice/guild_voice_region.erl index fc9b2ef9d..5f6deb9ad 100644 --- a/fluxer_gateway/src/guild/voice/guild_voice_region.erl +++ b/fluxer_gateway/src/guild/voice/guild_voice_region.erl @@ -40,7 +40,7 @@ switch_voice_region_channel_reply(Channel, State) -> -spec switch_voice_region(integer(), integer(), pid()) -> ok. switch_voice_region(GuildId, ChannelId, GuildPid) -> - case gen_server:call(GuildPid, {get_sessions}, 10000) of + case guild_voice_server_state:guild_state_call(GuildPid, 10000) of State when is_map(State) -> do_switch_voice_region(GuildId, ChannelId, GuildPid, State); _ -> @@ -53,7 +53,7 @@ do_switch_voice_region(GuildId, ChannelId, GuildPid, State) -> UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId), lists:foreach( fun(UserInfo) -> - maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, UserInfo) + maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, State, UserInfo) end, UsersInChannel ). @@ -70,17 +70,28 @@ collect_users_in_channel(VoiceStates, ChannelId) -> ). -spec maybe_send_region_switch_update( - integer(), integer(), pid(), {integer(), binary() | undefined, binary(), voice_state()} + integer(), + integer(), + pid(), + map(), + {integer(), binary() | undefined, binary(), voice_state()} ) -> ok. maybe_send_region_switch_update( - _GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS} + _GuildId, _ChannelId, _GuildPid, _State, {_UserId, undefined, _ConnId, _VS} ) -> ok; maybe_send_region_switch_update( - GuildId, ChannelId, GuildPid, {UserId, SessionId, ExistingConnectionId, VoiceState} + GuildId, ChannelId, GuildPid, State, {UserId, SessionId, ExistingConnectionId, VoiceState} ) -> - send_voice_server_update_for_region_switch( - GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, VoiceState, GuildPid + request_and_broadcast_region_switch( + GuildId, + ChannelId, + UserId, + SessionId, + ExistingConnectionId, + VoiceState, + GuildPid, + State ). -spec collect_user_in_channel(binary(), voice_state(), integer(), list()) -> list(). @@ -100,28 +111,6 @@ collect_user_voice_state(ConnectionId, VoiceState, Acc) -> [{UserId, SessionId, ConnectionId, VoiceState} | Acc] end. --spec send_voice_server_update_for_region_switch( - integer(), integer(), integer(), binary(), binary(), voice_state(), pid() -) -> ok. -send_voice_server_update_for_region_switch( - GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, ExistingVoiceState, GuildPid -) -> - case gen_server:call(GuildPid, {get_sessions}, 10000) of - State when is_map(State) -> - request_and_broadcast_region_switch( - GuildId, - ChannelId, - UserId, - SessionId, - ExistingConnectionId, - ExistingVoiceState, - GuildPid, - State - ); - _ -> - ok - end. - -spec request_and_broadcast_region_switch( integer(), integer(), @@ -297,6 +286,201 @@ collect_users_in_channel_test() -> ?assertEqual(<<"sess1">>, SessionId), ?assertEqual(<<"conn1">>, ConnectionId). +switch_voice_region_reads_guild_state_once_test() -> + with_region_stubs(fun() -> + {Calls, _Dispatches, _Pending} = run_region_switch(fun switch_voice_region/3), + ?assertEqual(1, Calls) + end). + +switch_voice_region_matches_per_user_refetch_reference_test() -> + with_region_stubs(fun region_reference_equivalence_scenario/0). + +region_reference_equivalence_scenario() -> + {RefCalls, RefDispatches, RefPending} = run_region_switch( + fun reference_switch_voice_region/3 + ), + {NewCalls, NewDispatches, NewPending} = run_region_switch(fun switch_voice_region/3), + ?assertEqual(3, RefCalls), + ?assertEqual(1, NewCalls), + ?assertEqual(2, length(RefDispatches)), + ?assertEqual(RefDispatches, NewDispatches), + ?assertEqual(RefPending, NewPending). + +switch_voice_region_skips_users_without_a_session_id_test() -> + with_region_stubs(fun region_missing_session_id_scenario/0). + +region_missing_session_id_scenario() -> + Anonymous = maps:remove( + <<"session_id">>, region_voice_state(<<"conn-c">>, 10, <<"sess1">>) + ), + State = region_state(), + Stateless = State#{voice_states => #{<<"conn-c">> => Anonymous}}, + {Calls, Dispatches, Pending} = run_region_switch_with( + fun switch_voice_region/3, Stateless + ), + ?assertEqual(1, Calls), + ?assertEqual([], Dispatches), + ?assertEqual([], Pending). + +%% The pre-change implementation, kept as a live oracle: it refetched the guild state once per +%% user in the channel instead of reusing the one the enumeration already fetched. +reference_switch_voice_region(GuildId, ChannelId, GuildPid) -> + case guild_voice_server_state:guild_state_call(GuildPid, 10000) of + State when is_map(State) -> + reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State); + _ -> + ok + end. + +reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State) -> + VoiceStates = voice_state_utils:voice_states(State), + UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId), + lists:foreach( + fun(UserInfo) -> reference_send_update(GuildId, ChannelId, GuildPid, UserInfo) end, + UsersInChannel + ). + +reference_send_update(_GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS}) -> + ok; +reference_send_update(GuildId, ChannelId, GuildPid, {UserId, SessionId, ConnId, VoiceState}) -> + case guild_voice_server_state:guild_state_call(GuildPid, 10000) of + State when is_map(State) -> + request_and_broadcast_region_switch( + GuildId, ChannelId, UserId, SessionId, ConnId, VoiceState, GuildPid, State + ); + _ -> + ok + end. + +run_region_switch(Fun) -> + run_region_switch_with(Fun, region_state()). + +run_region_switch_with(Fun, State) -> + drain_region_dispatches(), + drain_region_pending(), + erlang:put(region_token_index, 0), + GuildPid = spawn_region_guild(State), + try + Fun(999, 200, GuildPid), + {region_guild_calls(GuildPid), drain_region_dispatches(), drain_region_pending()} + after + GuildPid ! stop + end. + +with_region_stubs(Fun) -> + meck:new(rpc_client, [passthrough, no_link, non_strict]), + meck:expect(rpc_client, call, fun(_Request) -> {ok, next_region_token()} end), + meck:new(guild_voice_server, [passthrough, no_link]), + meck:expect(guild_voice_server, resolve, fun(_GuildId, GuildPid) -> GuildPid end), + try + Fun() + after + meck:unload(guild_voice_server), + meck:unload(rpc_client) + end. + +next_region_token() -> + Index = erlang:get(region_token_index) + 1, + erlang:put(region_token_index, Index), + #{ + <<"token">> => <<"tok-", (integer_to_binary(Index))/binary>>, + <<"endpoint">> => <<"wss://voice.example">>, + <<"connectionId">> => <<"new-conn-", (integer_to_binary(Index))/binary>>, + <<"regionId">> => <<"us-east">>, + <<"serverId">> => <<"voice-1">> + }. + +spawn_region_guild(State) -> + Owner = self(), + spawn(fun() -> region_guild_loop(State, Owner, 0) end). + +region_guild_loop(State, Owner, Calls) -> + receive + {'$gen_call', From, {get_voice_guild_state}} -> + gen_server:reply(From, State), + region_guild_loop(State, Owner, Calls + 1); + {'$gen_call', From, {store_pending_connection, ConnId, Meta}} -> + Owner ! {region_pending, ConnId, Meta}, + gen_server:reply(From, ok), + region_guild_loop(State, Owner, Calls); + {'$gen_call', From, _Other} -> + gen_server:reply(From, ok), + region_guild_loop(State, Owner, Calls); + {region_calls, Caller} -> + Caller ! {region_calls, Calls}, + region_guild_loop(State, Owner, Calls); + stop -> + ok + after 30000 -> + ok + end. + +region_guild_calls(GuildPid) -> + GuildPid ! {region_calls, self()}, + receive + {region_calls, Calls} -> Calls + after 1000 -> + error(region_guild_unresponsive) + end. + +drain_region_dispatches() -> + receive + {'$gen_cast', {dispatch, voice_server_update, Payload}} -> + [Payload | drain_region_dispatches()] + after 100 -> + [] + end. + +%% token_nonce and the timestamps are freshly generated per call by design, so they are +%% excluded before the reference and the threaded run are compared. +drain_region_pending() -> + receive + {region_pending, ConnId, Meta} -> + Stable = maps:without([token_nonce, created_at, expires_at], Meta), + [{ConnId, Stable} | drain_region_pending()] + after 100 -> + [] + end. + +region_state() -> + #{ + id => 999, + data => #{ + <<"id">> => <<"999">>, + <<"guild">> => #{<<"owner_id">> => <<"10">>}, + <<"channels">> => [#{<<"id">> => <<"200">>, <<"type">> => 2}], + <<"members">> => [region_member(10), region_member(11)] + }, + sessions => #{ + <<"sess1">> => #{user_id => 10, pid => self(), pending_connect => false}, + <<"sess2">> => #{user_id => 11, pid => self(), pending_connect => false} + }, + voice_states => #{ + <<"conn-a">> => region_voice_state(<<"conn-a">>, 10, <<"sess1">>), + <<"conn-b">> => region_voice_state(<<"conn-b">>, 11, <<"sess2">>) + }, + pending_voice_connections => #{} + }. + +region_member(UserId) -> + #{ + <<"user">> => #{<<"id">> => integer_to_binary(UserId)}, + <<"mute">> => false, + <<"deaf">> => false + }. + +region_voice_state(ConnId, UserId, SessionId) -> + #{ + <<"user_id">> => integer_to_binary(UserId), + <<"guild_id">> => <<"999">>, + <<"channel_id">> => <<"200">>, + <<"connection_id">> => ConnId, + <<"session_id">> => SessionId, + <<"self_mute">> => false, + <<"self_deaf">> => true, + <<"member">> => region_member(UserId) + }. + build_pending_metadata_preserves_voice_routing_test() -> Metadata = build_pending_metadata( 10, diff --git a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl index d74c665db..4f77dc31f 100644 --- a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl +++ b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl @@ -13,7 +13,9 @@ repair_voice_state_from_guild_cache/2, voice_state_rpc_entries/1, pending_join_rpc_entries/1, - fetch_guild_data/1 + fetch_guild_data/1, + guild_state_call/2, + guild_id_call/2 ]). -export_type([ @@ -335,9 +337,36 @@ pending_join_channel_id(Metadata) when is_map(Metadata) -> pending_join_channel_id(_) -> undefined. +-spec guild_state_call(pid(), timeout()) -> term(). +guild_state_call(GuildPid, Timeout) -> + case gen_server:call(GuildPid, {get_voice_guild_state}, Timeout) of + GuildState when is_map(GuildState) -> GuildState; + _ -> full_guild_state_call(GuildPid, Timeout) + end. + +-spec full_guild_state_call(pid(), timeout()) -> term(). +full_guild_state_call(GuildPid, Timeout) -> + gen_server:call(GuildPid, {get_sessions}, Timeout). + +-spec guild_id_call(pid(), timeout()) -> integer() | undefined. +guild_id_call(GuildPid, Timeout) -> + case gen_server:call(GuildPid, {get_guild_id}, Timeout) of + GuildId when is_integer(GuildId) -> GuildId; + _ -> guild_state_id(full_guild_state_call(GuildPid, Timeout)) + end. + +-spec guild_state_id(term()) -> integer() | undefined. +guild_state_id(GuildState) when is_map(GuildState) -> + case maps:get(id, GuildState, undefined) of + GuildId when is_integer(GuildId) -> GuildId; + _ -> undefined + end; +guild_state_id(_) -> + undefined. + -spec fetch_guild_data(pid()) -> map(). fetch_guild_data(GuildPid) -> - try gen_server:call(GuildPid, {get_sessions}, ?GUILD_CALL_TIMEOUT) of + try guild_state_call(GuildPid, ?GUILD_CALL_TIMEOUT) of GuildState when is_map(GuildState) -> GuildState; _ -> #{} catch @@ -407,3 +436,89 @@ normalize_rpc_millisecond(Value) when is_binary(Value), byte_size(Value) > 0 -> end; normalize_rpc_millisecond(_) -> 0. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +guild_state_call_uses_projection_test() -> + with_fake_guild(fun new_guild_handler/1, fun(Pid) -> + ?assertEqual(projected_guild_state(), guild_state_call(Pid, 1000)), + ?assertEqual([{get_voice_guild_state}], drain_requests()) + end). + +guild_state_call_retries_get_sessions_against_old_peer_test() -> + with_fake_guild(fun old_guild_handler/1, fun(Pid) -> + ?assertEqual(full_guild_state(), guild_state_call(Pid, 1000)), + ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests()) + end). + +fetch_guild_data_retries_get_sessions_against_old_peer_test() -> + with_fake_guild(fun old_guild_handler/1, fun(Pid) -> + ?assertEqual(full_guild_state(), fetch_guild_data(Pid)), + ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests()) + end). + +guild_id_call_uses_get_guild_id_test() -> + with_fake_guild(fun new_guild_handler/1, fun(Pid) -> + ?assertEqual(42, guild_id_call(Pid, 1000)), + ?assertEqual([{get_guild_id}], drain_requests()) + end). + +guild_id_call_returns_real_id_against_old_peer_test() -> + with_fake_guild(fun old_guild_handler/1, fun(Pid) -> + ?assertEqual(42, guild_id_call(Pid, 1000)), + ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests()) + end). + +guild_id_call_returns_undefined_when_peer_has_no_id_test() -> + with_fake_guild(fun(_) -> ok end, fun(Pid) -> + ?assertEqual(undefined, guild_id_call(Pid, 1000)), + ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests()) + end). + +new_guild_handler({get_voice_guild_state}) -> projected_guild_state(); +new_guild_handler({get_guild_id}) -> 42; +new_guild_handler({get_sessions}) -> full_guild_state(); +new_guild_handler(_) -> ok. + +old_guild_handler({get_sessions}) -> full_guild_state(); +old_guild_handler(_) -> ok. + +full_guild_state() -> + #{ + id => 42, + sessions => #{}, + voice_states => #{}, + data => #{<<"id">> => <<"42">>, <<"members">> => #{}} + }. + +projected_guild_state() -> + #{id => 42, sessions => #{}, voice_states => #{}, data => #{<<"id">> => <<"42">>}}. + +with_fake_guild(Handler, Fun) -> + _ = drain_requests(), + Owner = self(), + Pid = spawn(fun() -> fake_guild_loop(Owner, Handler) end), + try + Fun(Pid) + after + Pid ! stop + end. + +fake_guild_loop(Owner, Handler) -> + receive + {'$gen_call', From, Request} -> + Owner ! {fake_guild_request, Request}, + gen_server:reply(From, Handler(Request)), + fake_guild_loop(Owner, Handler); + stop -> + ok + end. + +drain_requests() -> + receive + {fake_guild_request, Request} -> [Request | drain_requests()] + after 0 -> [] + end. + +-endif. diff --git a/fluxer_gateway/src/presence/presence_broadcast.erl b/fluxer_gateway/src/presence/presence_broadcast.erl index a5d1bf20f..b7988af38 100644 --- a/fluxer_gateway/src/presence/presence_broadcast.erl +++ b/fluxer_gateway/src/presence/presence_broadcast.erl @@ -30,6 +30,8 @@ -type state() :: map(). -type sessions() :: #{binary() => map()}. +-define(CUSTOM_STATUS_EXPIRY_TIMER, custom_status_expiry_timer). + -spec publish_global_if_needed({reply, term(), state()} | {noreply, state()}) -> {reply, term(), state()} | {noreply, state()}. publish_global_if_needed({reply, Reply, NewState}) -> @@ -40,7 +42,8 @@ publish_global_if_needed({noreply, NewState}) -> {noreply, FinalState}. -spec publish_global_presence(sessions(), state()) -> state(). -publish_global_presence(_Sessions, State) -> +publish_global_presence(_Sessions, State0) -> + State = refresh_custom_status_expiry_timer(State0), {Payload, CurrentExternal, ExternalStatus} = build_presence_external(State), LastPublished = maps:get(last_published_presence, State, undefined), case presence_changed(LastPublished, CurrentExternal) of @@ -55,11 +58,33 @@ publish_global_presence(_Sessions, State) -> end. -spec force_publish_global_presence(state()) -> state(). -force_publish_global_presence(State) -> +force_publish_global_presence(State0) -> + State = refresh_custom_status_expiry_timer(State0), {Payload, CurrentExternal, ExternalStatus} = build_presence_external(State), NewState = publish_presence_payload(State, Payload, CurrentExternal, ExternalStatus), presence_update:maybe_update_push_eligibility(NewState). +-spec refresh_custom_status_expiry_timer(state()) -> state(). +refresh_custom_status_expiry_timer(State) -> + ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State, undefined)), + arm_expiry_timer( + custom_status_expiry:next_wakeup_ms(maps:get(custom_status, State, null)), State + ). + +-spec arm_expiry_timer({ok, pos_integer()} | none, state()) -> state(). +arm_expiry_timer(none, State) -> + maps:remove(?CUSTOM_STATUS_EXPIRY_TIMER, State); +arm_expiry_timer({ok, DelayMs}, State) -> + Ref = erlang:send_after(DelayMs, self(), custom_status_expiry:reconcile_message()), + State#{?CUSTOM_STATUS_EXPIRY_TIMER => Ref}. + +-spec cancel_expiry_timer(term()) -> ok. +cancel_expiry_timer(Ref) when is_reference(Ref) -> + _ = erlang:cancel_timer(Ref, [{async, true}, {info, false}]), + ok; +cancel_expiry_timer(_Ref) -> + ok. + -spec dispatch_global_presence(user_id(), map(), state()) -> {noreply, state()}. dispatch_global_presence(TargetId, Payload, State) -> UserId = maps:get(user_id, State), @@ -240,3 +265,97 @@ dispatch_global_user_update(TargetId, Payload, State) -> dispatch_to_sessions(Payload, State) -> SessionPids = presence_connect:collect_session_pids(State), gateway_dispatch_relay:dispatch_many(SessionPids, presence_update, Payload). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +expiry_timer_not_armed_while_disabled_test() -> + application:unset_env(fluxer_gateway, custom_status_expiry_enabled), + State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}), + ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State)). + +expiry_timer_not_armed_without_expires_at_test() -> + with_expiry_enabled(fun() -> + ?assertNot( + maps:is_key( + ?CUSTOM_STATUS_EXPIRY_TIMER, + refresh_custom_status_expiry_timer(#{custom_status => null}) + ) + ), + ?assertNot( + maps:is_key( + ?CUSTOM_STATUS_EXPIRY_TIMER, + refresh_custom_status_expiry_timer(#{ + custom_status => #{<<"text">> => <<"hi">>} + }) + ) + ) + end). + +expiry_timer_armed_for_a_future_expiry_test() -> + with_expiry_enabled(fun() -> + State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}), + ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))), + ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State)) + end). + +expiry_timer_not_armed_for_an_already_expired_status_test() -> + with_expiry_enabled(fun() -> + Expired = #{<<"expires_at">> => <<"2020-01-01T00:00:00.000Z">>}, + State = refresh_custom_status_expiry_timer(#{custom_status => Expired}), + ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State)) + end). + +expiry_timer_replaces_the_previous_one_test() -> + with_expiry_enabled(fun() -> + First = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}), + Second = refresh_custom_status_expiry_timer(First), + ?assertNotEqual( + maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, First), + maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second) + ), + ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second)) + end). + +expiry_timer_is_dropped_when_the_status_is_cleared_test() -> + with_expiry_enabled(fun() -> + Armed = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}), + Cleared = refresh_custom_status_expiry_timer(Armed#{custom_status => null}), + ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, Cleared)) + end). + +expiry_timer_delivers_a_reconcile_cast_test() -> + State = arm_expiry_timer({ok, 1}, #{}), + ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))), + Expected = custom_status_expiry:reconcile_message(), + receive + Expected -> ok + after 1000 -> ?assert(false) + end. + +cancel_expiry_timer_ignores_non_references_test() -> + ?assertEqual(ok, cancel_expiry_timer(undefined)), + ?assertEqual(ok, cancel_expiry_timer(not_a_ref)). + +future_custom_status() -> + ExpiresAt = calendar:system_time_to_rfc3339( + erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}] + ), + #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}. + +with_expiry_enabled(Fun) -> + Key = custom_status_expiry_enabled, + Previous = application:get_env(fluxer_gateway, Key), + application:set_env(fluxer_gateway, Key, true), + try + Fun() + after + restore_expiry_env(Key, Previous) + end. + +restore_expiry_env(Key, undefined) -> + application:unset_env(fluxer_gateway, Key); +restore_expiry_env(Key, {ok, Value}) -> + application:set_env(fluxer_gateway, Key, Value). + +-endif. diff --git a/fluxer_gateway/src/presence/presence_payload.erl b/fluxer_gateway/src/presence/presence_payload.erl index 44c2ed6c2..a1ea69bf3 100644 --- a/fluxer_gateway/src/presence/presence_payload.erl +++ b/fluxer_gateway/src/presence/presence_payload.erl @@ -32,9 +32,12 @@ ensure_status_binary(Status) when is_binary(Status) -> Status; ensure_status_binary(_) -> <<"offline">>. -spec custom_status_for(binary(), custom_status()) -> custom_status(). -custom_status_for(<<"offline">>, _CustomStatus) -> null; -custom_status_for(<<"invisible">>, _CustomStatus) -> null; -custom_status_for(_StatusBin, CustomStatus) -> normalize_custom_status(CustomStatus). +custom_status_for(<<"offline">>, _CustomStatus) -> + null; +custom_status_for(<<"invisible">>, _CustomStatus) -> + null; +custom_status_for(_StatusBin, CustomStatus) -> + custom_status_expiry:clear_if_expired(normalize_custom_status(CustomStatus)). -spec normalize_custom_status(term()) -> custom_status(). normalize_custom_status(null) -> null; @@ -81,16 +84,88 @@ normalize_custom_status_test() -> ?assertEqual(null, normalize_custom_status(123)). build_invisible_atom_normalized_to_offline_test() -> - User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>}, CustomStatus = #{<<"text">> => <<"hello">>}, - Result = build(User, invisible, false, false, CustomStatus), + Result = build(test_user(), invisible, false, false, CustomStatus), ?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)), ?assertEqual(null, maps:get(<<"custom_status">>, Result)). build_invisible_binary_normalized_to_offline_test() -> - User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>}, CustomStatus = #{<<"text">> => <<"hello">>}, - Result = build(User, <<"invisible">>, false, false, CustomStatus), + Result = build(test_user(), <<"invisible">>, false, false, CustomStatus), ?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)), ?assertEqual(null, maps:get(<<"custom_status">>, Result)). + +build_clears_expired_custom_status_test() -> + with_expiry_enabled(fun() -> + Result = build(test_user(), online, false, false, expired_custom_status()), + ?assertEqual(<<"online">>, maps:get(<<"status">>, Result)), + ?assertEqual(null, maps:get(<<"custom_status">>, Result)) + end). + +build_coalesces_an_expired_status_with_no_status_test() -> + with_expiry_enabled(fun() -> + ?assertEqual( + build(test_user(), online, false, false, null), + build(test_user(), online, false, false, expired_custom_status()) + ) + end). + +build_keeps_unexpired_custom_status_test() -> + with_expiry_enabled(fun() -> + Live = future_custom_status(), + Result = build(test_user(), dnd, false, false, Live), + ?assertEqual(Live, maps:get(<<"custom_status">>, Result)) + end). + +build_keeps_custom_status_without_expires_at_test() -> + with_expiry_enabled(fun() -> + Live = #{<<"text">> => <<"forever">>}, + Result = build(test_user(), idle, false, false, Live), + ?assertEqual(Live, maps:get(<<"custom_status">>, Result)) + end). + +build_keeps_malformed_expires_at_test() -> + with_expiry_enabled(fun() -> + Live = #{<<"text">> => <<"hi">>, <<"expires_at">> => <<"not-a-date">>}, + Result = build(test_user(), online, false, false, Live), + ?assertEqual(Live, maps:get(<<"custom_status">>, Result)) + end). + +build_leaves_expired_status_alone_while_disabled_test() -> + application:unset_env(fluxer_gateway, custom_status_expiry_enabled), + Expired = expired_custom_status(), + Result = build(test_user(), online, false, false, Expired), + ?assertEqual(Expired, maps:get(<<"custom_status">>, Result)). + +test_user() -> + #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>}. + +expired_custom_status() -> + #{ + <<"emoji_animated">> => false, + <<"emoji_name">> => null, + <<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>, + <<"text">> => <<"brb">> + }. + +future_custom_status() -> + ExpiresAt = calendar:system_time_to_rfc3339( + erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}] + ), + #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}. + +with_expiry_enabled(Fun) -> + Key = custom_status_expiry_enabled, + Previous = application:get_env(fluxer_gateway, Key), + application:set_env(fluxer_gateway, Key, true), + try + Fun() + after + restore_expiry_env(Key, Previous) + end. + +restore_expiry_env(Key, undefined) -> + application:unset_env(fluxer_gateway, Key); +restore_expiry_env(Key, {ok, Value}) -> + application:set_env(fluxer_gateway, Key, Value). -endif. diff --git a/fluxer_gateway/src/push/push.erl b/fluxer_gateway/src/push/push.erl index f721926dc..d7508c7d1 100644 --- a/fluxer_gateway/src/push/push.erl +++ b/fluxer_gateway/src/push/push.erl @@ -25,6 +25,36 @@ -define(EVICT_INTERVAL_MS, 60000). -define(DEFAULT_MAX_ENTRIES, 500000). +-define(PUSH_COUNTER_TABLE, push_worker_counter). +-define(CNT_WORKER_POOL, push_loss_worker_pool). +-define(CNT_WARM_INFLIGHT, push_blocked_ids_warm_inflight). +-define(CNT_FETCH_ATTEMPTS, push_blocked_ids_fetch_attempts). +-define(CNT_FETCH_FAILURES, push_blocked_ids_fetch_failures). +-define(CNT_SUPPRESSED, push_blocked_ids_suppressed). +-define(CNT_BUDGET_EXHAUSTED, push_blocked_ids_budget_exhausted). +-define(CNT_WARM_DROPPED, push_blocked_ids_warm_dropped). +-define(CNT_DISPATCH_DROPPED, push_loss_dispatch_dropped). +-define(CNT_DISPATCH_DROPPED_USERS, push_loss_dispatch_dropped_users). +-define(CNT_CLEAR_DROPPED, push_loss_clear_dropped). +-define(CNT_QUEUE_FULL, push_loss_queue_full). +-define(CNT_INVALID_JOB, push_loss_invalid_job). +-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout). +-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed). +-define(CNT_JOB_CRASHED, push_loss_job_crashed). +-define(CNT_WORKER_DIED, push_loss_worker_died). +-define(CNT_DISPATCHER_RESTARTS, push_dispatcher_restarts). +-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded). +-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued). +-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued). + +-define(MAX_WARM_INFLIGHT, 4). +-define(MAX_FETCH_RPCS, 8). +-define(DEFAULT_FETCH_USERS, 2000). +-define(MAX_FETCH_USERS, 5000). +-define(DEFAULT_FETCH_CHUNK, 500). +-define(MIN_FETCH_CHUNK, 50). +-define(MAX_FETCH_CHUNK, 1000). + -type state() :: #{ badge_counts_ttl_seconds := non_neg_integer(), max_entries := non_neg_integer() @@ -61,8 +91,7 @@ init([]) -> -spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}. handle_call(get_cache_stats, _From, State) -> - Stats = push_ets_cache:cache_stats(), - {reply, {ok, Stats}, State}; + {reply, {ok, cache_stats_with_counters()}, State}; handle_call(_Request, _From, State) -> {reply, ok, State}. @@ -96,6 +125,7 @@ handle_cast(_Msg, State) -> -spec handle_info(term(), state()) -> {noreply, state()}. handle_info(evict_caches, State) -> + unstick_warm_gate(), MaxEntries = maps:get(max_entries, State), push_ets_cache:evict_tables(#{ user_guild_settings => MaxEntries, @@ -199,9 +229,7 @@ is_push_noop() -> -spec clear_channel_notifications(integer(), integer(), integer()) -> ok. clear_channel_notifications(UserId, ChannelId, MessageId) -> - case - is_push_active() andalso persistent_term:get(push_clear_notifications_enabled, false) - of + case is_push_active() andalso clear_notifications_enabled() of true -> cast_to_push_owner( UserId, {clear_channel_notifications, UserId, ChannelId, MessageId} @@ -210,6 +238,13 @@ clear_channel_notifications(UserId, ChannelId, MessageId) -> ok end. +-spec clear_notifications_enabled() -> boolean(). +clear_notifications_enabled() -> + case persistent_term:get(push_clear_notifications_enabled, undefined) of + Value when is_boolean(Value) -> Value; + _ -> env_boolean(push_clear_notifications_enabled, false) + end. + -spec get_cache_stats() -> {ok, map()}. get_cache_stats() -> gen_server:call(?MODULE, get_cache_stats, 5000). @@ -299,13 +334,13 @@ do_handle_message_create_context(Context, State) -> MessageData, GuildDefaultNotifications, UserRolesMap, - ConnectedUsers + ConnectedUsers, + maps:get(large_guild_metadata, Context, undefined) ), logger:debug("Push: eligibility result", #{ message_id => MessageId, channel_id => ChannelId, - eligible_count => length(EligibleUsers), - eligible_user_ids => EligibleUsers + eligible_count => length(EligibleUsers) }), dispatch_if_eligible( EligibleUsers, @@ -320,7 +355,15 @@ do_handle_message_create_context(Context, State) -> ). -spec filter_eligible_users( - [integer()], integer(), integer(), integer(), map(), integer(), map(), map() + [integer()], + integer(), + integer(), + integer(), + map(), + integer(), + map(), + map(), + map() | undefined ) -> [integer()]. filter_eligible_users( UserIds, @@ -330,11 +373,13 @@ filter_eligible_users( MessageData, GuildDefaultNotifications, UserRolesMap, - ConnectedUsers + ConnectedUsers, + SuppliedMetadata ) -> - LargeGuildMetadata = large_guild_metadata(GuildId), - push_eligibility:prefetch_user_guild_settings(UserIds, AuthorId, GuildId), - lists:filter( + LargeGuildMetadata = resolve_large_guild_metadata(GuildId, SuppliedMetadata), + Candidates = drop_blocked_recipients(UserIds, AuthorId), + push_eligibility:prefetch_user_guild_settings(Candidates, AuthorId, GuildId), + EligibleUsers = lists:filter( fun(UserId) -> push_eligibility:is_eligible_for_push( UserId, @@ -348,15 +393,249 @@ filter_eligible_users( LargeGuildMetadata ) end, - UserIds - ). + Candidates + ), + warm_blocked_ids(EligibleUsers), + EligibleUsers. --spec large_guild_metadata(integer()) -> map() | undefined. -large_guild_metadata(0) -> +-spec resolve_large_guild_metadata(integer(), map() | undefined) -> map() | undefined. +resolve_large_guild_metadata(0, _SuppliedMetadata) -> undefined; -large_guild_metadata(GuildId) -> +resolve_large_guild_metadata(GuildId, SuppliedMetadata) -> + supplied_or_local_metadata(GuildId, SuppliedMetadata). + +-spec supplied_or_local_metadata(integer(), map() | undefined) -> map() | undefined. +supplied_or_local_metadata(GuildId, undefined) -> + large_guild_metadata_local(GuildId); +supplied_or_local_metadata(_GuildId, SuppliedMetadata) when is_map(SuppliedMetadata) -> + SuppliedMetadata; +supplied_or_local_metadata(GuildId, _SuppliedMetadata) -> + large_guild_metadata_local(GuildId). + +-spec large_guild_metadata_local(integer()) -> map() | undefined. +large_guild_metadata_local(GuildId) -> push_eligibility_checks:get_guild_large_metadata(GuildId). +-spec drop_blocked_recipients([integer()], integer()) -> [integer()]. +drop_blocked_recipients(UserIds, AuthorId) -> + Kept = lists:filter( + fun(UserId) -> not push_eligibility:is_user_blocked(UserId, AuthorId) end, + UserIds + ), + count_suppressed(length(UserIds) - length(Kept)), + Kept. + +-spec count_suppressed(non_neg_integer()) -> ok. +count_suppressed(0) -> + ok; +count_suppressed(Suppressed) -> + bump_counter(?CNT_SUPPRESSED, Suppressed). + +-spec warm_blocked_ids([integer()]) -> ok. +warm_blocked_ids([]) -> + ok; +warm_blocked_ids(EligibleUsers) -> + case missing_blocked_ids(EligibleUsers) of + [] -> ok; + Missing -> spawn_blocked_ids_warm(Missing) + end. + +-spec missing_blocked_ids([integer()]) -> [integer()]. +missing_blocked_ids(EligibleUsers) -> + lists:usort(lists:filter(fun is_blocked_ids_cache_miss/1, EligibleUsers)). + +-spec is_blocked_ids_cache_miss(integer()) -> boolean(). +is_blocked_ids_cache_miss(UserId) -> + push_ets_cache:get_blocked_ids(UserId) =:= undefined. + +-spec spawn_blocked_ids_warm([integer()]) -> ok. +spawn_blocked_ids_warm(Missing) -> + case claim_warm_slot() of + ok -> + _ = spawn(fun() -> run_blocked_ids_warm(Missing) end), + ok; + full -> + bump_counter(?CNT_WARM_DROPPED) + end. + +-spec run_blocked_ids_warm([integer()]) -> ok. +run_blocked_ids_warm(Missing) -> + try + warm_within_budget(Missing) + catch + throw:Reason -> warm_crashed(throw, Reason); + error:Reason -> warm_crashed(error, Reason); + exit:Reason -> warm_crashed(exit, Reason) + after + release_warm_slot() + end. + +-spec warm_crashed(throw | error | exit, term()) -> ok. +warm_crashed(Class, Reason) -> + bump_counter(?CNT_FETCH_FAILURES), + logger:debug("Push: blocked id warm crashed", #{class => Class, reason => Reason}), + ok. + +-spec warm_within_budget([integer()]) -> ok. +warm_within_budget(Missing) -> + {Budget, ChunkSize} = blocked_ids_fetch_budget(), + Budgeted = lists:sublist(Missing, Budget), + count_budget_exhausted(length(Missing) - length(Budgeted)), + fetch_blocked_ids_chunks(chunk_user_ids(Budgeted, ChunkSize, [])). + +-spec blocked_ids_fetch_budget() -> {pos_integer(), pos_integer()}. +blocked_ids_fetch_budget() -> + ChunkSize = blocked_ids_fetch_chunk(), + MaxUsers = blocked_ids_fetch_max_users(), + {min(MaxUsers, ChunkSize * ?MAX_FETCH_RPCS), ChunkSize}. + +-spec count_budget_exhausted(non_neg_integer()) -> ok. +count_budget_exhausted(0) -> + ok; +count_budget_exhausted(Skipped) -> + bump_counter(?CNT_BUDGET_EXHAUSTED, Skipped). + +-spec fetch_blocked_ids_chunks([[integer()]]) -> ok. +fetch_blocked_ids_chunks([]) -> + ok; +fetch_blocked_ids_chunks([Chunk | Rest]) -> + case fetch_blocked_ids_chunk(Chunk) of + ok -> fetch_blocked_ids_chunks(Rest); + error -> ok + end. + +-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]]. +chunk_user_ids([], _ChunkSize, Acc) -> + lists:reverse(Acc); +chunk_user_ids(UserIds, ChunkSize, Acc) -> + Chunk = lists:sublist(UserIds, ChunkSize), + chunk_user_ids(drop_prefix(UserIds, ChunkSize), ChunkSize, [Chunk | Acc]). + +-spec drop_prefix([integer()], non_neg_integer()) -> [integer()]. +drop_prefix(UserIds, 0) -> + UserIds; +drop_prefix([], _N) -> + []; +drop_prefix([_UserId | Rest], N) -> + drop_prefix(Rest, N - 1). + +-spec fetch_blocked_ids_chunk([integer()]) -> ok | error. +fetch_blocked_ids_chunk([]) -> + ok; +fetch_blocked_ids_chunk(UserIds) -> + bump_counter(?CNT_FETCH_ATTEMPTS), + Request = #{ + <<"type">> => <<"get_user_blocked_ids">>, + <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds] + }, + case rpc_client:call(Request) of + {ok, Data} when is_map(Data) -> + cache_blocked_ids_response(UserIds, Data); + {ok, _Data} -> + fetch_blocked_ids_failed(malformed_response, length(UserIds)); + {error, Reason} -> + fetch_blocked_ids_failed(Reason, length(UserIds)) + end. + +-spec fetch_blocked_ids_failed(term(), non_neg_integer()) -> error. +fetch_blocked_ids_failed(Reason, UserCount) -> + bump_counter(?CNT_FETCH_FAILURES), + logger:debug("Push: blocked id fetch failed", #{ + reason => Reason, user_count => UserCount + }), + error. + +-spec cache_blocked_ids_response([integer()], map()) -> ok. +cache_blocked_ids_response(UserIds, Data) -> + lists:foreach( + fun(UserId) -> cache_blocked_ids_entry(UserId, Data) end, + UserIds + ), + ok. + +-spec cache_blocked_ids_entry(integer(), map()) -> ok. +cache_blocked_ids_entry(UserId, Data) -> + Raw = maps:get(integer_to_binary(UserId), Data, []), + push_ets_cache:put_blocked_ids_fetched(UserId, blocked_ids_from_response(Raw)). + +-spec blocked_ids_from_response(term()) -> [integer()]. +blocked_ids_from_response(Values) when is_list(Values) -> + lists:filtermap(fun snowflake_id:filter/1, Values); +blocked_ids_from_response(_Values) -> + []. + +-spec blocked_ids_fetch_max_users() -> pos_integer(). +blocked_ids_fetch_max_users() -> + Value = app_pos_integer(push_blocked_ids_fetch_max_users, ?DEFAULT_FETCH_USERS), + min(Value, ?MAX_FETCH_USERS). + +-spec blocked_ids_fetch_chunk() -> pos_integer(). +blocked_ids_fetch_chunk() -> + Value = app_pos_integer(push_blocked_ids_fetch_chunk, ?DEFAULT_FETCH_CHUNK), + min(max(Value, ?MIN_FETCH_CHUNK), ?MAX_FETCH_CHUNK). + +-spec app_pos_integer(atom(), pos_integer()) -> pos_integer(). +app_pos_integer(Key, Default) -> + case application:get_env(fluxer_gateway, Key, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> Default + end. + +-spec blocked_ids_counters() -> map(). +blocked_ids_counters() -> + #{ + blocked_ids_fetch_attempts => read_counter(?CNT_FETCH_ATTEMPTS), + blocked_ids_fetch_failures => read_counter(?CNT_FETCH_FAILURES), + blocked_ids_suppressed => read_counter(?CNT_SUPPRESSED), + blocked_ids_budget_exhausted => read_counter(?CNT_BUDGET_EXHAUSTED), + blocked_ids_warm_dropped => read_counter(?CNT_WARM_DROPPED), + blocked_ids_warm_inflight => read_counter(?CNT_WARM_INFLIGHT) + }. + +-spec claim_warm_slot() -> ok | full. +claim_warm_slot() -> + try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, 1}) of + Value when is_integer(Value), Value =< ?MAX_WARM_INFLIGHT -> + ok; + _Value -> + release_warm_slot(), + full + catch + error:badarg -> claim_first_warm_slot() + end. + +-spec claim_first_warm_slot() -> ok | full. +claim_first_warm_slot() -> + try ets:insert_new(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 1}) of + true -> ok; + false -> full + catch + error:badarg -> full + end. + +-spec release_warm_slot() -> ok. +release_warm_slot() -> + try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, -1, 0, 0}) of + _Value -> ok + catch + error:badarg -> ok + end. + +-spec unstick_warm_gate() -> ok. +unstick_warm_gate() -> + case read_counter(?CNT_WARM_INFLIGHT) of + Value when is_integer(Value), Value >= ?MAX_WARM_INFLIGHT -> reset_warm_gate(); + _Value -> ok + end. + +-spec reset_warm_gate() -> ok. +reset_warm_gate() -> + try ets:insert(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 0}) of + _Value -> ok + catch + error:badarg -> ok + end. + -spec dispatch_if_eligible( [integer()], map(), @@ -408,15 +687,30 @@ dispatch_if_eligible( ok -> ok; dropped -> - logger:debug("Push: dispatcher saturated, dropping notification job", #{ - message_id => MessageId, - channel_id => ChannelId, - guild_id => GuildId, - eligible_count => length(EligibleUsers) - }), + EligibleCount = length(EligibleUsers), + count_dispatch_dropped(EligibleCount), + log_dispatch_drop( + loss_logging_enabled(), MessageId, ChannelId, GuildId, EligibleCount + ), ok end. +-spec log_dispatch_drop(boolean(), integer(), integer(), integer(), non_neg_integer()) -> ok. +log_dispatch_drop(true, MessageId, ChannelId, GuildId, EligibleCount) -> + logger:error("Push: dispatcher saturated, dropping notification job", #{ + message_id => MessageId, + channel_id => ChannelId, + guild_id => GuildId, + eligible_count => EligibleCount + }); +log_dispatch_drop(false, MessageId, ChannelId, GuildId, EligibleCount) -> + logger:debug("Push: dispatcher saturated, dropping notification job", #{ + message_id => MessageId, + channel_id => ChannelId, + guild_id => GuildId, + eligible_count => EligibleCount + }). + -spec handle_sync_user_blocked_ids(integer(), term(), state()) -> {noreply, state()}. handle_sync_user_blocked_ids(UserId, BlockedIds, State) -> case push_normalize:integer_list(BlockedIds) of @@ -449,22 +743,138 @@ handle_clear_channel_notifications(UserId, ChannelId, MessageId, State) -> ok -> ok; dropped -> - logger:debug("Push: dispatcher saturated, dropping clear notification job", #{ - user_id => UserId, channel_id => ChannelId, message_id => MessageId - }) + count_clear_dropped(), + log_clear_drop(loss_logging_enabled(), UserId, ChannelId, MessageId) end, {noreply, State}. +-spec log_clear_drop(boolean(), integer(), integer(), integer()) -> ok. +log_clear_drop(true, UserId, ChannelId, MessageId) -> + logger:warning("Push: dispatcher saturated, dropping clear notification job", #{ + user_id => UserId, channel_id => ChannelId, message_id => MessageId + }); +log_clear_drop(false, UserId, ChannelId, MessageId) -> + logger:debug("Push: dispatcher saturated, dropping clear notification job", #{ + user_id => UserId, channel_id => ChannelId, message_id => MessageId + }). + -spec log_message_worker_drop(ok | dropped, map()) -> ok. log_message_worker_drop(ok, _Params) -> ok; log_message_worker_drop(dropped, Params) -> + bump_counter(?CNT_WORKER_POOL), MessageData = maps:get(message_data, Params, #{}), + log_worker_pool_drop( + loss_logging_enabled(), + maps:get(<<"id">>, MessageData, undefined), + maps:get(<<"channel_id">>, MessageData, undefined) + ). + +-spec log_worker_pool_drop(boolean(), term(), term()) -> ok. +log_worker_pool_drop(true, MessageId, ChannelId) -> + logger:error("Push: worker pool saturated, dropping message create", #{ + message_id => MessageId, channel_id => ChannelId + }); +log_worker_pool_drop(false, MessageId, ChannelId) -> logger:debug("Push: worker pool saturated, dropping message create", #{ - message_id => maps:get(<<"id">>, MessageData, undefined), - channel_id => maps:get(<<"channel_id">>, MessageData, undefined) + message_id => MessageId, channel_id => ChannelId }). +-spec cache_stats_with_counters() -> map(). +cache_stats_with_counters() -> + Base = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()), + maps:merge(Base, push_loss_counters()). + +-spec push_loss_counters() -> map(). +push_loss_counters() -> + #{ + counters => counter_table_status(), + worker_pool_dropped => read_counter(?CNT_WORKER_POOL), + dispatch_dropped => read_counter(?CNT_DISPATCH_DROPPED), + dispatch_dropped_users => read_counter(?CNT_DISPATCH_DROPPED_USERS), + clear_dispatch_dropped => read_counter(?CNT_CLEAR_DROPPED), + dispatcher_queue_full => read_counter(?CNT_QUEUE_FULL), + dispatcher_invalid_job => read_counter(?CNT_INVALID_JOB), + dispatcher_enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT), + dispatcher_enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED), + dispatcher_job_crashed => read_counter(?CNT_JOB_CRASHED), + dispatcher_worker_died => read_counter(?CNT_WORKER_DIED), + dispatcher_restarts => read_counter(?CNT_DISPATCHER_RESTARTS), + dispatcher_restart_discarded => read_counter(?CNT_RESTART_DISCARDED), + dispatcher_queue_backlog => dispatcher_queue_backlog() + }. + +-spec count_dispatch_dropped(non_neg_integer()) -> ok. +count_dispatch_dropped(EligibleCount) -> + bump_counter(?CNT_DISPATCH_DROPPED), + bump_counter(?CNT_DISPATCH_DROPPED_USERS, EligibleCount). + +-spec count_clear_dropped() -> ok. +count_clear_dropped() -> + bump_counter(?CNT_CLEAR_DROPPED). + +-spec counter_table_status() -> live | unavailable. +counter_table_status() -> + case ets:info(?PUSH_COUNTER_TABLE, size) of + Size when is_integer(Size) -> live; + _ -> unavailable + end. + +-spec dispatcher_queue_backlog() -> non_neg_integer() | unavailable. +dispatcher_queue_backlog() -> + backlog(read_counter(?CNT_QUEUE_ENQUEUED), read_counter(?CNT_QUEUE_DEQUEUED)). + +-spec backlog(non_neg_integer() | unavailable, non_neg_integer() | unavailable) -> + non_neg_integer() | unavailable. +backlog(Enqueued, Dequeued) when is_integer(Enqueued), is_integer(Dequeued) -> + max(0, Enqueued - Dequeued); +backlog(Enqueued, unavailable) when is_integer(Enqueued) -> + Enqueued; +backlog(_Enqueued, _Dequeued) -> + unavailable. + +-spec loss_logging_enabled() -> boolean(). +loss_logging_enabled() -> + application:get_env(fluxer_gateway, push_loss_logging, false) =:= true. + +-spec read_counter(atom()) -> non_neg_integer() | unavailable. +read_counter(Key) -> + try ets:lookup(?PUSH_COUNTER_TABLE, Key) of + [{Key, Value}] when is_integer(Value), Value >= 0 -> Value; + _ -> unavailable + catch + error:badarg -> unavailable + end. + +-spec bump_counter(atom()) -> ok. +bump_counter(Key) -> + bump_counter(Key, 1). + +-spec bump_counter(atom(), non_neg_integer()) -> ok. +bump_counter(Key, Increment) -> + try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of + _Value -> ok + catch + error:badarg -> insert_missing_counter(Key, Increment) + end. + +-spec insert_missing_counter(atom(), non_neg_integer()) -> ok. +insert_missing_counter(Key, Increment) -> + try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of + true -> ok; + false -> retry_bump_counter(Key, Increment) + catch + error:badarg -> ok + end. + +-spec retry_bump_counter(atom(), non_neg_integer()) -> ok. +retry_bump_counter(Key, Increment) -> + try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of + _Value -> ok + catch + error:badarg -> ok + end. + -spec local_cache_mutation(fun(() -> ok)) -> ok. local_cache_mutation(Fun) -> case whereis(?MODULE) of @@ -521,6 +931,13 @@ env_boolean(Key) -> _ -> false end. +-spec env_boolean(atom(), boolean()) -> boolean(). +env_boolean(Key, Default) -> + case fluxer_gateway_env:get(Key) of + Value when is_boolean(Value) -> Value; + _ -> Default + end. + -spec env_non_neg_integer(atom(), non_neg_integer()) -> non_neg_integer(). env_non_neg_integer(Key, Default) -> case fluxer_gateway_env:get(Key) of @@ -580,12 +997,13 @@ filter_eligible_users_fetches_large_metadata_once_test() -> fun(UserId) -> push_ets_cache:put_user_guild_settings(UserId, 42, #{}) end, [1, 2, 3] ), + lists:foreach(fun(UserId) -> push_ets_cache:put_blocked_ids(UserId, []) end, [1, 2, 3]), Self = self(), ok = meck:new(push_eligibility_checks, [passthrough, no_link]), try ok = meck:expect(push_eligibility_checks, get_guild_large_metadata, fun(42) -> Self ! metadata_lookup, - #{member_count => 300, features => []} + #{member_count => 3000, features => []} end), MessageData = #{ <<"channel_type">> => 0, @@ -593,7 +1011,9 @@ filter_eligible_users_fetches_large_metadata_once_test() -> }, ?assertEqual( [1], - filter_eligible_users([1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{}) + filter_eligible_users( + [1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{}, undefined + ) ), ?assertEqual(1, drain_metadata_lookup_count(0)) after @@ -604,6 +1024,212 @@ filter_eligible_users_fetches_large_metadata_once_test() -> ) end. +blocked_ids_fetch_defaults_are_bounded_test() -> + ?assertEqual(?DEFAULT_FETCH_USERS, blocked_ids_fetch_max_users()), + ?assertEqual(?DEFAULT_FETCH_CHUNK, blocked_ids_fetch_chunk()). + +blocked_ids_from_response_skips_unparsable_entries_test() -> + ?assertEqual([123, 456], blocked_ids_from_response([<<"123">>, <<"abc">>, 456, null])), + ?assertEqual([], blocked_ids_from_response(<<"not_a_list">>)). + +chunk_user_ids_splits_into_bounded_batches_test() -> + ?assertEqual([], chunk_user_ids([], 2, [])), + ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])), + ?assertEqual([[1, 2, 3]], chunk_user_ids([1, 2, 3], 500, [])). + +blocked_ids_fetch_budget_is_capped_test() -> + assert_budget_capped(), + with_fetch_env(1, 1000000, fun assert_budget_capped/0), + with_fetch_env(1000000, 1000000, fun assert_budget_capped/0), + assert_budget_capped(). + +assert_budget_capped() -> + {Budget, ChunkSize} = blocked_ids_fetch_budget(), + ?assert(ChunkSize >= ?MIN_FETCH_CHUNK), + ?assert(ChunkSize =< ?MAX_FETCH_CHUNK), + ?assert(Budget =< ?MAX_FETCH_USERS), + ?assert(length(chunk_user_ids(lists:seq(1, Budget), ChunkSize, [])) =< ?MAX_FETCH_RPCS). + +synced_blocked_recipients_are_dropped_test() -> + push_ets_cache:init(), + ok = push_ets_cache:put_blocked_ids(5020, [999]), + ?assertEqual([], filter_dm_recipients([5020], 999)). + +fetched_blocked_recipients_are_dropped_test() -> + push_ets_cache:init(), + ok = push_ets_cache:put_blocked_ids_fetched(5031, [999]), + ?assertEqual([], filter_dm_recipients([5031], 999)). + +block_suppression_drops_blocked_recipient_test() -> + push_ets_cache:init(), + ok = push_ets_cache:put_blocked_ids(5002, [999]), + ok = push_ets_cache:put_blocked_ids(5003, []), + ?assertEqual([5003], filter_dm_recipients([5002, 5003], 999)). + +blocked_ids_fetch_stops_after_a_failing_chunk_test() -> + ?assertEqual(ok, fetch_blocked_ids_chunks([])), + ?assertEqual(ok, fetch_blocked_ids_chunks([[], []])). + +cached_recipients_need_no_blocked_ids_warm_test() -> + push_ets_cache:init(), + ok = push_ets_cache:put_blocked_ids_fetched(5004, []), + ?assertEqual([], missing_blocked_ids([5004])), + ?assertEqual(ok, warm_blocked_ids([5004])). + +blocked_ids_counters_are_unavailable_without_the_shared_table_test() -> + delete_counter_table(), + ?assertEqual(unavailable, read_counter(?CNT_SUPPRESSED)), + ?assertEqual(full, claim_warm_slot()), + ?assertEqual(ok, release_warm_slot()), + ?assertEqual(ok, unstick_warm_gate()). + +blocked_ids_warm_gate_bounds_concurrency_test() -> + with_counter_table(fun() -> + Claims = [claim_warm_slot() || _ <- lists:seq(1, ?MAX_WARM_INFLIGHT + 2)], + ?assertEqual(?MAX_WARM_INFLIGHT, length([ok || ok <- Claims])), + ?assertEqual(?MAX_WARM_INFLIGHT, read_counter(?CNT_WARM_INFLIGHT)), + lists:foreach( + fun(_) -> release_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT + 2) + ), + ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT)) + end). + +blocked_ids_warm_gate_unsticks_in_the_eviction_sweep_test() -> + with_counter_table(fun() -> + lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)), + ?assertEqual(full, claim_warm_slot()), + unstick_warm_gate(), + ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT)), + ?assertEqual(ok, claim_warm_slot()) + end). + +blocked_ids_warm_is_dropped_when_the_gate_is_full_test() -> + push_ets_cache:init(), + with_counter_table(fun() -> + lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)), + ?assertEqual(ok, warm_blocked_ids([5099])), + ?assertEqual(1, read_counter(?CNT_WARM_DROPPED)) + end). + +blocked_ids_counters_are_exposed_in_cache_stats_test() -> + push_ets_cache:init(), + with_counter_table(fun() -> + ok = push_ets_cache:put_blocked_ids(5010, [999]), + ?assertEqual([], filter_dm_recipients([5010], 999)), + ?assertEqual(1, read_counter(?CNT_SUPPRESSED)), + assert_cache_stats_expose_blocked_ids() + end). + +assert_cache_stats_expose_blocked_ids() -> + Stats = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()), + lists:foreach( + fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end, + [ + blocked_ids_size, + blocked_ids_fetch_attempts, + blocked_ids_fetch_failures, + blocked_ids_suppressed, + blocked_ids_budget_exhausted, + blocked_ids_warm_dropped, + blocked_ids_warm_inflight + ] + ). + +push_loss_counters_expose_every_counter_push_writes_test() -> + with_counter_table(fun() -> + ok = log_message_worker_drop(dropped, #{}), + ok = count_dispatch_dropped(12), + ok = count_clear_dropped(), + Stats = push_loss_counters(), + ?assertEqual(live, maps:get(counters, Stats)), + ?assertEqual(1, maps:get(worker_pool_dropped, Stats)), + ?assertEqual(1, maps:get(dispatch_dropped, Stats)), + ?assertEqual(12, maps:get(dispatch_dropped_users, Stats)), + ?assertEqual(1, maps:get(clear_dispatch_dropped, Stats)) + end). + +push_loss_counters_are_unavailable_without_the_shared_table_test() -> + delete_counter_table(), + Stats = push_loss_counters(), + ?assertEqual(unavailable, maps:get(counters, Stats)), + ?assertEqual(unavailable, maps:get(worker_pool_dropped, Stats)), + ?assertEqual(unavailable, maps:get(dispatch_dropped, Stats)), + ?assertEqual(unavailable, maps:get(dispatcher_enqueue_timeout, Stats)), + ?assertEqual(unavailable, maps:get(dispatcher_queue_backlog, Stats)). + +push_loss_counters_keep_a_genuine_zero_distinct_from_absent_test() -> + with_counter_table(fun() -> + ok = count_dispatch_dropped(0), + Stats = push_loss_counters(), + ?assertEqual(1, maps:get(dispatch_dropped, Stats)), + ?assertEqual(0, maps:get(dispatch_dropped_users, Stats)), + ?assertEqual(unavailable, maps:get(clear_dispatch_dropped, Stats)) + end). + +dispatcher_queue_backlog_survives_an_untrappable_dispatcher_kill_test() -> + with_counter_table(fun() -> + ?assertEqual(unavailable, dispatcher_queue_backlog()), + ok = bump_counter(?CNT_QUEUE_ENQUEUED, 9), + ?assertEqual(9, dispatcher_queue_backlog()), + ok = bump_counter(?CNT_QUEUE_DEQUEUED, 4), + ?assertEqual(5, dispatcher_queue_backlog()) + end). + +cache_stats_with_counters_carries_the_loss_surface_test() -> + push_ets_cache:init(), + with_counter_table(fun() -> + Stats = cache_stats_with_counters(), + lists:foreach( + fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end, + [ + blocked_ids_size, + blocked_ids_suppressed, + counters, + worker_pool_dropped, + dispatch_dropped, + dispatch_dropped_users, + clear_dispatch_dropped, + dispatcher_queue_full, + dispatcher_enqueue_timeout, + dispatcher_job_crashed, + dispatcher_worker_died, + dispatcher_restarts, + dispatcher_restart_discarded, + dispatcher_queue_backlog + ] + ) + end). + +with_counter_table(Fun) -> + delete_counter_table(), + _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]), + try + Fun() + after + delete_counter_table() + end. + +delete_counter_table() -> + try ets:delete(?PUSH_COUNTER_TABLE) of + _ -> ok + catch + error:badarg -> ok + end. + +filter_dm_recipients(UserIds, AuthorId) -> + MessageData = #{<<"channel_type">> => 1}, + filter_eligible_users(UserIds, AuthorId, 0, 10, MessageData, 0, #{}, #{}, undefined). + +with_fetch_env(ChunkSize, MaxUsers, Fun) -> + application:set_env(fluxer_gateway, push_blocked_ids_fetch_chunk, ChunkSize), + application:set_env(fluxer_gateway, push_blocked_ids_fetch_max_users, MaxUsers), + try + Fun() + after + application:unset_env(fluxer_gateway, push_blocked_ids_fetch_chunk), + application:unset_env(fluxer_gateway, push_blocked_ids_fetch_max_users) + end. + filter_eligible_users_batches_missing_settings_lookups_test() -> push_ets_cache:init(), Self = self(), @@ -615,7 +1241,7 @@ filter_eligible_users_batches_missing_settings_lookups_test() -> end), ?assertEqual( [1, 2, 3], - filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{}) + filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{}, undefined) ), ?assertEqual(1, drain_settings_request_count(0)) after diff --git a/fluxer_gateway/src/push/push_dispatcher.erl b/fluxer_gateway/src/push/push_dispatcher.erl index 73ca0158e..93e9755c2 100644 --- a/fluxer_gateway/src/push/push_dispatcher.erl +++ b/fluxer_gateway/src/push/push_dispatcher.erl @@ -17,6 +17,19 @@ -define(DEFAULT_MAX_QUEUE, 10000). -define(ENQUEUE_TIMEOUT_MS, 1000). +-define(PUSH_COUNTER_TABLE, push_worker_counter). +-define(CNT_QUEUE_FULL, push_loss_queue_full). +-define(CNT_INVALID_JOB, push_loss_invalid_job). +-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout). +-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed). +-define(CNT_JOB_CRASHED, push_loss_job_crashed). +-define(CNT_WORKER_DIED, push_loss_worker_died). +-define(CNT_WORKER_POOL, push_loss_worker_pool). +-define(CNT_RESTARTS, push_dispatcher_restarts). +-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded). +-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued). +-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued). + -type push_job() :: #{ type := message_create, @@ -44,7 +57,27 @@ inflight := non_neg_integer(), workers := #{reference() => true}, max_inflight := pos_integer(), - max_queue := pos_integer() + max_queue := pos_integer(), + started_at => integer() +}. + +-type counter_value() :: non_neg_integer() | unavailable. + +-type stats() :: #{ + queued := non_neg_integer(), + inflight := non_neg_integer(), + counters := live | unavailable, + dispatcher_uptime_seconds := non_neg_integer() | undefined, + dispatcher_restarts := counter_value(), + restart_discarded_jobs := counter_value(), + queue_backlog_lost := counter_value(), + queue_full_dropped := counter_value(), + invalid_job_dropped := counter_value(), + enqueue_timeout := counter_value(), + enqueue_failed := counter_value(), + job_crashed := counter_value(), + worker_died := counter_value(), + worker_pool_dropped := counter_value() }. -spec start_link() -> {ok, pid()} | {error, term()} | ignore. @@ -135,25 +168,57 @@ enqueue_clear_notifications(UserId, ChannelId, MessageId, BadgeCountsTtlSeconds) ), safe_enqueue(Job). --spec stats() -> #{queued := non_neg_integer(), inflight := non_neg_integer()} | #{}. +-spec stats() -> stats() | #{}. stats() -> - try - gen_server:call(?MODULE, stats, 1000) + Enqueued = read_counter(?CNT_QUEUE_ENQUEUED), + try gen_server:call(?MODULE, stats, 1000) of + #{queued := Queued, inflight := Inflight} = Reply -> + StartedAt = maps:get(started_at, Reply, undefined), + stats_with_loss(Queued, Inflight, StartedAt, Enqueued); + _ -> + #{} catch exit:_ -> #{}; error:_ -> #{} end. +-spec stats_with_loss(non_neg_integer(), non_neg_integer(), term(), term()) -> stats(). +stats_with_loss(Queued, Inflight, StartedAt, Enqueued) -> + #{ + queued => Queued, + inflight => Inflight, + counters => counter_table_status(), + dispatcher_uptime_seconds => uptime_seconds(StartedAt), + dispatcher_restarts => read_counter(?CNT_RESTARTS), + restart_discarded_jobs => read_counter(?CNT_RESTART_DISCARDED), + queue_backlog_lost => queue_backlog_lost(Enqueued, Queued), + queue_full_dropped => read_counter(?CNT_QUEUE_FULL), + invalid_job_dropped => read_counter(?CNT_INVALID_JOB), + enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT), + enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED), + job_crashed => read_counter(?CNT_JOB_CRASHED), + worker_died => read_counter(?CNT_WORKER_DIED), + worker_pool_dropped => read_counter(?CNT_WORKER_POOL) + }. + +-spec uptime_seconds(term()) -> non_neg_integer() | undefined. +uptime_seconds(StartedAt) when is_integer(StartedAt) -> + max(0, erlang:monotonic_time(second) - StartedAt); +uptime_seconds(_StartedAt) -> + undefined. + -spec init([]) -> {ok, state()}. init([]) -> erlang:process_flag(fullsweep_after, 10), + bump_counter(?CNT_RESTARTS), {ok, #{ queue => queue:new(), queued => 0, inflight => 0, workers => #{}, max_inflight => budget_aware_max_inflight(), - max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE) + max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE), + started_at => erlang:monotonic_time(second) }}. -spec budget_aware_max_inflight() -> pos_integer(). @@ -167,7 +232,12 @@ budget_aware_max_inflight() -> -spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}. handle_call(stats, _From, #{queued := Queued, inflight := Inflight} = State) -> - {reply, #{queued => Queued, inflight => Inflight}, State}; + Reply = #{ + queued => Queued, + inflight => Inflight, + started_at => maps:get(started_at, State, undefined) + }, + {reply, Reply, State}; handle_call({enqueue, Job}, _From, State) -> {Result, State1} = handle_enqueue(Job, State), {reply, Result, State1}; @@ -183,11 +253,12 @@ handle_cast(_Msg, State) -> -spec handle_info(term(), state()) -> {noreply, state()}. handle_info( - {'DOWN', Ref, process, _Pid, _Reason}, + {'DOWN', Ref, process, _Pid, Reason}, #{workers := Workers, inflight := Inflight} = State ) -> case maps:is_key(Ref, Workers) of true -> + count_worker_down(Reason), RemainingWorkers = maps:remove(Ref, Workers), DecrementedInflight = max(0, Inflight - 1), drain_queue(State#{ @@ -201,8 +272,15 @@ handle_info(_Info, State) -> {noreply, State}. -spec terminate(term(), state()) -> ok. -terminate(_Reason, _State) -> - ok. +terminate(_Reason, State) -> + bump_counter(?CNT_RESTART_DISCARDED, maps:get(queued, State, 0)). + +-spec count_worker_down(term()) -> ok. +count_worker_down(normal) -> + ok; +count_worker_down(_Reason) -> + bump_counter(?CNT_JOB_CRASHED), + bump_counter(?CNT_WORKER_DIED). -spec code_change(term(), state(), term()) -> {ok, state()}. code_change(_OldVsn, State, _Extra) -> @@ -239,22 +317,40 @@ maybe_enqueue_or_start(Job, #{inflight := Inflight, max_inflight := MaxInflight} maybe_enqueue(Job, #{queued := Queued, max_queue := MaxQueue, queue := Queue0} = State) -> case Queued < MaxQueue of true -> + bump_counter(?CNT_QUEUE_ENQUEUED), Queue1 = queue:in(Job, Queue0), {ok, State#{queue := Queue1, queued := Queued + 1}}; false -> + bump_counter(?CNT_QUEUE_FULL), DropCount = bump_drop_count(), - logger:warning( - "Push: queue full, dropping job", - #{ - message_id => maps:get(message_id, Job, undefined), - queued => Queued, - max_queue => MaxQueue, - total_dropped => DropCount - } - ), + log_queue_full_drop(loss_logging_enabled(), Job, Queued, MaxQueue, DropCount), {dropped, State} end. +-spec log_queue_full_drop( + boolean(), push_job(), non_neg_integer(), pos_integer(), non_neg_integer() +) -> ok. +log_queue_full_drop(true, Job, Queued, MaxQueue, _DropCount) -> + logger:error( + "Push: queue full, dropping job", + #{ + message_id => maps:get(message_id, Job, undefined), + queued => Queued, + max_queue => MaxQueue, + total_dropped => read_counter(?CNT_QUEUE_FULL) + } + ); +log_queue_full_drop(false, Job, Queued, MaxQueue, DropCount) -> + logger:warning( + "Push: queue full, dropping job", + #{ + message_id => maps:get(message_id, Job, undefined), + queued => Queued, + max_queue => MaxQueue, + total_dropped => DropCount + } + ). + -spec bump_drop_count() -> non_neg_integer(). bump_drop_count() -> Current = @@ -266,6 +362,68 @@ bump_drop_count() -> erlang:put(push_dispatcher_drop_count, Updated), Updated. +-spec loss_logging_enabled() -> boolean(). +loss_logging_enabled() -> + application:get_env(fluxer_gateway, push_loss_logging, false) =:= true. + +-spec counter_table_status() -> live | unavailable. +counter_table_status() -> + case ets:info(?PUSH_COUNTER_TABLE, size) of + Size when is_integer(Size) -> live; + _ -> unavailable + end. + +-spec queue_backlog_lost(term(), non_neg_integer()) -> counter_value(). +queue_backlog_lost(Enqueued, Queued) when is_integer(Enqueued) -> + max(0, Enqueued - dequeued_total() - Queued); +queue_backlog_lost(_Enqueued, _Queued) -> + unavailable. + +-spec dequeued_total() -> non_neg_integer(). +dequeued_total() -> + case read_counter(?CNT_QUEUE_DEQUEUED) of + Dequeued when is_integer(Dequeued) -> Dequeued; + unavailable -> 0 + end. + +-spec read_counter(atom()) -> counter_value(). +read_counter(Key) -> + try ets:lookup(?PUSH_COUNTER_TABLE, Key) of + [{Key, Value}] when is_integer(Value), Value >= 0 -> Value; + _ -> unavailable + catch + error:badarg -> unavailable + end. + +-spec bump_counter(atom()) -> ok. +bump_counter(Key) -> + bump_counter(Key, 1). + +-spec bump_counter(atom(), non_neg_integer()) -> ok. +bump_counter(Key, Increment) -> + try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of + _Value -> ok + catch + error:badarg -> insert_missing_counter(Key, Increment) + end. + +-spec insert_missing_counter(atom(), non_neg_integer()) -> ok. +insert_missing_counter(Key, Increment) -> + try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of + true -> ok; + false -> retry_bump_counter(Key, Increment) + catch + error:badarg -> ok + end. + +-spec retry_bump_counter(atom(), non_neg_integer()) -> ok. +retry_bump_counter(Key, Increment) -> + try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of + _Value -> ok + catch + error:badarg -> ok + end. + -spec start_job(push_job(), state()) -> state(). start_job(Job, #{workers := Workers, inflight := Inflight} = State) -> {_Pid, Ref} = @@ -340,6 +498,7 @@ log_enqueue_send_notifications(UserIds, GuildId, ChannelId, MessageId) -> state() ) -> {noreply, state()}. drain_available_queue({{value, Job}, Queue1}, Queued, State) -> + bump_counter(?CNT_QUEUE_DEQUEUED), State1 = State#{queue := Queue1, queued := max(0, Queued - 1)}, State2 = start_job(Job, State1), drain_queue(State2); @@ -353,14 +512,29 @@ run_job(#{message_id := MessageId} = Job) -> logger:debug("Push: worker completed", #{message_id => MessageId}), ok catch - Class:Reason -> - logger:debug( - "Push: worker crashed", - #{message_id => MessageId, class => Class, reason => Reason} - ), + Class:Reason:Stacktrace -> + bump_counter(?CNT_JOB_CRASHED), + log_job_crash(loss_logging_enabled(), MessageId, Class, Reason, Stacktrace), ok end. +-spec log_job_crash(boolean(), integer(), atom(), term(), list()) -> ok. +log_job_crash(true, MessageId, Class, Reason, Stacktrace) -> + logger:error( + "Push: worker crashed", + #{ + message_id => MessageId, + class => Class, + reason => Reason, + stacktrace => Stacktrace + } + ); +log_job_crash(false, MessageId, Class, Reason, _Stacktrace) -> + logger:debug( + "Push: worker crashed", + #{message_id => MessageId, class => Class, reason => Reason} + ). + -spec run_typed_job(message_create | clear_channel, push_job()) -> ok. run_typed_job(message_create, #{ user_ids := UserIds, @@ -412,20 +586,39 @@ get_int_or_default(Key, Default) -> -spec safe_enqueue(push_job()) -> ok | dropped. safe_enqueue(Job) -> try gen_server:call(?MODULE, {enqueue, Job}, ?ENQUEUE_TIMEOUT_MS) of - ok -> ok; - dropped -> dropped; - _ -> dropped + ok -> + ok; + dropped -> + dropped; + _ -> + bump_counter(?CNT_ENQUEUE_FAILED), + dropped catch - throw:_Reason -> dropped; - error:_Reason -> dropped; - exit:_Reason -> dropped + throw:_Reason -> + bump_counter(?CNT_ENQUEUE_FAILED), + dropped; + error:_Reason -> + bump_counter(?CNT_ENQUEUE_FAILED), + dropped; + exit:Reason -> + count_enqueue_exit(Reason), + dropped end. +-spec count_enqueue_exit(term()) -> ok. +count_enqueue_exit({timeout, _Call}) -> + bump_counter(?CNT_ENQUEUE_TIMEOUT); +count_enqueue_exit(_Reason) -> + bump_counter(?CNT_ENQUEUE_FAILED). + -spec handle_enqueue(term(), state()) -> {ok | dropped, state()}. handle_enqueue(Job0, State) -> case push_job(Job0) of - {ok, Job} -> maybe_enqueue_or_start(Job, State); - error -> {dropped, State} + {ok, Job} -> + maybe_enqueue_or_start(Job, State); + error -> + bump_counter(?CNT_INVALID_JOB), + {dropped, State} end. -spec push_job(term()) -> {ok, push_job()} | error. @@ -547,3 +740,135 @@ optional_binary(Value) when is_binary(Value) -> {ok, Value}; optional_binary(_) -> error. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +absent_counter_table_reads_unavailable_test() -> + delete_counter_table(), + ?assertEqual(unavailable, counter_table_status()), + ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)), + ?assertEqual(ok, bump_counter(?CNT_QUEUE_FULL)), + ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)). + +absent_counter_key_is_distinct_from_zero_test() -> + with_counter_table(fun() -> + ?assertEqual(live, counter_table_status()), + ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)), + ok = bump_counter(?CNT_RESTART_DISCARDED, 0), + ?assertEqual(0, read_counter(?CNT_RESTART_DISCARDED)) + end). + +bump_counter_creates_then_increments_key_test() -> + with_counter_table(fun() -> + ok = bump_counter(?CNT_QUEUE_FULL), + ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)), + ok = bump_counter(?CNT_QUEUE_FULL, 4), + ?assertEqual(5, read_counter(?CNT_QUEUE_FULL)) + end). + +enqueue_exit_separates_timeout_from_failure_test() -> + with_counter_table(fun() -> + ok = count_enqueue_exit({timeout, {gen_server, call, []}}), + ok = count_enqueue_exit({noproc, {gen_server, call, []}}), + ?assertEqual(1, read_counter(?CNT_ENQUEUE_TIMEOUT)), + ?assertEqual(1, read_counter(?CNT_ENQUEUE_FAILED)) + end). + +worker_down_counts_only_abnormal_exits_test() -> + with_counter_table(fun() -> + ok = count_worker_down(normal), + ?assertEqual(unavailable, read_counter(?CNT_WORKER_DIED)), + ok = count_worker_down(killed), + ok = count_worker_down({shutdown, restarting}), + ?assertEqual(2, read_counter(?CNT_WORKER_DIED)) + end). + +terminate_counts_discarded_queue_depth_test() -> + with_counter_table(fun() -> + ?assertEqual(ok, terminate(shutdown, dispatcher_state(7, 10))), + ?assertEqual(7, read_counter(?CNT_RESTART_DISCARDED)) + end). + +invalid_job_enqueue_is_counted_test() -> + with_counter_table(fun() -> + State = dispatcher_state(0, 10), + ?assertEqual({dropped, State}, handle_enqueue(#{type => invalid}, State)), + ?assertEqual(1, read_counter(?CNT_INVALID_JOB)) + end). + +uptime_seconds_reports_undefined_without_start_time_test() -> + ?assertEqual(undefined, uptime_seconds(undefined)), + ?assertEqual(0, uptime_seconds(erlang:monotonic_time(second) + 5)), + ?assert(is_integer(uptime_seconds(erlang:monotonic_time(second) - 3))). + +abnormal_worker_down_also_counts_a_crashed_job_test() -> + with_counter_table(fun() -> + ok = count_worker_down(normal), + ?assertEqual(unavailable, read_counter(?CNT_JOB_CRASHED)), + ok = count_worker_down(killed), + ?assertEqual(1, read_counter(?CNT_JOB_CRASHED)), + ?assertEqual(1, read_counter(?CNT_WORKER_DIED)) + end). + +enqueue_records_queue_growth_outside_the_process_test() -> + with_counter_table(fun() -> + {ok, State1} = maybe_enqueue(clear_job(), dispatcher_state(0, 10)), + ?assertEqual(1, maps:get(queued, State1)), + ?assertEqual(1, read_counter(?CNT_QUEUE_ENQUEUED)), + ?assertEqual(unavailable, read_counter(?CNT_QUEUE_DEQUEUED)) + end). + +queue_full_drop_does_not_record_queue_growth_test() -> + with_counter_table(fun() -> + {dropped, _State1} = maybe_enqueue(clear_job(), dispatcher_state(3, 3)), + ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)), + ?assertEqual(unavailable, read_counter(?CNT_QUEUE_ENQUEUED)) + end). + +queue_backlog_lost_survives_an_untrappable_kill_test() -> + with_counter_table(fun() -> + ?assertEqual(unavailable, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)), + ok = bump_counter(?CNT_QUEUE_ENQUEUED, 7), + ok = bump_counter(?CNT_QUEUE_DEQUEUED, 2), + ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 5)), + ?assertEqual(5, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)), + ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 900)) + end). + +dispatcher_state(Queued, MaxQueue) -> + #{ + queue => queue:new(), + queued => Queued, + inflight => 0, + workers => #{}, + max_inflight => 1, + max_queue => MaxQueue + }. + +clear_job() -> + #{ + type => clear_channel, + user_id => 1, + channel_id => 2, + message_id => 3, + badge_counts_ttl_seconds => 0 + }. + +with_counter_table(Fun) -> + delete_counter_table(), + _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]), + try + Fun() + after + delete_counter_table() + end. + +delete_counter_table() -> + try ets:delete(?PUSH_COUNTER_TABLE) of + _ -> ok + catch + error:badarg -> ok + end. + +-endif. diff --git a/fluxer_gateway/src/push/push_eligibility_checks.erl b/fluxer_gateway/src/push/push_eligibility_checks.erl index 2d4ec072f..c13d6c20b 100644 --- a/fluxer_gateway/src/push/push_eligibility_checks.erl +++ b/fluxer_gateway/src/push/push_eligibility_checks.erl @@ -16,11 +16,12 @@ -export([override_for_large_guild/2]). -export([enforce_only_mentions/1]). -export([is_large_guild/2]). +-export([large_guild_threshold/0]). -export([has_large_guild_override/1]). -export([get_guild_large_metadata/1]). -export([check_temp_muted/1]). --define(LARGE_GUILD_THRESHOLD, 250). +-define(LARGE_GUILD_THRESHOLD, 2500). -define(LARGE_GUILD_OVERRIDE_FEATURE, <<"LARGE_GUILD_OVERRIDE">>). -define(MESSAGE_NOTIFICATIONS_NULL, -1). -define(MESSAGE_NOTIFICATIONS_ALL, 0). @@ -215,10 +216,14 @@ enforce_only_mentions(CurrentLevel) -> CurrentLevel. -spec is_large_guild(integer() | term(), list()) -> boolean(). is_large_guild(Count, Features) when is_integer(Count) -> - Count > ?LARGE_GUILD_THRESHOLD orelse has_large_guild_override(Features); + Count > large_guild_threshold() orelse has_large_guild_override(Features); is_large_guild(_, Features) -> has_large_guild_override(Features). +-spec large_guild_threshold() -> pos_integer(). +large_guild_threshold() -> + ?LARGE_GUILD_THRESHOLD. + -spec has_large_guild_override(list() | term()) -> boolean(). has_large_guild_override(Features) when is_list(Features) -> lists:member(?LARGE_GUILD_OVERRIDE_FEATURE, Features); @@ -380,11 +385,27 @@ enforce_only_mentions_test() -> ?assertEqual(2, enforce_only_mentions(2)). is_large_guild_test() -> - ?assertEqual(true, is_large_guild(300, [])), + ?assertEqual(true, is_large_guild(3000, [])), + ?assertEqual(false, is_large_guild(300, [])), + ?assertEqual(true, is_large_guild(?LARGE_GUILD_THRESHOLD + 1, [])), + ?assertEqual(false, is_large_guild(?LARGE_GUILD_THRESHOLD, [])), ?assertEqual(false, is_large_guild(100, [])), ?assertEqual(true, is_large_guild(100, [<<"LARGE_GUILD_OVERRIDE">>])), + ?assertEqual(true, is_large_guild(300, [<<"LARGE_GUILD_OVERRIDE">>])), ?assertEqual(true, is_large_guild(undefined, [<<"LARGE_GUILD_OVERRIDE">>])). +large_guild_threshold_is_the_production_value_test() -> + ?assertEqual(2500, ?LARGE_GUILD_THRESHOLD), + ?assertEqual(?LARGE_GUILD_THRESHOLD, large_guild_threshold()). + +mid_sized_guilds_keep_all_notifications_test() -> + MessageData = #{<<"channel_type">> => 0}, + Metadata = #{member_count => 1000, features => []}, + ?assertEqual( + true, + check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, Metadata) + ). + has_large_guild_override_test() -> ?assertEqual(true, has_large_guild_override([<<"LARGE_GUILD_OVERRIDE">>])), ?assertEqual(false, has_large_guild_override([<<"OTHER">>])), @@ -392,7 +413,7 @@ has_large_guild_override_test() -> cached_large_guild_metadata_overrides_all_notifications_test() -> MessageData = #{<<"channel_type">> => 0}, - LargeMetadata = #{member_count => 300, features => []}, + LargeMetadata = #{member_count => 3000, features => []}, ?assertEqual( false, check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata) @@ -403,7 +424,7 @@ cached_large_guild_metadata_keeps_mentions_allowed_test() -> <<"channel_type">> => 0, <<"mentions">> => [#{<<"id">> => <<"100">>}] }, - LargeMetadata = #{member_count => 300, features => []}, + LargeMetadata = #{member_count => 3000, features => []}, ?assertEqual( true, check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata) diff --git a/fluxer_gateway/src/push/push_ets_cache.erl b/fluxer_gateway/src/push/push_ets_cache.erl index e36272826..9a45b4f7c 100644 --- a/fluxer_gateway/src/push/push_ets_cache.erl +++ b/fluxer_gateway/src/push/push_ets_cache.erl @@ -14,6 +14,7 @@ delete_subscriptions/1, get_blocked_ids/1, put_blocked_ids/2, + put_blocked_ids_fetched/2, get_badge_count/1, put_badge_count/3, delete_badge_count/1, @@ -31,6 +32,10 @@ -define(MAX_TABLE_ENTRIES, 500000). -define(EVICT_BATCH, 4096). +-define(MAX_EVICT_RESEEKS, 8). +-define(DEFAULT_BLOCKED_IDS_TTL, 300). +-define(MIN_BLOCKED_IDS_TTL, 30). +-define(MAX_BLOCKED_IDS_TTL, 3600). -define(ETS_OPTS, [ named_table, public, set, {read_concurrency, true}, {write_concurrency, true} @@ -125,17 +130,54 @@ get_blocked_ids(UserId) -> -spec get_blocked_ids_ets(integer()) -> [integer()] | undefined. get_blocked_ids_ets(UserId) -> try ets:lookup(?BLOCKED_IDS, UserId) of - [{UserId, BlockedIds}] -> BlockedIds; - _ -> undefined + [{UserId, BlockedIds}] -> + BlockedIds; + [{UserId, BlockedIds, infinity}] -> + BlockedIds; + [{UserId, BlockedIds, ExpiresAt}] when is_integer(ExpiresAt) -> + live_fetched_blocked_ids(BlockedIds, ExpiresAt); + _ -> + undefined catch error:badarg -> undefined end. +-spec live_fetched_blocked_ids([integer()], integer()) -> [integer()] | undefined. +live_fetched_blocked_ids(BlockedIds, ExpiresAt) -> + case erlang:system_time(second) < ExpiresAt of + true -> BlockedIds; + false -> undefined + end. + -spec put_blocked_ids(integer(), [integer()]) -> ok. put_blocked_ids(UserId, BlockedIds) -> + insert_blocked_ids(UserId, BlockedIds, infinity). + +-spec put_blocked_ids_fetched(integer(), [integer()]) -> ok. +put_blocked_ids_fetched(UserId, BlockedIds) -> + ExpiresAt = erlang:system_time(second) + blocked_ids_ttl_seconds(), + insert_blocked_ids(UserId, BlockedIds, ExpiresAt). + +-spec insert_blocked_ids(integer(), [integer()], infinity | integer()) -> ok. +insert_blocked_ids(UserId, BlockedIds, ExpiresAt) -> guard_table_size(?BLOCKED_IDS), - ets:insert(?BLOCKED_IDS, {UserId, BlockedIds}), - ok. + try ets:insert(?BLOCKED_IDS, {UserId, BlockedIds, ExpiresAt}) of + _ -> ok + catch + error:badarg -> ok + end. + +-spec blocked_ids_ttl_seconds() -> pos_integer(). +blocked_ids_ttl_seconds() -> + Value = app_pos_integer(push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL), + min(max(Value, ?MIN_BLOCKED_IDS_TTL), ?MAX_BLOCKED_IDS_TTL). + +-spec app_pos_integer(atom(), pos_integer()) -> pos_integer(). +app_pos_integer(Key, Default) -> + case application:get_env(fluxer_gateway, Key, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> Default + end. -spec get_badge_count(integer()) -> {non_neg_integer(), integer()} | undefined. get_badge_count(UserId) -> @@ -233,12 +275,24 @@ cache_stats() -> -spec evict_tables(map()) -> ok. evict_tables(MaxEntries) -> + _ = expire_blocked_ids(), evict_table(?USER_GUILD_SETTINGS, maps:get(user_guild_settings, MaxEntries, undefined)), evict_table(?SUBSCRIPTIONS, maps:get(subscriptions, MaxEntries, undefined)), evict_table(?BLOCKED_IDS, maps:get(blocked_ids, MaxEntries, undefined)), evict_table(?BADGE_COUNTS, maps:get(badge_counts, MaxEntries, undefined)), ok. +-spec expire_blocked_ids() -> non_neg_integer(). +expire_blocked_ids() -> + Now = erlang:system_time(second), + MatchSpec = [{{'_', '_', '$1'}, [{is_integer, '$1'}, {'=<', '$1', Now}], [true]}], + try ets:select_delete(?BLOCKED_IDS, MatchSpec) of + Deleted when is_integer(Deleted) -> Deleted; + _ -> 0 + catch + error:badarg -> 0 + end. + -spec guard_table_size(atom()) -> ok. guard_table_size(Table) -> case table_size(Table) >= ?MAX_TABLE_ENTRIES of @@ -279,20 +333,52 @@ evict_table(Table, MaxEntries) -> case Size > MaxEntries of true -> ToDelete = Size - MaxEntries, - evict_n(Table, ets:first(Table), ToDelete); + evict_scan(Table, ToDelete, ?MAX_EVICT_RESEEKS); false -> ok end. --spec evict_n(atom(), term(), non_neg_integer()) -> ok. -evict_n(_Table, '$end_of_table', _N) -> +-spec evict_scan(atom(), non_neg_integer(), non_neg_integer()) -> ok. +evict_scan(_Table, 0, _Reseeks) -> ok; +evict_scan(_Table, _N, 0) -> + ok; +evict_scan(Table, N, Reseeks) -> + case evict_n(Table, first_key(Table), N) of + {done, _Remaining} -> ok; + {interrupted, Remaining} -> evict_scan(Table, Remaining, Reseeks - 1) + end. + +-spec evict_n(atom(), term(), non_neg_integer()) -> + {done, non_neg_integer()} | {interrupted, non_neg_integer()}. +evict_n(_Table, '$end_of_table', N) -> + {done, N}; evict_n(_Table, _Key, 0) -> - ok; + {done, 0}; evict_n(Table, Key, N) -> - Next = ets:next(Table, Key), - ets:delete(Table, Key), - evict_n(Table, Next, N - 1). + case next_key(Table, Key) of + {ok, Next} -> + safe_delete(Table, Key), + evict_n(Table, Next, N - 1); + {error, badarg} -> + {interrupted, N} + end. + +-spec first_key(atom()) -> term(). +first_key(Table) -> + try ets:first(Table) of + Key -> Key + catch + error:badarg -> '$end_of_table' + end. + +-spec next_key(atom(), term()) -> {ok, term()} | {error, badarg}. +next_key(Table, Key) -> + try ets:next(Table, Key) of + Next -> {ok, Next} + catch + error:badarg -> {error, badarg} + end. -spec rebalance_table(atom(), fun((term()) -> integer() | undefined)) -> non_neg_integer(). rebalance_table(Table, UserIdFun) -> @@ -350,3 +436,67 @@ user_id_from_key(UserId) when is_integer(UserId) -> UserId; user_id_from_key(_) -> undefined. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +synced_blocked_ids_are_stored_without_an_expiry_test() -> + init(), + try + ok = put_blocked_ids(4001, [4002]), + ?assertEqual([{4001, [4002], infinity}], ets:lookup(?BLOCKED_IDS, 4001)) + after + safe_delete(?BLOCKED_IDS, 4001) + end. + +synced_blocked_ids_never_expire_test() -> + init(), + try + ok = put_blocked_ids(4002, [4003]), + ?assertEqual([4003], get_blocked_ids(4002)), + ok = evict_tables(#{}), + ?assertEqual([4003], get_blocked_ids(4002)) + after + safe_delete(?BLOCKED_IDS, 4002) + end. + +fetched_blocked_ids_expire_and_are_reclaimed_test() -> + init(), + try + ok = put_blocked_ids_fetched(4004, [4005]), + ?assertEqual([4005], get_blocked_ids(4004)), + Stale = erlang:system_time(second) - 1, + true = ets:insert(?BLOCKED_IDS, {4004, [4005], Stale}), + ?assertEqual(undefined, get_blocked_ids(4004)), + ok = evict_tables(#{}), + ?assertEqual([], ets:lookup(?BLOCKED_IDS, 4004)) + after + safe_delete(?BLOCKED_IDS, 4004) + end. + +legacy_untimed_blocked_ids_are_still_honoured_test() -> + init(), + try + true = ets:insert(?BLOCKED_IDS, {4008, [4009]}), + ?assertEqual([4009], get_blocked_ids(4008)), + ok = evict_tables(#{}), + ?assertEqual([4009], get_blocked_ids(4008)) + after + safe_delete(?BLOCKED_IDS, 4008) + end. + +blocked_ids_ttl_is_clamped_test() -> + ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, blocked_ids_ttl_seconds()), + ?assertEqual(?MIN_BLOCKED_IDS_TTL, with_ttl_env(1, fun blocked_ids_ttl_seconds/0)), + ?assertEqual(?MAX_BLOCKED_IDS_TTL, with_ttl_env(999999, fun blocked_ids_ttl_seconds/0)), + ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, with_ttl_env(0, fun blocked_ids_ttl_seconds/0)). + +with_ttl_env(Value, Fun) -> + application:set_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds, Value), + try + Fun() + after + application:unset_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds) + end. + +-endif. diff --git a/fluxer_gateway/src/push/push_fcm_payload.erl b/fluxer_gateway/src/push/push_fcm_payload.erl index 5296254f8..9796c03b0 100644 --- a/fluxer_gateway/src/push/push_fcm_payload.erl +++ b/fluxer_gateway/src/push/push_fcm_payload.erl @@ -40,9 +40,9 @@ build_clear_message(DeviceToken, Payload) -> <<"token">> => DeviceToken, <<"data">> => Data, <<"android">> => #{ - <<"priority">> => <<"HIGH">>, - <<"ttl">> => <<"300s">>, - <<"collapse_key">> => Tag + <<"priority">> => <<"NORMAL">>, + <<"ttl">> => <<"3600s">>, + <<"collapse_key">> => <<"clear:", Tag/binary>> }, <<"fcm_options">> => #{ <<"analytics_label">> => <<"notification_clear">> @@ -63,9 +63,8 @@ build_notification_message(DeviceToken, Payload) -> <<"image">>, ImageUrl, #{<<"title">> => Title, <<"body">> => Body} ), Data = build_notification_data(Payload, Title, Body, Tag, ImageUrl), - Group = resolve_notification_group(maps:get(<<"data">>, Payload, #{}), Tag), AndroidNotification = build_android_notification(Tag, ImageUrl), - wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group). + wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification). -spec build_android_notification(binary(), binary() | undefined) -> map(). build_android_notification(Tag, ImageUrl) -> @@ -75,8 +74,8 @@ build_android_notification(Tag, ImageUrl) -> <<"click_action">> => <<"FLUXER_MESSAGE">> }). --spec wrap_notification_message(binary(), map(), map(), map(), binary()) -> map(). -wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group) -> +-spec wrap_notification_message(binary(), map(), map(), map()) -> map(). +wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification) -> #{ <<"message">> => #{ <<"token">> => DeviceToken, @@ -85,7 +84,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati <<"android">> => #{ <<"priority">> => <<"HIGH">>, <<"ttl">> => <<"86400s">>, - <<"collapse_key">> => Group, <<"notification">> => AndroidNotification }, <<"fcm_options">> => #{ @@ -94,41 +92,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati } }. --spec resolve_notification_group(map(), binary()) -> binary(). -resolve_notification_group(Data, MessageTag) -> - case - push_utils:normalize_binary( - maps:get(<<"notification_tag">>, Data, undefined), undefined - ) - of - Group when is_binary(Group), byte_size(Group) > 0 -> - Group; - _ -> - resolve_channel_notification_group(Data, MessageTag) - end. - --spec resolve_channel_notification_group(map(), binary()) -> binary(). -resolve_channel_notification_group(Data, MessageTag) -> - case - push_utils:normalize_binary( - maps:get(<<"channel_id">>, Data, undefined), undefined - ) - of - ChannelId when is_binary(ChannelId), byte_size(ChannelId) > 0 -> - <<"channel:", ChannelId/binary>>; - _ -> - derive_channel_tag_from_message_tag(MessageTag) - end. - --spec derive_channel_tag_from_message_tag(binary()) -> binary(). -derive_channel_tag_from_message_tag(MessageTag) -> - case binary:split(MessageTag, <<":">>, [global]) of - [<<"channel">>, ChannelId, _MessageId] when byte_size(ChannelId) > 0 -> - <<"channel:", ChannelId/binary>>; - _ -> - MessageTag - end. - -spec build_notification_data(map(), binary(), binary(), binary(), binary() | undefined) -> map(). build_notification_data(Payload, Title, Body, Tag, ImageUrl) -> @@ -349,7 +312,7 @@ build_message_includes_android_chat_notification_fields_test() -> ?assertEqual(<<"fluxer_default_push">>, maps:get(<<"channel_id">>, AndroidNotification)), ?assertEqual(<<"channel:123:456">>, maps:get(<<"tag">>, AndroidNotification)), Android = maps:get(<<"android">>, Message), - ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)), + ?assertEqual(false, maps:is_key(<<"collapse_key">>, Android)), ?assertEqual( <<"https://cdn.example/image.png">>, maps:get(<<"image">>, AndroidNotification) ), @@ -370,9 +333,9 @@ build_clear_message_is_data_only_and_collapsible_test() -> #{<<"message">> := Message} = build_message(<<"device-token">>, Payload), ?assertEqual(false, maps:is_key(<<"notification">>, Message)), Android = maps:get(<<"android">>, Message), - ?assertEqual(<<"HIGH">>, maps:get(<<"priority">>, Android)), - ?assertEqual(<<"300s">>, maps:get(<<"ttl">>, Android)), - ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)), + ?assertEqual(<<"NORMAL">>, maps:get(<<"priority">>, Android)), + ?assertEqual(<<"3600s">>, maps:get(<<"ttl">>, Android)), + ?assertEqual(<<"clear:channel:123">>, maps:get(<<"collapse_key">>, Android)), Data = maps:get(<<"data">>, Message), ?assertEqual(<<"notification_clear">>, maps:get(<<"type">>, Data)), ?assertEqual(<<"clear_channel">>, maps:get(<<"action">>, Data)), diff --git a/fluxer_gateway/src/push/push_message_params.erl b/fluxer_gateway/src/push/push_message_params.erl index a328dcad7..846b40ee3 100644 --- a/fluxer_gateway/src/push/push_message_params.erl +++ b/fluxer_gateway/src/push/push_message_params.erl @@ -7,6 +7,8 @@ -export_type([context/0]). +-define(MAX_GUILD_FEATURES, 64). + -type context() :: #{ message_data := map(), user_ids := [pos_integer()], @@ -20,7 +22,8 @@ role_names := map(), user_roles := map(), connected_users := map(), - markdown_context := map() + markdown_context := map(), + large_guild_metadata := map() | undefined }. -spec context(map()) -> {ok, context()} | {error, term()}. @@ -58,9 +61,34 @@ context_from_message_data(Params, MessageData) -> connected_users => maps:get(connected_users, Params, #{}), markdown_context => markdown_context( MessageData, GuildId, RoleNames, maps:get(markdown_context, Params, undefined) - ) + ), + large_guild_metadata => large_guild_metadata(Params) }). +-spec large_guild_metadata(map()) -> map() | undefined. +large_guild_metadata(Params) -> + MemberCount = maps:get(guild_member_count, Params, undefined), + Features = maps:get(guild_features, Params, undefined), + build_large_guild_metadata(MemberCount, Features). + +-spec build_large_guild_metadata(term(), term()) -> map() | undefined. +build_large_guild_metadata(MemberCount, Features) when + is_integer(MemberCount), MemberCount >= 0, is_list(Features) +-> + #{member_count => MemberCount, features => bounded_features(Features, [], 0)}; +build_large_guild_metadata(_MemberCount, _Features) -> + undefined. + +-spec bounded_features(term(), [binary()], non_neg_integer()) -> [binary()]. +bounded_features(_Features, Acc, ?MAX_GUILD_FEATURES) -> + lists:reverse(Acc); +bounded_features([Feature | Rest], Acc, Count) when is_binary(Feature) -> + bounded_features(Rest, [Feature | Acc], Count + 1); +bounded_features([_Feature | Rest], Acc, Count) -> + bounded_features(Rest, Acc, Count); +bounded_features(_Features, Acc, _Count) -> + lists:reverse(Acc). + -spec owner_key(map()) -> term(). owner_key(Params) -> case user_ids(maps:get(user_ids, Params, [])) of diff --git a/fluxer_gateway/src/push/push_notification.erl b/fluxer_gateway/src/push/push_notification.erl index 2bd103511..71fa239ec 100644 --- a/fluxer_gateway/src/push/push_notification.erl +++ b/fluxer_gateway/src/push/push_notification.erl @@ -133,7 +133,7 @@ assemble_payload( ContentPreview, AuthorAvatarUrl ) -> - Data = build_data(Ctx), + Data = build_data(Ctx, AuthorAvatarUrl), Notification = build_notification_body(Ctx, Title, ContentPreview, AuthorAvatarUrl, Data), maps:merge( #{ @@ -149,19 +149,23 @@ assemble_payload( ImageFields ). --spec build_data(push_ctx()) -> map(). -build_data(#{ - channel_id := ChannelId, - message_id := MessageId, - guild_id := GuildId, - navigate_url := NavigateUrl, - badge_value := BadgeValue, - target_user_id := TargetUserId, - image_url := ImageUrl, - image_fields := ImageFields -}) -> +-spec build_data(push_ctx(), binary()) -> map(). +build_data( + #{ + channel_id := ChannelId, + message_id := MessageId, + guild_id := GuildId, + navigate_url := NavigateUrl, + badge_value := BadgeValue, + target_user_id := TargetUserId, + image_url := ImageUrl, + image_fields := ImageFields + }, + AuthorAvatarUrl +) -> BaseData = #{ <<"channel_id">> => integer_to_binary(ChannelId), + <<"author_avatar_url">> => AuthorAvatarUrl, <<"message_id">> => integer_to_binary(MessageId), <<"notification_tag">> => build_channel_tag(ChannelId), <<"guild_id">> => diff --git a/fluxer_gateway/src/push/push_notification_format.erl b/fluxer_gateway/src/push/push_notification_format.erl index 1fbcb8b3d..a9d2c6097 100644 --- a/fluxer_gateway/src/push/push_notification_format.erl +++ b/fluxer_gateway/src/push/push_notification_format.erl @@ -326,9 +326,18 @@ first_nonempty_binary([Value | Rest]) -> -spec truncate_preview(binary()) -> binary(). truncate_preview(Content) when byte_size(Content) > 100 -> - binary:part(Content, 0, 100); + valid_utf8_prefix(binary:part(Content, 0, 100)); truncate_preview(Content) -> - Content. + valid_utf8_prefix(Content). + +-spec valid_utf8_prefix(binary()) -> binary(). +valid_utf8_prefix(Content) -> + case unicode:characters_to_binary(Content, utf8, utf8) of + Valid when is_binary(Valid) -> Valid; + {incomplete, Valid, _Rest} when is_binary(Valid) -> Valid; + {error, Valid, _Rest} when is_binary(Valid) -> Valid; + _ -> <<>> + end. -spec build_content_fallback_preview(map()) -> binary(). build_content_fallback_preview(MessageData) -> @@ -568,6 +577,23 @@ lowercase_binary(Value) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +truncate_preview_never_splits_a_utf8_character_test() -> + Emoji = <<"\xF0\x9F\x98\x80">>, + Content = <<(binary:copy(<<"a">>, 98))/binary, Emoji/binary>>, + ?assert(byte_size(Content) > 100), + Truncated = truncate_preview(Content), + ?assert(byte_size(Truncated) =< 100), + ?assertMatch(Bin when is_binary(Bin), unicode:characters_to_binary(Truncated, utf8, utf8)), + ?assertEqual(binary:copy(<<"a">>, 98), Truncated). + +truncate_preview_is_unchanged_for_valid_ascii_test() -> + Content = binary:copy(<<"a">>, 150), + ?assertEqual(binary:copy(<<"a">>, 100), truncate_preview(Content)). + +truncate_preview_keeps_short_valid_content_identical_test() -> + Content = <<"hello \xC3\xA9 world">>, + ?assertEqual(Content, truncate_preview(Content)). + build_url_dm_test() -> ?assertEqual(<<"/channels/@me/456/789">>, build_url(0, 456, 789)). diff --git a/fluxer_gateway/src/push/push_sender.erl b/fluxer_gateway/src/push/push_sender.erl index 3726a801a..7d7c05bcd 100644 --- a/fluxer_gateway/src/push/push_sender.erl +++ b/fluxer_gateway/src/push/push_sender.erl @@ -13,6 +13,11 @@ -export_type([send_context/0]). +-define(DEFAULT_BADGE_FETCH_BATCH, 2000). +-define(BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, 3). +-define(DEFAULT_BADGE_FETCH_BUDGET_MS, 120000). +-define(PUSH_COUNTERS, push_worker_counter). + -type send_context() :: #{ message_data := map(), guild_id := integer(), @@ -257,16 +262,128 @@ check_badge_cache(UserId, TTL, Now, Acc, MissingAcc) -> -spec fetch_badge_counts([integer()], map(), integer()) -> map(). fetch_badge_counts(UserIds, Counts, CachedAt) -> + Batches = chunk_badge_user_ids(UserIds, badge_fetch_batch_size(), []), + {Counted, FailedBatches, DefaultedUsers, _Consecutive} = + fetch_badge_count_batches(Batches, {Counts, 0, 0, 0}, CachedAt), + report_badge_fetch_failures(FailedBatches, DefaultedUsers), + Counted. + +-type badge_batch_acc() :: {map(), non_neg_integer(), non_neg_integer(), non_neg_integer()}. + +-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer()) -> + badge_batch_acc(). +fetch_badge_count_batches(Batches, Acc, CachedAt) -> + Deadline = erlang:monotonic_time(millisecond) + badge_fetch_budget_ms(), + fetch_badge_count_batches(Batches, Acc, CachedAt, Deadline). + +-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer(), integer()) -> + badge_batch_acc(). +fetch_badge_count_batches([], Acc, _CachedAt, _Deadline) -> + Acc; +fetch_badge_count_batches(Remaining, Acc, CachedAt, Deadline) -> + case badge_fetch_exhausted(Acc, Deadline) of + true -> abandon_badge_batches(Remaining, Acc); + false -> fetch_next_badge_batch(Remaining, Acc, CachedAt, Deadline) + end. + +-spec badge_fetch_exhausted(badge_batch_acc(), integer()) -> boolean(). +badge_fetch_exhausted({_Counts, _FailedBatches, _DefaultedUsers, Consecutive}, Deadline) -> + Consecutive >= ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES orelse + erlang:monotonic_time(millisecond) >= Deadline. + +-spec abandon_badge_batches([[integer()]], badge_batch_acc()) -> badge_batch_acc(). +abandon_badge_batches(Remaining, {Counts, FailedBatches, DefaultedUsers, Consecutive}) -> + {Counts, FailedBatches + length(Remaining), + DefaultedUsers + badge_batched_user_count(Remaining, 0), Consecutive}. + +-spec badge_batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer(). +badge_batched_user_count([], Acc) -> + Acc; +badge_batched_user_count([Batch | Rest], Acc) -> + badge_batched_user_count(Rest, Acc + length(Batch)). + +-spec fetch_next_badge_batch([[integer()]], badge_batch_acc(), integer(), integer()) -> + badge_batch_acc(). +fetch_next_badge_batch( + [Batch | Rest], {Counts, FailedBatches, DefaultedUsers, Consecutive}, CachedAt, Deadline +) -> Request = #{ <<"type">> => <<"get_badge_counts">>, - <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds] + <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch] }, case rpc_client:call(Request) of {ok, Data} -> BadgeData = maps:get(<<"badge_counts">>, Data, #{}), - merge_badge_data(UserIds, BadgeData, Counts, CachedAt); + Merged = merge_badge_data(Batch, BadgeData, Counts, CachedAt), + fetch_badge_count_batches( + Rest, {Merged, FailedBatches, DefaultedUsers, 0}, CachedAt, Deadline + ); {error, _Reason} -> - Counts + fetch_badge_count_batches( + Rest, + {Counts, FailedBatches + 1, DefaultedUsers + length(Batch), Consecutive + 1}, + CachedAt, + Deadline + ) + end. + +-spec report_badge_fetch_failures(non_neg_integer(), non_neg_integer()) -> ok. +report_badge_fetch_failures(0, _DefaultedUsers) -> + ok; +report_badge_fetch_failures(FailedBatches, DefaultedUsers) -> + bump_counter(badge_fetch_calls_failed, FailedBatches), + bump_counter(badge_fetch_users_defaulted, DefaultedUsers), + logger:warning("Push: badge count batches failed; users default to zero badge", #{ + failed_batches => FailedBatches, users_defaulted => DefaultedUsers + }). + +-spec bump_counter(atom(), integer()) -> ok. +bump_counter(Key, Delta) -> + try + _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}), + ok + catch + error:badarg -> seed_and_bump_counter(Key, Delta) + end. + +-spec seed_and_bump_counter(atom(), integer()) -> ok. +seed_and_bump_counter(Key, Delta) -> + try + _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}), + _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}), + ok + catch + error:badarg -> ok + end. + +-spec chunk_badge_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]]. +chunk_badge_user_ids([], _BatchSize, Acc) -> + lists:reverse(Acc); +chunk_badge_user_ids(UserIds, BatchSize, Acc) -> + {Batch, Rest} = take_badge_user_id_batch(UserIds, BatchSize, []), + chunk_badge_user_ids(Rest, BatchSize, [Batch | Acc]). + +-spec take_badge_user_id_batch([integer()], non_neg_integer(), [integer()]) -> + {[integer()], [integer()]}. +take_badge_user_id_batch(Rest, 0, Acc) -> + {lists:reverse(Acc), Rest}; +take_badge_user_id_batch([], _Remaining, Acc) -> + {lists:reverse(Acc), []}; +take_badge_user_id_batch([UserId | Rest], Remaining, Acc) -> + take_badge_user_id_batch(Rest, Remaining - 1, [UserId | Acc]). + +-spec badge_fetch_batch_size() -> pos_integer(). +badge_fetch_batch_size() -> + case application:get_env(fluxer_gateway, push_badge_fetch_batch_size, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_BADGE_FETCH_BATCH + end. + +-spec badge_fetch_budget_ms() -> pos_integer(). +badge_fetch_budget_ms() -> + case application:get_env(fluxer_gateway, push_badge_fetch_budget_ms, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_BADGE_FETCH_BUDGET_MS end. -spec merge_badge_data([integer()], map(), map(), integer()) -> map(). @@ -285,3 +402,119 @@ merge_badge_data(UserIds, BadgeData, Counts, CachedAt) -> -spec normalize_badge_count(integer() | term()) -> non_neg_integer(). normalize_badge_count(Value) when is_integer(Value), Value >= 0 -> Value; normalize_badge_count(_) -> 0. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +chunk_badge_user_ids_uses_bounded_batches_test() -> + ?assertEqual([[1, 2], [3, 4], [5]], chunk_badge_user_ids([1, 2, 3, 4, 5], 2, [])), + ?assertEqual([], chunk_badge_user_ids([], 2, [])). + +fetch_badge_counts_in_batches_keeps_successful_batches_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + ok = meck:new(push_ets_cache, [passthrough, no_link]), + application:set_env(fluxer_gateway, push_badge_fetch_batch_size, 2), + try + ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end), + ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) -> + case Ids of + [<<"1">>, <<"2">>] -> + {ok, #{<<"badge_counts">> => #{<<"1">> => 3, <<"2">> => 4}}}; + _ -> + {error, unavailable} + end + end), + ?assertEqual(#{1 => 3, 2 => 4}, fetch_badge_counts([1, 2, 3, 4], #{}, 0)) + after + application:unset_env(fluxer_gateway, push_badge_fetch_batch_size), + meck:unload(push_ets_cache), + meck:unload(rpc_client) + end. + +fetch_badge_count_batches_stops_after_consecutive_failures_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + try + ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end), + Batches = [[N] || N <- lists:seq(1, 10)], + ?assertEqual( + {#{}, 10, 10, ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES}, + fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0) + ), + ?assertEqual(?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, length(meck:history(rpc_client))) + after + meck:unload(rpc_client) + end. + +fetch_badge_count_batches_stops_at_the_wall_clock_budget_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + application:set_env(fluxer_gateway, push_badge_fetch_budget_ms, 1), + try + ok = meck:expect(rpc_client, call, fun(_Req) -> + {ok, #{<<"badge_counts">> => #{}}} + end), + Batches = [[N] || N <- lists:seq(1, 10)], + Deadline = erlang:monotonic_time(millisecond) - 1, + ?assertEqual( + {#{}, 10, 10, 0}, + fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0, Deadline) + ), + ?assertEqual(0, length(meck:history(rpc_client))) + after + application:unset_env(fluxer_gateway, push_badge_fetch_budget_ms), + meck:unload(rpc_client) + end. + +badge_fetch_defaults_missing_users_to_zero_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + ok = meck:new(push_ets_cache, [passthrough, no_link]), + try + ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end), + ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) -> + ?assertEqual([<<"1">>, <<"2">>, <<"3">>], Ids), + {ok, #{<<"badge_counts">> => #{<<"1">> => 1}}} + end), + ?assertEqual(#{1 => 1, 2 => 0, 3 => 0}, fetch_badge_counts([1, 2, 3], #{}, 0)) + after + meck:unload(push_ets_cache), + meck:unload(rpc_client) + end. + +badge_fetch_failure_counts_defaulted_users_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(badge_fetch_calls_failed), + ok = reset_test_counter(badge_fetch_users_defaulted), + try + ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end), + ?assertEqual(#{}, fetch_badge_counts([1, 2, 3], #{}, 0)), + ?assertEqual(1, counter_value(badge_fetch_calls_failed)), + ?assertEqual(3, counter_value(badge_fetch_users_defaulted)) + after + meck:unload(rpc_client) + end. + +counter_value(Key) -> + try ets:lookup(?PUSH_COUNTERS, Key) of + [{Key, Value}] when is_integer(Value) -> Value; + _ -> 0 + catch + error:badarg -> 0 + end. + +ensure_test_counter_table() -> + case ets:info(?PUSH_COUNTERS, name) of + undefined -> + _ = ets:new(?PUSH_COUNTERS, [named_table, public, set, {write_concurrency, true}]), + ok; + _ -> + ok + end. + +reset_test_counter(Key) -> + try ets:insert(?PUSH_COUNTERS, {Key, 0}) of + _ -> ok + catch + error:badarg -> ok + end. + +-endif. diff --git a/fluxer_gateway/src/push/push_sender_delivery.erl b/fluxer_gateway/src/push/push_sender_delivery.erl index 73ab037f6..e0f7d52b3 100644 --- a/fluxer_gateway/src/push/push_sender_delivery.erl +++ b/fluxer_gateway/src/push/push_sender_delivery.erl @@ -561,7 +561,11 @@ maybe_retry_transient_waits_before_retry_test() -> fun retry_request_meck/6 ), ?assertEqual(false, maybe_retry_transient(Ctx, RecordSize, 0, {error, timeout})), - ?assertEqual({retry_wait, ?BASE_RETRY_DELAY_MS}, receive_retry_wait()), + %% retry_delay/1 adds rand:uniform(Base div 4) of jitter, so the first + %% retry is anywhere in [Base, Base + Base div 4 - 1]. + {retry_wait, DelayMs} = receive_retry_wait(), + ?assert(DelayMs >= ?BASE_RETRY_DELAY_MS), + ?assert(DelayMs =< ?BASE_RETRY_DELAY_MS + (?BASE_RETRY_DELAY_MS div 4) - 1), ?assertEqual(ok, receive_retried_push_request(Endpoint)), ?assert(meck:validate(gateway_retry_timer)), ?assert(meck:validate(push_utils)), diff --git a/fluxer_gateway/src/push/push_subscriptions.erl b/fluxer_gateway/src/push/push_subscriptions.erl index 5e3193651..c18e168d5 100644 --- a/fluxer_gateway/src/push/push_subscriptions.erl +++ b/fluxer_gateway/src/push/push_subscriptions.erl @@ -11,10 +11,18 @@ -export([fetch_and_cache_user_guild_settings/2]). -export([delete_failed_subscriptions/1]). -export([delivery_concurrency/0]). +-export([delivery_stats/0]). -define(DEFAULT_DELIVERY_CONCURRENCY, 8). --define(DELIVERY_WORKER_TIMEOUT_MS, 30000). --define(SUBSCRIPTION_FETCH_CHUNK_SIZE, 256). +-define(DEFAULT_DELIVERY_MAX_WORKERS, 64). +-define(DEFAULT_DELIVERY_TASKS_PER_WORKER, 128). +-define(DEFAULT_SUBSCRIPTION_FETCH_BATCH, 2000). +-define(SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES, 3). +-define(DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS, 120000). +-define(DELIVERY_IDLE_TIMEOUT_MS, 30000). +-define(PUSH_COUNTERS, push_worker_counter). +-define(CNT_LIVE_WORKERS, delivery_live_workers). +-define(CTX_BADGE_COUNTS, 9). -spec fetch_and_send_subscriptions( [integer()], @@ -92,60 +100,118 @@ send_missing_if_any(MissingUserIds, Ctx) -> -spec fetch_and_send_missing_subscriptions([integer()], tuple()) -> ok. fetch_and_send_missing_subscriptions(UserIds, Ctx) -> - {FetchedUserIds, SubscriptionsData} = fetch_missing_subscriptions(UserIds), - send_fetched_subscriptions(FetchedUserIds, SubscriptionsData, Ctx). + Batches = chunk_user_ids(UserIds, subscription_fetch_batch_size(), []), + logger:debug("Push: fetching subscriptions via batched RPC", #{ + user_count => length(UserIds), batch_count => length(Batches) + }), + {Tasks, FailedBatches, FailedUsers, _Consecutive} = + fetch_subscription_batches(Batches, {[], 0, 0, 0}), + report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx), + deliver_subscription_tasks(lists:reverse(Tasks), Ctx). --spec fetch_missing_subscriptions([integer()]) -> {[integer()], map()}. -fetch_missing_subscriptions(UserIds) -> - Chunks = chunk_user_ids(UserIds, ?SUBSCRIPTION_FETCH_CHUNK_SIZE, []), - {FetchedChunks, SubscriptionsData} = lists:foldl( - fun fetch_subscriptions_chunk/2, - {[], #{}}, - Chunks - ), - {lists:append(lists:reverse(FetchedChunks)), SubscriptionsData}. +-type subscription_batch_acc() :: + {[{integer(), list()}], non_neg_integer(), non_neg_integer(), non_neg_integer()}. --spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]]. -chunk_user_ids([], _ChunkSize, Acc) -> - lists:reverse(Acc); -chunk_user_ids(UserIds, ChunkSize, Acc) when length(UserIds) =< ChunkSize -> - lists:reverse([UserIds | Acc]); -chunk_user_ids(UserIds, ChunkSize, Acc) -> - {Chunk, Rest} = lists:split(ChunkSize, UserIds), - chunk_user_ids(Rest, ChunkSize, [Chunk | Acc]). +-spec fetch_subscription_batches([[integer()]], subscription_batch_acc()) -> + subscription_batch_acc(). +fetch_subscription_batches(Batches, Acc) -> + Deadline = erlang:monotonic_time(millisecond) + subscription_fetch_budget_ms(), + fetch_subscription_batches(Batches, Acc, Deadline). --spec fetch_subscriptions_chunk([integer()], {[[integer()]], map()}) -> {[[integer()]], map()}. -fetch_subscriptions_chunk(UserIds, {FetchedAcc, DataAcc}) -> - SubscriptionsReq = #{ +-spec fetch_subscription_batches([[integer()]], subscription_batch_acc(), integer()) -> + subscription_batch_acc(). +fetch_subscription_batches([], Acc, _Deadline) -> + Acc; +fetch_subscription_batches(Remaining, Acc, Deadline) -> + case subscription_fetch_exhausted(Acc, Deadline) of + true -> abandon_subscription_batches(Remaining, Acc); + false -> fetch_next_subscription_batch(Remaining, Acc, Deadline) + end. + +-spec subscription_fetch_exhausted(subscription_batch_acc(), integer()) -> boolean(). +subscription_fetch_exhausted({_Tasks, _FailedBatches, _FailedUsers, Consecutive}, Deadline) -> + Consecutive >= ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES orelse + erlang:monotonic_time(millisecond) >= Deadline. + +-spec abandon_subscription_batches([[integer()]], subscription_batch_acc()) -> + subscription_batch_acc(). +abandon_subscription_batches(Remaining, {Tasks, FailedBatches, FailedUsers, Consecutive}) -> + {Tasks, FailedBatches + length(Remaining), FailedUsers + batched_user_count(Remaining, 0), + Consecutive}. + +-spec fetch_next_subscription_batch([[integer()]], subscription_batch_acc(), integer()) -> + subscription_batch_acc(). +fetch_next_subscription_batch( + [Batch | Rest], {Tasks, FailedBatches, FailedUsers, Consecutive}, Deadline +) -> + Req = #{ <<"type">> => <<"get_push_subscriptions">>, - <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds] + <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch] }, - logger:debug("Push: fetching subscriptions via RPC", #{user_count => length(UserIds)}), - case rpc_client:call(SubscriptionsReq) of - {ok, SubscriptionsData} -> - logger:debug("Push: RPC returned subscriptions", #{ - user_count => length(UserIds), - response_keys => maps:keys(SubscriptionsData) - }), - {[UserIds | FetchedAcc], maps:merge(DataAcc, SubscriptionsData)}; + case rpc_client:call(Req) of + {ok, BatchData} -> + BatchTasks = lists:foldl( + fun(UserId, Acc) -> + add_fetched_user_subscription_task(UserId, BatchData, Acc) + end, + Tasks, + Batch + ), + fetch_subscription_batches( + Rest, {BatchTasks, FailedBatches, FailedUsers, 0}, Deadline + ); {error, Reason} -> logger:debug( "Push: RPC failed to fetch subscriptions", - #{user_count => length(UserIds), reason => Reason} + #{user_count => length(Batch), reason => Reason} ), - {FetchedAcc, DataAcc} + fetch_subscription_batches( + Rest, + {Tasks, FailedBatches + 1, FailedUsers + length(Batch), Consecutive + 1}, + Deadline + ) end. --spec send_fetched_subscriptions([integer()], map(), tuple()) -> ok. -send_fetched_subscriptions(UserIds, SubscriptionsData, Ctx) -> - Tasks = lists:foldl( - fun(UserId, Acc) -> - add_fetched_user_subscription_task(UserId, SubscriptionsData, Acc) - end, - [], - UserIds - ), - send_subscription_tasks(lists:reverse(Tasks), Ctx). +-spec batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer(). +batched_user_count([], Acc) -> + Acc; +batched_user_count([Batch | Rest], Acc) -> + batched_user_count(Rest, Acc + length(Batch)). + +-spec report_subscription_fetch_failures(non_neg_integer(), non_neg_integer(), tuple()) -> ok. +report_subscription_fetch_failures(0, _FailedUsers, _Ctx) -> + ok; +report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx) -> + bump_counter(subscription_fetch_calls_failed, FailedBatches), + bump_counter(subscription_fetch_users_dropped, FailedUsers), + {GuildId, MessageId} = delivery_ids(Ctx), + logger:warning("Push: subscription fetch batches failed; recipients not attempted", #{ + guild_id => GuildId, + message_id => MessageId, + failed_batches => FailedBatches, + recipients_not_attempted => FailedUsers + }). + +-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]]. +chunk_user_ids([], _BatchSize, Acc) -> + lists:reverse(Acc); +chunk_user_ids(UserIds, BatchSize, Acc) -> + {Batch, Rest} = take_user_id_batch(UserIds, BatchSize, []), + chunk_user_ids(Rest, BatchSize, [Batch | Acc]). + +-spec take_user_id_batch([integer()], non_neg_integer(), [integer()]) -> + {[integer()], [integer()]}. +take_user_id_batch(Rest, 0, Acc) -> + {lists:reverse(Acc), Rest}; +take_user_id_batch([], _Remaining, Acc) -> + {lists:reverse(Acc), []}; +take_user_id_batch([UserId | Rest], Remaining, Acc) -> + take_user_id_batch(Rest, Remaining - 1, [UserId | Acc]). + +-spec count_subscription_fetch_loss(non_neg_integer()) -> ok. +count_subscription_fetch_loss(UserCount) -> + bump_counter(subscription_fetch_calls_failed, 1), + bump_counter(subscription_fetch_users_dropped, UserCount). -spec add_fetched_user_subscription_task(integer(), map(), [{integer(), list()}]) -> [{integer(), list()}]. @@ -205,6 +271,7 @@ send_clear_rpc_result(UserId, ChannelId, MessageId, BadgeCount, {ok, Subscriptio UserId, ChannelId, MessageId, BadgeCount, maps:get(UserIdBin, SubscriptionsData, []) ); send_clear_rpc_result(UserId, _ChannelId, _MessageId, _BadgeCount, {error, Reason}) -> + count_subscription_fetch_loss(1), logger:debug( "Push: RPC failed to fetch subscriptions for notification clear", #{user_id => UserId, reason => Reason} @@ -238,7 +305,7 @@ send_cached_subscriptions(SubscriptionsByUser, Ctx) -> [], SubscriptionsByUser ), - send_subscription_tasks(Tasks, Ctx). + deliver_subscription_tasks(Tasks, Ctx). -spec add_cached_user_task(integer(), list(), [{integer(), list()}]) -> [{integer(), list()}]. @@ -250,21 +317,335 @@ add_cached_user_task(UserId, Subscriptions, Acc) -> [{UserId, Subscriptions} | Acc] end. --spec send_subscription_tasks([{integer(), list()}], tuple()) -> ok. -send_subscription_tasks(Tasks, Ctx) -> - send_subscription_tasks(Tasks, Ctx, delivery_concurrency()). +-spec deliver_subscription_tasks([{integer(), list()}], tuple()) -> ok. +deliver_subscription_tasks(Tasks, Ctx) -> + send_bounded_subscription_tasks(Tasks, Ctx). --spec send_subscription_tasks([{integer(), list()}], tuple(), pos_integer()) -> ok. -send_subscription_tasks([], _Ctx, _MaxConcurrency) -> +-spec send_bounded_subscription_tasks([{integer(), list()}], tuple()) -> ok. +send_bounded_subscription_tasks([], _Ctx) -> ok; -send_subscription_tasks(Tasks, Ctx, 1) -> - send_subscription_task_chunk(Tasks, Ctx); -send_subscription_tasks(Tasks, Ctx, MaxConcurrency) -> +send_bounded_subscription_tasks(Tasks, Ctx) -> TaskCount = length(Tasks), - WorkerCount = min(TaskCount, MaxConcurrency), + Desired = bounded_worker_count(TaskCount, delivery_concurrency(), delivery_max_workers()), + Granted = claim_delivery_slots(Desired, gateway_http_client:push_max_concurrency()), + try + run_claimed_delivery(Tasks, TaskCount, Ctx, Granted) + after + release_delivery_slots(Granted) + end. + +-spec run_claimed_delivery([{integer(), list()}], pos_integer(), tuple(), non_neg_integer()) -> + ok. +run_claimed_delivery(Tasks, _TaskCount, Ctx, 0) -> + bump_counter(delivery_budget_exhausted, 1), + run_inline_delivery(Tasks, Ctx); +run_claimed_delivery(Tasks, TaskCount, Ctx, WorkerCount) -> ChunkSize = max(1, (TaskCount + WorkerCount - 1) div WorkerCount), Chunks = chunk_subscription_tasks(Tasks, ChunkSize, []), - wait_delivery_workers(start_delivery_workers(Chunks, Ctx, #{})). + Workers = start_progress_workers( + Chunks, shared_ctx_payload(Ctx), ctx_badge_counts(Ctx), self(), #{} + ), + await_progress_workers(Workers, 0, TaskCount, delivery_ids(Ctx)). + +-spec run_inline_delivery([{integer(), list()}], tuple()) -> ok. +run_inline_delivery([], _Ctx) -> + ok; +run_inline_delivery([Task | Rest], Ctx) -> + run_subscription_task(Task, Ctx), + run_inline_delivery(Rest, Ctx). + +-spec claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer(). +claim_delivery_slots(Desired, Budget) -> + try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of + Reserved -> settle_delivery_claim(Desired, Budget, Reserved) + catch + error:badarg -> claim_first_delivery_slots(Desired, Budget) + end. + +-spec claim_first_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer(). +claim_first_delivery_slots(Desired, Budget) -> + try ets:insert_new(?PUSH_COUNTERS, {?CNT_LIVE_WORKERS, 0}) of + _Inserted -> retry_claim_delivery_slots(Desired, Budget) + catch + error:badarg -> 0 + end. + +-spec retry_claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer(). +retry_claim_delivery_slots(Desired, Budget) -> + try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of + Reserved -> settle_delivery_claim(Desired, Budget, Reserved) + catch + error:badarg -> 0 + end. + +-spec settle_delivery_claim(pos_integer(), pos_integer(), integer()) -> non_neg_integer(). +settle_delivery_claim(Desired, Budget, Reserved) -> + Granted = min(Desired, max(0, Budget - (Reserved - Desired))), + release_delivery_slots(Desired - Granted), + Granted. + +-spec release_delivery_slots(non_neg_integer()) -> ok. +release_delivery_slots(0) -> + ok; +release_delivery_slots(Count) -> + try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, -Count, 0, 0}) of + _Value -> ok + catch + error:badarg -> ok + end. + +-spec bounded_worker_count(pos_integer(), pos_integer(), pos_integer()) -> pos_integer(). +bounded_worker_count(TaskCount, Base, Bound) -> + TasksPerWorker = delivery_tasks_per_worker(), + Sized = (TaskCount + TasksPerWorker - 1) div TasksPerWorker, + min(TaskCount, min(Bound, max(Base, Sized))). + +-type delivery_workers() :: #{pid() => {reference(), non_neg_integer()}}. + +-spec start_progress_workers( + [[{integer(), list()}]], binary(), map(), pid(), delivery_workers() +) -> delivery_workers(). +start_progress_workers([], _Shared, _BadgeCounts, _Coordinator, Workers) -> + Workers; +start_progress_workers([Chunk | Rest], Shared, BadgeCounts, Coordinator, Workers) -> + ChunkSize = length(Chunk), + ChunkBadges = chunk_badge_counts(Chunk, BadgeCounts), + bump_counter(delivery_workers_spawned, 1), + {Pid, Ref} = spawn_monitor(fun() -> + run_progress_chunk(Chunk, restore_chunk_ctx(Shared, ChunkBadges), Coordinator) + end), + start_progress_workers( + Rest, Shared, BadgeCounts, Coordinator, Workers#{Pid => {Ref, ChunkSize}} + ). + +-spec shared_ctx_payload(tuple()) -> binary(). +shared_ctx_payload(Ctx) -> + term_to_binary(setelement(?CTX_BADGE_COUNTS, Ctx, #{})). + +-spec restore_chunk_ctx(binary(), map()) -> tuple(). +restore_chunk_ctx(Shared, BadgeCounts) -> + case binary_to_term(Shared) of + Ctx when is_tuple(Ctx) -> setelement(?CTX_BADGE_COUNTS, Ctx, BadgeCounts) + end. + +-spec chunk_badge_counts([{integer(), list()}], map()) -> map(). +chunk_badge_counts(Chunk, BadgeCounts) -> + maps:with([UserId || {UserId, _Subscriptions} <- Chunk], BadgeCounts). + +-spec ctx_badge_counts(tuple()) -> map(). +ctx_badge_counts(Ctx) -> + { + _MsgData, + _GuildId, + _ChannelId, + _MessageId, + _GuildName, + _ChannelName, + _MarkdownContext, + _ContentPreview, + BadgeCounts + } = Ctx, + BadgeCounts. + +-spec run_progress_chunk([{integer(), list()}], tuple(), pid()) -> ok. +run_progress_chunk([], _Ctx, _Coordinator) -> + ok; +run_progress_chunk([Task | Rest], Ctx, Coordinator) -> + run_subscription_task(Task, Ctx), + Coordinator ! {push_delivery_progress, self()}, + run_progress_chunk(Rest, Ctx, Coordinator). + +-spec run_subscription_task({integer(), list()}, tuple()) -> ok. +run_subscription_task({UserId, _Subscriptions} = Task, Ctx) -> + try + send_subscription_task(Task, Ctx) + catch + Class:Reason -> + bump_counter(delivery_task_failures, 1), + logger:warning("Push: delivery task raised", #{ + user_id => UserId, class => Class, reason => Reason + }) + end, + ok. + +-spec await_progress_workers( + delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()} +) -> ok. +await_progress_workers(Workers, Unattempted, TaskCount, Ids) when map_size(Workers) =:= 0 -> + report_unattempted_recipients(Unattempted, TaskCount, Ids); +await_progress_workers(Workers, Unattempted, TaskCount, Ids) -> + receive + {push_delivery_progress, Pid} -> + await_progress_workers(record_task_done(Pid, Workers), Unattempted, TaskCount, Ids); + {'DOWN', _Ref, process, Pid, normal} -> + await_progress_workers(maps:remove(Pid, Workers), Unattempted, TaskCount, Ids); + {'DOWN', _Ref, process, Pid, _Reason} -> + {Lost, Rest} = discard_crashed_worker(Pid, Workers), + await_progress_workers(Rest, Unattempted + Lost, TaskCount, Ids) + after ?DELIVERY_IDLE_TIMEOUT_MS -> + report_stalled_delivery(Workers, Unattempted, TaskCount, Ids) + end. + +-spec record_task_done(pid(), delivery_workers()) -> delivery_workers(). +record_task_done(Pid, Workers) -> + case Workers of + #{Pid := {Ref, Left}} -> Workers#{Pid := {Ref, max(0, Left - 1)}}; + _ -> Workers + end. + +-spec discard_crashed_worker(pid(), delivery_workers()) -> + {non_neg_integer(), delivery_workers()}. +discard_crashed_worker(Pid, Workers) -> + case maps:take(Pid, Workers) of + {{_Ref, Left}, Rest} -> + bump_counter(delivery_worker_crashes, 1), + bump_counter(delivery_recipients_not_attempted, Left), + {Left, Rest}; + error -> + {0, Workers} + end. + +-spec report_unattempted_recipients( + non_neg_integer(), non_neg_integer(), {integer(), integer()} +) -> ok. +report_unattempted_recipients(0, _TaskCount, _Ids) -> + ok; +report_unattempted_recipients(Unattempted, TaskCount, {GuildId, MessageId}) -> + logger:warning("Push: delivery worker died with recipients unattempted", #{ + guild_id => GuildId, + message_id => MessageId, + recipients_total => TaskCount, + recipients_not_attempted => Unattempted + }). + +-spec report_stalled_delivery( + delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()} +) -> ok. +report_stalled_delivery(Workers, Unattempted, TaskCount, {GuildId, MessageId}) -> + Abandoned = map_size(Workers), + Stranded = abandon_delivery_workers(maps:to_list(Workers), 0), + bump_counter(delivery_workers_abandoned, Abandoned), + bump_counter(delivery_recipients_unconfirmed, Stranded), + logger:error("Push: subscription delivery stalled with recipients unattempted", #{ + guild_id => GuildId, + message_id => MessageId, + recipients_total => TaskCount, + recipients_not_attempted => Unattempted, + recipients_unconfirmed => Stranded, + workers_still_running => Abandoned, + idle_timeout_ms => ?DELIVERY_IDLE_TIMEOUT_MS + }). + +-spec abandon_delivery_workers( + [{pid(), {reference(), non_neg_integer()}}], non_neg_integer() +) -> non_neg_integer(). +abandon_delivery_workers([], Stranded) -> + Stranded; +abandon_delivery_workers([{_Pid, {Ref, Left}} | Rest], Stranded) -> + _ = erlang:demonitor(Ref, [flush]), + abandon_delivery_workers(Rest, Stranded + Left). + +-spec delivery_ids(tuple()) -> {integer(), integer()}. +delivery_ids(Ctx) -> + { + _MsgData, + GuildId, + _ChannelId, + MessageId, + _GuildName, + _ChannelName, + _MarkdownContext, + _ContentPreview, + _BadgeCounts + } = Ctx, + {GuildId, MessageId}. + +-spec delivery_stats() -> #{atom() => term()}. +delivery_stats() -> + Counters = maps:from_list([{Key, counter_value(Key)} || Key <- delivery_counter_keys()]), + Counters#{delivery_last_crash => last_delivery_crash()}. + +-spec delivery_counter_keys() -> [atom()]. +delivery_counter_keys() -> + [ + ?CNT_LIVE_WORKERS, + delivery_workers_spawned, + delivery_worker_crashes, + delivery_workers_abandoned, + delivery_recipients_not_attempted, + delivery_recipients_unconfirmed, + delivery_task_failures, + delivery_budget_exhausted, + subscription_fetch_calls_failed, + subscription_fetch_users_dropped, + badge_fetch_calls_failed, + badge_fetch_users_defaulted + ]. + +-spec counter_value(atom()) -> non_neg_integer(). +counter_value(Key) -> + try ets:lookup(?PUSH_COUNTERS, Key) of + [{Key, Value}] when is_integer(Value), Value >= 0 -> Value; + _ -> 0 + catch + error:badarg -> 0 + end. + +-spec last_delivery_crash() -> term(). +last_delivery_crash() -> + try ets:lookup(?PUSH_COUNTERS, delivery_last_crash) of + [{delivery_last_crash, Summary}] -> Summary; + _ -> none + catch + error:badarg -> none + end. + +-spec bump_counter(atom(), integer()) -> ok. +bump_counter(Key, Delta) -> + try + _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}), + ok + catch + error:badarg -> seed_and_bump_counter(Key, Delta) + end. + +-spec seed_and_bump_counter(atom(), integer()) -> ok. +seed_and_bump_counter(Key, Delta) -> + try + _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}), + _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}), + ok + catch + error:badarg -> ok + end. + +-spec delivery_max_workers() -> pos_integer(). +delivery_max_workers() -> + case application:get_env(fluxer_gateway, push_delivery_max_workers, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_DELIVERY_MAX_WORKERS + end. + +-spec delivery_tasks_per_worker() -> pos_integer(). +delivery_tasks_per_worker() -> + case application:get_env(fluxer_gateway, push_delivery_tasks_per_worker, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_DELIVERY_TASKS_PER_WORKER + end. + +-spec subscription_fetch_batch_size() -> pos_integer(). +subscription_fetch_batch_size() -> + case application:get_env(fluxer_gateway, push_subscription_fetch_batch_size, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BATCH + end. + +-spec subscription_fetch_budget_ms() -> pos_integer(). +subscription_fetch_budget_ms() -> + case application:get_env(fluxer_gateway, push_subscription_fetch_budget_ms, undefined) of + Value when is_integer(Value), Value > 0 -> Value; + _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS + end. -spec delivery_concurrency() -> pos_integer(). delivery_concurrency() -> @@ -290,35 +671,6 @@ take_subscription_chunk([], _Remaining, Acc) -> take_subscription_chunk([Task | Rest], Remaining, Acc) -> take_subscription_chunk(Rest, Remaining - 1, [Task | Acc]). --spec start_delivery_workers([[{integer(), list()}]], tuple(), #{reference() => true}) -> - #{reference() => true}. -start_delivery_workers([], _Ctx, Refs) -> - Refs; -start_delivery_workers([Chunk | Rest], Ctx, Refs) -> - {_Pid, Ref} = spawn_monitor(fun() -> - send_subscription_task_chunk(Chunk, Ctx) - end), - start_delivery_workers(Rest, Ctx, Refs#{Ref => true}). - --spec wait_delivery_workers(#{reference() => true}) -> ok. -wait_delivery_workers(Refs) when map_size(Refs) =:= 0 -> - ok; -wait_delivery_workers(Refs) -> - receive - {'DOWN', Ref, process, _Pid, _Reason} -> - wait_delivery_workers(maps:remove(Ref, Refs)) - after ?DELIVERY_WORKER_TIMEOUT_MS -> - logger:warning("Push: timed out waiting for subscription delivery workers", #{ - remaining_workers => map_size(Refs) - }), - ok - end. - --spec send_subscription_task_chunk([{integer(), list()}], tuple()) -> ok. -send_subscription_task_chunk(Tasks, Ctx) -> - lists:foreach(fun(Task) -> send_subscription_task(Task, Ctx) end, Tasks), - ok. - -spec send_subscription_task({integer(), list()}, tuple()) -> ok. send_subscription_task({UserId, Subscriptions}, Ctx) -> push_sender:send_to_user_subscriptions( @@ -473,7 +825,7 @@ send_context(UserId, Ctx) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -send_subscription_tasks_sends_all_users_with_bounded_workers_test() -> +deliver_subscription_tasks_sends_every_user_its_badge_count_test() -> Self = self(), ok = meck:new(push_sender, [passthrough, no_link]), try @@ -486,7 +838,7 @@ send_subscription_tasks_sends_all_users_with_bounded_workers_test() -> ok end), Tasks = [{1, [sub1]}, {2, [sub2]}, {3, [sub3]}], - ?assertEqual(ok, send_subscription_tasks(Tasks, test_send_ctx(), 2)), + ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())), ?assertEqual( [{1, [sub1], 5}, {2, [sub2], 6}, {3, [sub3], 7}], lists:sort(collect_sent_subscription_tasks(3, [])) @@ -503,54 +855,290 @@ chunk_subscription_tasks_uses_bounded_chunk_size_test() -> chunk_subscription_tasks(Tasks, 2, []) ). -chunk_user_ids_uses_bounded_chunk_size_test() -> - ?assertEqual([], chunk_user_ids([], 2, [])), - ?assertEqual([[1, 2]], chunk_user_ids([1, 2], 2, [])), - ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])). +bounded_worker_count_scales_with_recipients_and_stays_bounded_test() -> + application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker), + ?assertEqual(3, bounded_worker_count(3, 8, 64)), + ?assertEqual(8, bounded_worker_count(100, 8, 64)), + ?assertEqual(20, bounded_worker_count(2500, 8, 64)), + ?assertEqual(64, bounded_worker_count(10000, 8, 64)), + ?assertEqual(8, bounded_worker_count(10000, 8, 8)). -add_fetched_user_subscription_task_caches_empty_result_test() -> - push_ets_cache:init(), - push_ets_cache:delete_subscriptions(4242), - ?assertEqual([], add_fetched_user_subscription_task(4242, #{}, [])), - ?assertEqual([], push_ets_cache:get_subscriptions(4242)), - push_ets_cache:delete_subscriptions(4242). +bounded_worker_count_treats_the_bound_as_a_real_ceiling_test() -> + application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker), + ?assertEqual(4, bounded_worker_count(10000, 8, 4)), + ?assertEqual(1, bounded_worker_count(10000, 8, 1)), + ?assertEqual(2, bounded_worker_count(2500, 8, 2)). -fetch_missing_subscriptions_chunks_and_merges_test() -> +shared_ctx_payload_carries_no_badge_counts_test() -> + Ctx = test_send_ctx(), + Shared = shared_ctx_payload(Ctx), + ?assert(is_binary(Shared)), + ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, binary_to_term(Shared))), + ?assert(byte_size(Shared) < byte_size(term_to_binary(Ctx))). + +restore_chunk_ctx_rebuilds_the_ctx_with_only_the_chunk_badges_test() -> + Ctx = test_send_ctx(), + Shared = shared_ctx_payload(Ctx), + Chunk = [{2, [sub]}], + Restored = restore_chunk_ctx(Shared, chunk_badge_counts(Chunk, ctx_badge_counts(Ctx))), + ?assertEqual(9, tuple_size(Restored)), + ?assertEqual(#{2 => 6}, element(?CTX_BADGE_COUNTS, Restored)), + ?assertEqual(element(1, Ctx), element(1, Restored)), + ?assertEqual(element(7, Ctx), element(7, Restored)), + ?assertEqual(element(8, Ctx), element(8, Restored)), + ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, restore_chunk_ctx(Shared, #{}))). + +bounded_fanout_isolates_a_raising_task_and_counts_it_test() -> + Self = self(), + ok = meck:new(push_sender, [passthrough, no_link]), + ensure_test_counter_table(), + Before = counter_value(delivery_task_failures), + application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1), + try + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) -> + case UserId of + 3 -> error(deliberate_task_crash); + _ -> Self ! {bounded_sent, UserId} + end, + ok + end), + Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 5)], + ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())), + ?assertEqual([1, 2, 4, 5], lists:sort(collect_bounded_sent(4, []))), + ?assertEqual(Before + 1, counter_value(delivery_task_failures)) + after + application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker), + meck:unload(push_sender) + end. + +bounded_fanout_returns_reserved_slots_to_the_push_budget_test() -> + ok = meck:new(push_sender, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(?CNT_LIVE_WORKERS), + try + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) -> + ok + end), + Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 40)], + ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())), + ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)) + after + meck:unload(push_sender) + end. + +bounded_fanout_releases_reserved_slots_when_workers_are_killed_test() -> + ok = meck:new(push_sender, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(?CNT_LIVE_WORKERS), + ok = reset_test_counter(delivery_recipients_not_attempted), + application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1), + try + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) -> + exit(self(), kill) + end), + Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 4)], + ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())), + ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)), + ?assertEqual(4, counter_value(delivery_recipients_not_attempted)) + after + application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker), + meck:unload(push_sender) + end. + +claim_delivery_slots_reserves_before_the_next_job_reads_the_budget_test() -> + ensure_test_counter_table(), + ok = reset_test_counter(?CNT_LIVE_WORKERS), + ?assertEqual(8, claim_delivery_slots(8, 10)), + ?assertEqual(8, counter_value(?CNT_LIVE_WORKERS)), + ?assertEqual(2, claim_delivery_slots(8, 10)), + ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)), + ?assertEqual(0, claim_delivery_slots(4, 10)), + ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)), + ok = release_delivery_slots(10), + ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)), + ok = release_delivery_slots(5), + ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)). + +bounded_fanout_delivers_inline_when_the_push_budget_is_exhausted_test() -> + Self = self(), + ok = meck:new(push_sender, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(delivery_budget_exhausted), + Budget = gateway_http_client:push_max_concurrency(), + ok = reset_test_counter(?CNT_LIVE_WORKERS), + _ = claim_delivery_slots(Budget, Budget), + try + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) -> + Self ! {bounded_sent, UserId}, + ok + end), + Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 6)], + ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())), + ?assertEqual(lists:seq(1, 6), lists:sort(collect_bounded_sent(6, []))), + ?assertEqual(1, counter_value(delivery_budget_exhausted)), + ?assertEqual(Budget, counter_value(?CNT_LIVE_WORKERS)) + after + ok = release_delivery_slots(Budget), + meck:unload(push_sender) + end. + +delivery_stats_exposes_cumulative_loss_without_logs_test() -> + ensure_test_counter_table(), + Before = maps:get(delivery_recipients_not_attempted, delivery_stats()), + ok = bump_counter(delivery_recipients_not_attempted, 3), + Stats = delivery_stats(), + ?assertEqual(Before + 3, maps:get(delivery_recipients_not_attempted, Stats)), + ?assertEqual( + lists:sort([delivery_last_crash | delivery_counter_keys()]), + lists:sort(maps:keys(Stats)) + ). + +bump_counter_never_creates_the_counter_table_test() -> + case drop_test_counter_table() of + ok -> + ?assertEqual(ok, bump_counter(delivery_recipients_not_attempted, 1)), + ?assertEqual(0, counter_value(delivery_recipients_not_attempted)), + ?assertEqual(undefined, ets:info(?PUSH_COUNTERS, name)); + skip -> + ok + end. + +ensure_test_counter_table() -> + case ets:info(?PUSH_COUNTERS, name) of + undefined -> + _ = ets:new(?PUSH_COUNTERS, [ + named_table, public, set, {write_concurrency, true} + ]), + ok; + _ -> + ok + end. + +drop_test_counter_table() -> + case ets:info(?PUSH_COUNTERS, name) of + undefined -> + ok; + _ -> + try ets:delete(?PUSH_COUNTERS) of + _ -> ok + catch + error:badarg -> skip + end + end. + +reset_test_counter(Key) -> + try ets:insert(?PUSH_COUNTERS, {Key, 0}) of + _ -> ok + catch + error:badarg -> ok + end. + +bounded_fanout_delivers_every_recipient_test() -> + Self = self(), + ok = meck:new(push_sender, [passthrough, no_link]), + try + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) -> + Self ! {bounded_sent, UserId}, + ok + end), + Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 200)], + ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())), + ?assertEqual(lists:seq(1, 200), lists:sort(collect_bounded_sent(200, []))) + after + meck:unload(push_sender) + end. + +chunk_user_ids_uses_bounded_batches_test() -> + ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])), + ?assertEqual([], chunk_user_ids([], 2, [])). + +fetch_subscription_batches_stops_after_consecutive_failures_test() -> ok = meck:new(rpc_client, [passthrough, no_link]), try - ok = meck:expect(rpc_client, call, fun subscriptions_chunk_stub/1), - UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2), - Expected = #{ - <<"1">> => [{chunk, ?SUBSCRIPTION_FETCH_CHUNK_SIZE}], - integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}] - }, - ?assertEqual({UserIds, Expected}, fetch_missing_subscriptions(UserIds)), - ?assertEqual(2, meck:num_calls(rpc_client, call, '_')) + ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end), + Batches = [[N] || N <- lists:seq(1, 10)], + ?assertEqual( + {[], 10, 10, ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES}, + fetch_subscription_batches(Batches, {[], 0, 0, 0}) + ), + ?assertEqual( + ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES, + length(meck:history(rpc_client)) + ) after meck:unload(rpc_client) end. -fetch_missing_subscriptions_keeps_other_chunks_when_one_fails_test() -> +fetch_missing_in_batches_sends_only_successful_batches_test() -> + Self = self(), + ok = meck:new(push_sender, [passthrough, no_link]), ok = meck:new(rpc_client, [passthrough, no_link]), + ok = meck:new(push_ets_cache, [passthrough, no_link]), + application:set_env(fluxer_gateway, push_subscription_fetch_batch_size, 2), try - ok = meck:expect(rpc_client, call, fun failing_first_chunk_stub/1), - UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2), - TailUserIds = [?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2], - Expected = #{ - integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}] - }, - ?assertEqual({TailUserIds, Expected}, fetch_missing_subscriptions(UserIds)) + ok = meck:expect(push_ets_cache, put_subscriptions, fun(_UserId, _Subs) -> ok end), + ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) -> + case Ids of + [<<"1">>, <<"2">>] -> {ok, #{<<"1">> => [sub1], <<"2">> => [sub2]}}; + _ -> {error, unavailable} + end + end), + ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, Subs, _Ctx) -> + Self ! {bounded_sent, {UserId, Subs}}, + ok + end), + ?assertEqual( + ok, fetch_and_send_missing_subscriptions([1, 2, 3, 4], test_send_ctx()) + ), + ?assertEqual([{1, [sub1]}, {2, [sub2]}], lists:sort(collect_bounded_sent(2, []))) + after + application:unset_env(fluxer_gateway, push_subscription_fetch_batch_size), + meck:unload(push_ets_cache), + meck:unload(rpc_client), + meck:unload(push_sender) + end. + +subscription_fetch_failure_counts_dropped_recipients_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(subscription_fetch_calls_failed), + ok = reset_test_counter(subscription_fetch_users_dropped), + try + ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end), + ?assertEqual( + ok, fetch_and_send_missing_subscriptions([1, 2, 3], test_send_ctx()) + ), + ?assertEqual(1, counter_value(subscription_fetch_calls_failed)), + ?assertEqual(3, counter_value(subscription_fetch_users_dropped)) after meck:unload(rpc_client) end. -subscriptions_chunk_stub(#{<<"user_ids">> := [FirstUserIdBin | _] = UserIdBins}) -> - {ok, #{FirstUserIdBin => [{chunk, length(UserIdBins)}]}}. +clear_notification_fetch_failure_counts_the_dropped_recipient_test() -> + ok = meck:new(rpc_client, [passthrough, no_link]), + ensure_test_counter_table(), + ok = reset_test_counter(subscription_fetch_calls_failed), + ok = reset_test_counter(subscription_fetch_users_dropped), + try + ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end), + ?assertEqual( + ok, fetch_and_send_clear_notification_from_rpc(7, 8, 9, 0) + ), + ?assertEqual(1, counter_value(subscription_fetch_calls_failed)), + ?assertEqual(1, counter_value(subscription_fetch_users_dropped)) + after + meck:unload(rpc_client) + end. -failing_first_chunk_stub(#{<<"user_ids">> := [<<"1">> | _]}) -> - {error, timeout}; -failing_first_chunk_stub(Request) -> - subscriptions_chunk_stub(Request). +collect_bounded_sent(0, Acc) -> + Acc; +collect_bounded_sent(Count, Acc) -> + receive + {bounded_sent, Sent} -> + collect_bounded_sent(Count - 1, [Sent | Acc]) + after 5000 -> + ?assert(false) + end. collect_sent_subscription_tasks(0, Acc) -> Acc; diff --git a/fluxer_gateway/src/session/session_init.erl b/fluxer_gateway/src/session/session_init.erl index 0270fbb43..cd0156745 100644 --- a/fluxer_gateway/src/session/session_init.erl +++ b/fluxer_gateway/src/session/session_init.erl @@ -63,9 +63,21 @@ normalize_seq(_) -> 0. -spec normalize_buffer(term()) -> [map()]. normalize_buffer(Buffer) when is_list(Buffer) -> lists:filtermap(fun normalize_buffer_event/1, Buffer); +normalize_buffer(Buffer) when is_map(Buffer) -> + lists:filtermap(fun normalize_buffer_event/1, deque_to_list(Buffer)); normalize_buffer(_) -> []. +-spec deque_to_list(map()) -> [term()]. +deque_to_list(Buffer) -> + try limited_deque:to_list(eqwalizer:dynamic_cast(Buffer)) of + List when is_list(List) -> List; + _ -> [] + catch + error:_Reason -> []; + exit:_Reason -> [] + end. + -spec normalize_buffer_event(term()) -> {true, map()} | false. normalize_buffer_event(Event) when is_map(Event) -> case maps:get(seq, Event, -1) of @@ -75,19 +87,43 @@ normalize_buffer_event(Event) when is_map(Event) -> normalize_buffer_event(_) -> false. +-spec init_buffer_bytes(term(), [map()]) -> non_neg_integer(). +init_buffer_bytes(_Raw, []) -> + 0; +init_buffer_bytes(Raw, Buffer) when is_map(Raw) -> + restored_deque_bytes(Raw, Buffer); +init_buffer_bytes(_Raw, Buffer) -> + replay_buffer_bytes(Buffer). + +-spec restored_deque_bytes(map(), [map()]) -> non_neg_integer(). +restored_deque_bytes(Raw, Buffer) -> + try limited_deque:bytes(eqwalizer:dynamic_cast(Raw)) of + Bytes when is_integer(Bytes), Bytes >= 0 -> Bytes; + _ -> replay_buffer_bytes(Buffer) + catch + error:_Reason -> replay_buffer_bytes(Buffer); + exit:_Reason -> replay_buffer_bytes(Buffer) + end. + -spec replay_buffer_bytes([map()]) -> non_neg_integer(). replay_buffer_bytes(Buffer) -> WordSize = erlang:system_info(wordsize), - trunc( - lists:foldl( - fun(Event, Acc) -> - Acc + erts_debug:flat_size(Event) * WordSize - end, - 0, - Buffer - ) + lists:foldl( + fun(Event, Acc) -> Acc + erts_debug:flat_size(Event) * WordSize end, + 0, + Buffer ). +-spec replay_payload_bytes([map()]) -> non_neg_integer(). +replay_payload_bytes(Buffer) -> + lists:foldl(fun(Event, Acc) -> Acc + entry_payload_bytes(Event) end, 0, Buffer). + +-spec entry_payload_bytes(term()) -> non_neg_integer(). +entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) -> + byte_size(Payload); +entry_payload_bytes(_Event) -> + 0. + -spec build_ignored_events_map(term()) -> #{binary() => true}. build_ignored_events_map(Events) when is_list(Events) -> maps:from_list([{Event, true} || Event <- Events, is_binary(Event)]); @@ -193,7 +229,8 @@ extract_core_fields( token_hash => TokenHash, auth_session_id_hash => AuthSessionIdHash, buffer => Buffer, - buffer_bytes => replay_buffer_bytes(Buffer), + buffer_bytes => init_buffer_bytes(maps:get(buffer, D, []), Buffer), + replay_payload_bytes => replay_payload_bytes(Buffer), seq => Seq, ack_seq => AckSeq, properties => Properties, @@ -285,7 +322,7 @@ schedule_timers(#{bot := Bot, guilds := GuildsMap}) -> init_prewarm_guilds(Bot, GuildIds), lists:foreach(fun(Gid) -> self() ! {guild_connect, Gid, 0} end, GuildIds), erlang:send_after(5000, self(), premature_readiness), - erlang:send_after(200, self(), enable_presence_updates), + erlang:send_after(10000, self(), enable_presence_updates), erlang:send_after(60000, self(), check_ack_lag), ok. @@ -468,4 +505,68 @@ normalize_buffer_filters_invalid_entries_test() -> ?assertEqual([#{seq => 1}], normalize_buffer(Buffer)), ?assertEqual([], normalize_buffer(undefined)). +normalize_buffer_accepts_deque_test() -> + Deque = test_deque([#{seq => 1}, #{seq => 2}]), + ?assertEqual([#{seq => 1}, #{seq => 2}], normalize_buffer(Deque)). + +normalize_buffer_filters_invalid_deque_entries_test() -> + Deque = test_deque([#{seq => 1}, #{seq => -1}, #{no_seq => true}, <<"bad">>]), + ?assertEqual([#{seq => 1}], normalize_buffer(Deque)). + +normalize_buffer_ignores_non_deque_map_test() -> + ?assertEqual([], normalize_buffer(#{seq => 1})), + ?assertEqual([], normalize_buffer(#{})), + ?assertEqual([], normalize_buffer(undefined)). + +build_state_preserves_transferred_deque_buffer_test() -> + Data = transferred_session_data(test_deque([#{seq => 13}, #{seq => 14}])), + State = build_state(Data), + ?assertEqual([#{seq => 13}, #{seq => 14}], maps:get(buffer, State)), + ?assertEqual( + replay_buffer_bytes([#{seq => 13}, #{seq => 14}]), + maps:get(buffer_bytes, State) + ), + ?assertEqual(14, maps:get(seq, State)), + ?assertEqual(12, maps:get(ack_seq, State)). + +build_state_reuses_deque_byte_total_test() -> + Events = [#{seq => 21}, #{seq => 22}], + Data = transferred_session_data(test_deque(Events)), + State = build_state(Data), + ?assertEqual(limited_deque:bytes(test_deque(Events)), maps:get(buffer_bytes, State)), + ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)). + +replay_buffer_bytes_agrees_with_deque_accounting_test() -> + Entries = [ + #{seq => 1, event => message_create, data => #{<<"content">> => <<"hi">>}}, + #{seq => 2, event => message_create, data => {pre_encoded, binary:copy(<<"x">>, 70000)}} + ], + lists:foreach( + fun(Entry) -> + ?assertEqual( + limited_deque:bytes(limited_deque:from_list([Entry], 4096, 0)), + replay_buffer_bytes([Entry]) + ) + end, + Entries + ). + +build_state_walks_list_buffer_bytes_test() -> + Events = [#{seq => 13}, #{seq => 14}], + Data = transferred_session_data(Events), + State = build_state(Data), + ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)). + +build_state_zeroes_bytes_for_non_deque_map_buffer_test() -> + Data = transferred_session_data(#{seq => 13}), + State = build_state(Data), + ?assertEqual([], maps:get(buffer, State)), + ?assertEqual(0, maps:get(buffer_bytes, State)). + +transferred_session_data(Buffer) -> + (base_session_data(#{}))#{buffer => Buffer, seq => 14, ack_seq => 12}. + +test_deque(Events) -> + limited_deque:from_list(Events, 4096, 16777216). + -endif. diff --git a/fluxer_gateway/src/session/session_lifecycle.erl b/fluxer_gateway/src/session/session_lifecycle.erl index 8d77ae453..54f2577dc 100644 --- a/fluxer_gateway/src/session/session_lifecycle.erl +++ b/fluxer_gateway/src/session/session_lifecycle.erl @@ -34,6 +34,9 @@ -type status() :: session:status(). -type voice_state_entry() :: {binary(), map()}. +-define(MAX_INLINE_RESUME_REPLAY, 256). +-define(MAX_TRANSFER_PRE_ENCODED_BYTES, 4194304). + -spec handle_terminate_call([binary()], session_state()) -> {stop, normal, terminated, session_state()} | {reply, ignored, session_state()}. handle_terminate_call(Hashes, #{auth_session_id_hash := AuthHash} = State) -> @@ -287,10 +290,25 @@ handle_heartbeat_ack(Seq, #{buffer := Buffer} = State) -> true -> session_init:replay_buffer_bytes(NewBuffer); false -> limited_deque:bytes(NewBuffer) end, - NewState0 = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes}, + NewState0 = ack_state(State, AckedSeq, NewBuffer, NewBytes), NewState = session_connection_guild:repair_stalled_guild_connects(NewState0), {reply, true, NewState}. +-spec ack_state(session_state(), seq(), term(), non_neg_integer()) -> session_state(). +ack_state(State, AckedSeq, NewBuffer, NewBytes) -> + Base = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes}, + Base#{replay_payload_bytes => buffer_payload_bytes(NewBuffer)}. + +-spec buffer_payload_bytes(term()) -> non_neg_integer(). +buffer_payload_bytes(Buffer) when is_list(Buffer) -> + lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Buffer); +buffer_payload_bytes(Buffer) -> + try limited_deque:to_list(Buffer) of + Entries -> buffer_payload_bytes(Entries) + catch + _:_ -> 0 + end. + -spec drop_acked_buffer( seq(), eqwalizer:dynamic(limited_deque:deque() | [map()]) ) -> eqwalizer:dynamic(limited_deque:deque() | [map()]). @@ -325,11 +343,41 @@ handle_resume(Seq, SocketPid, #{seq := CurrentSeq} = State) -> MissedEvents = events_after_seq(Seq, BufferList), NewState0 = cancel_offline_timer(cancel_resume_timer(State)), NewState1 = replace_socket(SocketPid, NewState0), + ReplyEvents = replay_missed_events_inline(MissedEvents, SocketPid), NewState = NewState1#{status => ResumeStatus, resume_status => ResumeStatus}, NewState2 = ensure_presence_attached_on_resume( NewState, SessionId, ResumeStatus, Afk, Mobile ), - {reply, {ok, MissedEvents, CurrentSeq}, NewState2}. + {reply, {ok, ReplyEvents, CurrentSeq}, NewState2}. + +-spec replay_missed_events_inline([map()], pid()) -> [map()]. +replay_missed_events_inline(MissedEvents, SocketPid) -> + case inline_replay_allowed(MissedEvents) of + true -> + lists:foreach(fun(Event) -> send_missed_event(SocketPid, Event) end, MissedEvents), + []; + false -> + MissedEvents + end. + +-spec inline_replay_allowed([map()]) -> boolean(). +inline_replay_allowed(MissedEvents) -> + within_inline_replay_limit(MissedEvents, ?MAX_INLINE_RESUME_REPLAY). + +-spec within_inline_replay_limit([map()], non_neg_integer()) -> boolean(). +within_inline_replay_limit([], _Remaining) -> + true; +within_inline_replay_limit(_MissedEvents, 0) -> + false; +within_inline_replay_limit([_Event | Rest], Remaining) -> + within_inline_replay_limit(Rest, Remaining - 1). + +-spec send_missed_event(pid(), map()) -> ok. +send_missed_event(SocketPid, #{event := Event, data := Data, seq := EventSeq}) -> + SocketPid ! {dispatch, Event, guild_data_wire:payload(Data), EventSeq}, + ok; +send_missed_event(_SocketPid, _Event) -> + ok. -spec ensure_presence_attached_on_resume( session_state(), session_id(), status(), boolean(), boolean() @@ -509,7 +557,6 @@ serialize_state(State) -> resume_status => maps:get(resume_status, State, maps:get(status, State)), afk => maps:get(afk, State), mobile => maps:get(mobile, State), - buffer => maps:get(buffer, State), ready => maps:get(ready, State), bot => maps:get(bot, State, false), shard => maps:get(shard, State, undefined), @@ -553,20 +600,340 @@ serialize_transfer_identity(State) -> debounce_reactions => maps:get(debounce_reactions, State, false) }. --spec transfer_buffer(eqwalizer:dynamic(limited_deque:deque() | [map()])) -> [term()]. -transfer_buffer(Buffer) when is_list(Buffer) -> Buffer; -transfer_buffer(Buffer) -> limited_deque:to_list(Buffer). - -spec serialize_transfer_runtime(session_state()) -> map(). serialize_transfer_runtime(State) -> + {Buffer, AckSeq} = transfer_buffer( + maps:get(buffer, State, []), maps:get(ack_seq, State, 0) + ), #{ channels => maps:get(channels, State, #{}), relationships => maps:get(relationships, State, #{}), seq => maps:get(seq, State, 0), - ack_seq => maps:get(ack_seq, State, 0), - buffer => transfer_buffer(maps:get(buffer, State, [])), + ack_seq => AckSeq, + buffer => Buffer, collected_guild_states => maps:get(collected_guild_states, State, []), collected_sessions => maps:get(collected_sessions, State, []), collected_presences => maps:get(collected_presences, State, []), guild_subscription_state => maps:get(guild_subscription_state, State, #{}) }. + +%% Always normalises to a list before trimming: a limited_deque left in the transfer +%% map does not survive the handoff to the receiving node's session_init. +-spec transfer_buffer( + eqwalizer:dynamic(limited_deque:deque() | [map()]), seq() +) -> {[map()], seq()}. +transfer_buffer(Buffer, AckSeq) when is_list(Buffer) -> + trim_transfer_list(Buffer, transfer_payload_bytes(Buffer), AckSeq); +transfer_buffer(Buffer, AckSeq) -> + ListBuffer = limited_deque:to_list(Buffer), + trim_transfer_list(ListBuffer, transfer_payload_bytes(ListBuffer), AckSeq). + +-spec trim_transfer_list([map()], non_neg_integer(), seq()) -> {[map()], seq()}. +trim_transfer_list(Buffer, Bytes, AckSeq) when Bytes =< ?MAX_TRANSFER_PRE_ENCODED_BYTES -> + {Buffer, AckSeq}; +trim_transfer_list([], _Bytes, AckSeq) -> + {[], AckSeq}; +trim_transfer_list([Dropped | Rest], Bytes, AckSeq) -> + trim_transfer_list( + Rest, + max(0, Bytes - entry_payload_bytes(Dropped)), + dropped_ack_seq(Dropped, AckSeq) + ). + +-spec transfer_payload_bytes([term()]) -> non_neg_integer(). +transfer_payload_bytes(Entries) -> + lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Entries). + +-spec entry_payload_bytes(term()) -> non_neg_integer(). +entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) -> + byte_size(Payload); +entry_payload_bytes(_Entry) -> + 0. + +-spec dropped_ack_seq(term(), seq()) -> seq(). +dropped_ack_seq(#{seq := Seq}, AckSeq) when is_integer(Seq), Seq > AckSeq -> + Seq; +dropped_ack_seq(_Dropped, AckSeq) -> + AckSeq. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +resume_flag_state(Overrides) -> + maps:merge( + #{ + id => <<"session-resume-flag-test">>, + status => online, + afk => false, + mobile => false, + presence_pid => undefined, + presence_mref => undefined, + socket_pid => undefined, + socket_mref => undefined, + resume_timer => undefined, + offline_timer => undefined, + ack_seq => 0, + buffer => [], + seq => 0 + }, + Overrides + ). + +buffered_event(Seq) -> + #{seq => Seq, event => message_create, data => #{<<"id">> => Seq}}. + +with_env(Key, Value, Fun) -> + Previous = application:get_env(fluxer_gateway, Key), + application:set_env(fluxer_gateway, Key, Value), + try + Fun() + after + restore_env(Key, Previous) + end. + +restore_env(Key, {ok, Previous}) -> + application:set_env(fluxer_gateway, Key, Previous); +restore_env(Key, undefined) -> + application:unset_env(fluxer_gateway, Key). + +next_dispatch() -> + receive + {dispatch, _Event, _Data, _Seq} = Msg -> Msg + after 200 -> no_dispatch + end. + +drain_dispatches() -> + receive + {dispatch, _Event, _Data, _Seq} -> drain_dispatches() + after 0 -> ok + end. + +count_dispatches() -> + count_dispatches(0). + +count_dispatches(Count) -> + receive + {dispatch, _Event, _Data, _Seq} -> count_dispatches(Count + 1) + after 0 -> Count + end. + +inline_replay_sends_backlog_and_replies_empty_test() -> + drain_dispatches(), + State0 = resume_flag_state(#{seq => 2, buffer => [buffered_event(1), buffered_event(2)]}), + {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0), + First = next_dispatch(), + Second = next_dispatch(), + ?assertEqual( + {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 1}), 1}, First + ), + ?assertEqual( + {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2}, Second + ), + ?assertEqual(no_dispatch, next_dispatch()). + +inline_replay_sends_wire_payload_test() -> + drain_dispatches(), + State0 = resume_flag_state(#{seq => 1, buffer => [buffered_event(1)]}), + {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0), + {dispatch, message_create, Data, 1} = next_dispatch(), + ?assertEqual(#{<<"id">> => <<"1">>}, Data). + +inline_replay_skips_malformed_buffer_entry_test() -> + drain_dispatches(), + State0 = resume_flag_state(#{seq => 2, buffer => [#{seq => 1}, buffered_event(2)]}), + {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0), + ?assertEqual( + {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2}, + next_dispatch() + ), + ?assertEqual(no_dispatch, next_dispatch()). + +inline_replay_sends_nothing_when_backlog_empty_test() -> + drain_dispatches(), + State0 = resume_flag_state(#{seq => 1, ack_seq => 1, buffer => [buffered_event(1)]}), + {reply, {ok, [], 1}, _State1} = handle_resume(1, self(), State0), + ?assertEqual(no_dispatch, next_dispatch()). + +inline_replay_bounded_batch_is_sent_inline_test() -> + drain_dispatches(), + Events = [buffered_event(Seq) || Seq <- lists:seq(1, ?MAX_INLINE_RESUME_REPLAY)], + State0 = resume_flag_state(#{seq => ?MAX_INLINE_RESUME_REPLAY, buffer => Events}), + {reply, {ok, [], ?MAX_INLINE_RESUME_REPLAY}, _State1} = handle_resume(0, self(), State0), + ?assertEqual(?MAX_INLINE_RESUME_REPLAY, count_dispatches()). + +inline_replay_oversized_batch_falls_back_to_reply_test() -> + drain_dispatches(), + Over = ?MAX_INLINE_RESUME_REPLAY + 1, + Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)], + State0 = resume_flag_state(#{seq => Over, buffer => Events}), + {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0), + ?assertEqual(Over, length(Missed)), + ?assertEqual(no_dispatch, next_dispatch()). + +inline_replay_oversized_batch_still_attaches_socket_test() -> + drain_dispatches(), + Over = ?MAX_INLINE_RESUME_REPLAY + 1, + Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)], + State0 = resume_flag_state(#{seq => Over, buffer => Events}), + {reply, {ok, _Missed, Over}, State1} = handle_resume(0, self(), State0), + ?assertEqual(self(), maps:get(socket_pid, State1)). + +pre_encoded_buffered_event(Seq) -> + #{seq => Seq, event => message_create, data => {pre_encoded, <<"{}">>}}. + +heartbeat_ack_trims_pre_encoded_entries_test() -> + Buffer = limited_deque:from_list( + [pre_encoded_buffered_event(1), pre_encoded_buffered_event(2)], 4096, 16777216 + ), + State0 = resume_flag_state(#{seq => 2, buffer => Buffer}), + {reply, true, State1} = handle_heartbeat_ack(1, State0), + Remaining = limited_deque:to_list(maps:get(buffer, State1)), + ?assertEqual([2], [maps:get(seq, Event) || Event <- Remaining]). + +heartbeat_ack_keeps_trimming_pre_encoded_head_test() -> + Buffer = limited_deque:from_list( + [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, 4)], 4096, 16777216 + ), + State0 = resume_flag_state(#{seq => 4, buffer => Buffer}), + {reply, true, State1} = handle_heartbeat_ack(4, State0), + ?assertEqual(0, limited_deque:size(maps:get(buffer, State1))). + +inline_replay_sends_pre_encoded_entry_unchanged_test() -> + drain_dispatches(), + Entry = pre_encoded_buffered_event(1), + State0 = resume_flag_state(#{seq => 1, buffer => [Entry]}), + {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0), + ?assertEqual({dispatch, message_create, maps:get(data, Entry), 1}, next_dispatch()). + +oversized_replay_returns_pre_encoded_entries_unchanged_test() -> + drain_dispatches(), + Over = ?MAX_INLINE_RESUME_REPLAY + 1, + Entries = [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, Over)], + State0 = resume_flag_state(#{seq => Over, buffer => Entries}), + {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0), + ?assertEqual(Entries, Missed), + ?assertEqual(no_dispatch, next_dispatch()). + +resume_ignores_withdrawn_buffer_floor_env_test() -> + drain_dispatches(), + State0 = resume_flag_state(#{seq => 5, buffer => [buffered_event(4), buffered_event(5)]}), + with_env(session_resume_buffer_floor, true, fun() -> + {reply, {ok, [], 5}, _State1} = handle_resume(2, self(), State0), + ?assertMatch({dispatch, _, _, 4}, next_dispatch()), + ?assertMatch({dispatch, _, _, 5}, next_dispatch()), + ?assertEqual(no_dispatch, next_dispatch()) + end). + +serialize_state_omits_replay_buffer_test() -> + State = resume_flag_state(#{ + user_id => 12345, + user_data => #{}, + version => 9, + properties => #{}, + ready => undefined, + guilds => #{}, + collected_guild_states => [], + collected_sessions => [], + buffer => [buffered_event(1)] + }), + Serialized = serialize_state(State), + ?assertEqual(false, maps:is_key(buffer, Serialized)), + ?assertEqual(false, maps:is_key(buffer_bytes, Serialized)), + ?assertEqual(0, maps:get(seq, Serialized)). + +transfer_entry(Seq, PayloadBytes) -> + #{ + seq => Seq, + event => message_create, + data => {pre_encoded, binary:copy(<<"x">>, PayloadBytes)} + }. + +transfer_deque(Entries) -> + limited_deque:from_list(Entries, 4096, 16777216). + +transfer_runtime_state(Buffer, AckSeq) -> + #{ + channels => #{}, + relationships => #{}, + seq => 4, + ack_seq => AckSeq, + buffer => Buffer, + collected_guild_states => [], + collected_sessions => [], + collected_presences => [], + guild_subscription_state => #{} + }. + +oversized_transfer_entries() -> + [transfer_entry(Seq, 1500000) || Seq <- lists:seq(1, 4)]. + +transfer_buffer_is_bounded_test() -> + Deque = transfer_deque(oversized_transfer_entries()), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + Entries = maps:get(buffer, Runtime), + ?assert(transfer_payload_bytes(Entries) =< ?MAX_TRANSFER_PRE_ENCODED_BYTES), + ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- Entries]), + ?assertEqual(2, maps:get(ack_seq, Runtime)). + +transfer_buffer_admits_one_full_size_event_test() -> + Single = transfer_entry(1, 2097152), + Deque = transfer_deque([Single]), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + ?assertEqual([Single], maps:get(buffer, Runtime)), + ?assertEqual(0, maps:get(ack_seq, Runtime)). + +transfer_buffer_bound_keeps_entry_representation_test() -> + Deque = transfer_deque(oversized_transfer_entries()), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + Entries = maps:get(buffer, Runtime), + ?assertEqual(Entries, session_init:normalize_buffer(Entries)), + ?assertEqual([transfer_entry(3, 1500000), transfer_entry(4, 1500000)], Entries). + +transfer_buffer_bound_leaves_small_buffers_alone_test() -> + Deque = transfer_deque([buffered_event(1), buffered_event(2)]), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)), + ?assertEqual(0, maps:get(ack_seq, Runtime)). + +fat_map_entry(Seq) -> + #{ + seq => Seq, + event => message_create, + data => #{<<"content">> => lists:duplicate(200000, $x)} + }. + +transfer_buffer_bound_never_truncates_map_entries_test() -> + Entries = [fat_map_entry(Seq) || Seq <- lists:seq(1, 2)], + Deque = transfer_deque(Entries), + ?assert(limited_deque:bytes(Deque) > ?MAX_TRANSFER_PRE_ENCODED_BYTES), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)), + ?assertEqual(0, maps:get(ack_seq, Runtime)). + +transfer_buffer_bound_never_lowers_ack_seq_test() -> + Deque = transfer_deque([buffered_event(9)]), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 8)), + ?assertEqual(8, maps:get(ack_seq, Runtime)). + +transfer_buffer_bounds_list_buffers_test() -> + Entries = oversized_transfer_entries(), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Entries, 0)), + ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- maps:get(buffer, Runtime)]), + ?assertEqual(2, maps:get(ack_seq, Runtime)). + +transfer_buffer_bound_makes_stale_resume_invalid_test() -> + Deque = transfer_deque(oversized_transfer_entries()), + Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)), + Restored = resume_flag_state(#{ + seq => 4, + ack_seq => maps:get(ack_seq, Runtime), + buffer => maps:get(buffer, Runtime) + }), + ?assertMatch({reply, invalid_seq, _}, handle_resume(1, self(), Restored)), + drain_dispatches(), + {reply, {ok, [], 4}, _} = handle_resume(2, self(), Restored), + ?assertMatch({dispatch, _, _, 3}, next_dispatch()), + ?assertMatch({dispatch, _, _, 4}, next_dispatch()), + ?assertEqual(no_dispatch, next_dispatch()). + +-endif. diff --git a/fluxer_gateway/src/session/session_ready_dispatch.erl b/fluxer_gateway/src/session/session_ready_dispatch.erl index 910c564b6..905c30d9a 100644 --- a/fluxer_gateway/src/session/session_ready_dispatch.erl +++ b/fluxer_gateway/src/session/session_ready_dispatch.erl @@ -70,7 +70,8 @@ dispatch_ready_to_socket(State) -> IsBot, CollectedGuilds, Guilds, StateAfterReady ), schedule_call_creates(StateAfterGuilds, SessionId), - FinalState = StateAfterGuilds#{ + StateAfterPresences = release_pending_presences(StateAfterGuilds, CollectedPresences), + FinalState = StateAfterPresences#{ ready => undefined, collected_guild_states => [], collected_sessions => [], @@ -79,6 +80,65 @@ dispatch_ready_to_socket(State) -> erlang:garbage_collect(), {noreply, FinalState}. +-spec release_pending_presences(session_state(), [map()]) -> session_state(). +release_pending_presences(State, CollectedPresences) -> + Covered = covered_presence_ids(CollectedPresences), + Pruned = drop_covered_pending_presences(State, Covered), + Flushed = session_dispatch:flush_all_pending_presences(Pruned), + Flushed#{suppress_presence_updates => false}. + +-spec covered_presence_ids([map()]) -> #{integer() => true}. +covered_presence_ids(CollectedPresences) -> + lists:foldl( + fun(Presence, Acc) -> + case presence_user_id(Presence) of + undefined -> Acc; + UserId -> Acc#{UserId => true} + end + end, + #{}, + CollectedPresences + ). + +-spec drop_covered_pending_presences(session_state(), #{integer() => true}) -> session_state(). +drop_covered_pending_presences(State, Covered) when map_size(Covered) =:= 0 -> + State; +drop_covered_pending_presences(State, Covered) -> + case maps:get(pending_presences, State, undefined) of + undefined -> + State; + Pending -> + State#{pending_presences => filter_pending_presences(Pending, Covered)} + end. + +-spec filter_pending_presences(term(), #{integer() => true}) -> term(). +filter_pending_presences(Pending, Covered) -> + Keep = fun(Entry) -> not covered_entry(Entry, Covered) end, + try queue:filter(Keep, Pending) of + Filtered -> Filtered + catch + error:_ when is_list(Pending) -> lists:filter(Keep, Pending); + error:_ -> Pending + end. + +-spec covered_entry(term(), #{integer() => true}) -> boolean(). +covered_entry(#{user_id := UserId}, Covered) when is_integer(UserId) -> + maps:is_key(UserId, Covered); +covered_entry(_Entry, _Covered) -> + false. + +-spec presence_user_id(term()) -> integer() | undefined. +presence_user_id(Presence) when is_map(Presence) -> + User = maps:get(<<"user">>, Presence, #{}), + presence_id(User); +presence_user_id(_Presence) -> + undefined. + +-spec presence_id(term()) -> integer() | undefined. +presence_id(User) when is_map(User) -> + snowflake_id:parse_maybe(maps:get(<<"id">>, User, undefined)); +presence_id(_User) -> + undefined. -spec is_staff_session(session_state()) -> boolean(). is_staff_session(#{is_staff := true}) -> true; diff --git a/fluxer_gateway/src/utils/custom_status_expiry.erl b/fluxer_gateway/src/utils/custom_status_expiry.erl new file mode 100644 index 000000000..b0f47974a --- /dev/null +++ b/fluxer_gateway/src/utils/custom_status_expiry.erl @@ -0,0 +1,406 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(custom_status_expiry). +-typing([eqwalizer]). + +-export([ + enabled/0, + clear_if_expired/1, + clear_if_expired/2, + remaining_ms/2, + next_wakeup_ms/1, + reconcile_message/0, + repair/1 +]). + +-export_type([custom_status/0]). + +-type custom_status() :: map() | null. +-type user_id() :: integer(). +-type wakeup() :: {ok, pos_integer()} | none. +-type offset() :: {ok, integer()} | none. + +-define(ENABLED_KEY, custom_status_expiry_enabled). +-define(MAX_REPAIR_WINDOW_SECONDS, 86400). +-define(MAX_TIMER_MS, 86400000). +-define(WAKEUP_JITTER_MS, 5000). + +-spec enabled() -> boolean(). +enabled() -> + application:get_env(fluxer_gateway, ?ENABLED_KEY, false) =:= true. + +-spec clear_if_expired(term()) -> custom_status(). +clear_if_expired(CustomStatus) -> + clear_when_enabled(enabled(), CustomStatus). + +-spec clear_when_enabled(boolean(), term()) -> custom_status(). +clear_when_enabled(true, CustomStatus) -> + clear_if_expired(CustomStatus, erlang:system_time(millisecond)); +clear_when_enabled(false, CustomStatus) when is_map(CustomStatus) -> + CustomStatus; +clear_when_enabled(false, _CustomStatus) -> + null. + +-spec clear_if_expired(term(), term()) -> custom_status(). +clear_if_expired(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) -> + keep_or_clear(expired(CustomStatus, NowMs), CustomStatus); +clear_if_expired(CustomStatus, _NowMs) when is_map(CustomStatus) -> + CustomStatus; +clear_if_expired(_CustomStatus, _NowMs) -> + null. + +-spec keep_or_clear(boolean(), map()) -> custom_status(). +keep_or_clear(true, _CustomStatus) -> null; +keep_or_clear(false, CustomStatus) -> CustomStatus. + +-spec expired(map(), integer()) -> boolean(). +expired(CustomStatus, NowMs) -> + case remaining_ms(CustomStatus, NowMs) of + {ok, RemainingMs} -> RemainingMs =< 0; + none -> false + end. + +-spec remaining_ms(term(), term()) -> offset(). +remaining_ms(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) -> + offset_from(expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, undefined)), NowMs); +remaining_ms(_CustomStatus, _NowMs) -> + none. + +-spec offset_from(offset(), integer()) -> offset(). +offset_from({ok, ExpiryMs}, NowMs) -> {ok, ExpiryMs - NowMs}; +offset_from(none, _NowMs) -> none. + +-spec expires_at_ms(term()) -> offset(). +expires_at_ms(Value) when is_binary(Value) -> + parse_rfc3339(binary_to_list(Value)); +expires_at_ms(Value) when is_list(Value) -> + parse_rfc3339(Value); +expires_at_ms(_Value) -> + none. + +-spec parse_rfc3339(term()) -> offset(). +parse_rfc3339(Chars) -> + try calendar:rfc3339_to_system_time(Chars, [{unit, millisecond}]) of + Ms when is_integer(Ms) -> {ok, Ms}; + _Other -> none + catch + _Class:_Reason -> none + end. + +-spec next_wakeup_ms(term()) -> wakeup(). +next_wakeup_ms(CustomStatus) -> + next_wakeup_ms(enabled(), CustomStatus). + +-spec next_wakeup_ms(boolean(), term()) -> wakeup(). +next_wakeup_ms(false, _CustomStatus) -> + none; +next_wakeup_ms(true, CustomStatus) -> + wakeup_delay(remaining_ms(CustomStatus, erlang:system_time(millisecond))). + +-spec wakeup_delay(offset()) -> wakeup(). +wakeup_delay(none) -> + none; +wakeup_delay({ok, RemainingMs}) when RemainingMs =< 0 -> + none; +wakeup_delay({ok, RemainingMs}) when RemainingMs > ?MAX_TIMER_MS -> + {ok, jittered(?MAX_TIMER_MS)}; +wakeup_delay({ok, RemainingMs}) -> + {ok, jittered(RemainingMs)}. + +-spec jittered(pos_integer()) -> pos_integer(). +jittered(DelayMs) -> + DelayMs + rand:uniform(?WAKEUP_JITTER_MS). + +-spec reconcile_message() -> {'$gen_cast', reconcile_flattened_presence}. +reconcile_message() -> + {'$gen_cast', reconcile_flattened_presence}. + +-spec repair(term()) -> map(). +repair(WindowSeconds) when + is_integer(WindowSeconds), WindowSeconds >= 0, WindowSeconds =< ?MAX_REPAIR_WINDOW_SECONDS +-> + Considered = local_presence_user_ids(), + Pids = expiring_presence_pids(Considered), + Total = length(Pids), + ok = schedule_reconciles(Pids, Total, WindowSeconds * 1000), + #{ + scheduled => Total, + considered => length(Considered), + window_seconds => WindowSeconds, + enabled => enabled() + }; +repair(_WindowSeconds) -> + #{error => invalid_window_seconds}. + +%% Only presences whose cached payload actually carries an expires_at are worth +%% nudging. Nudging every local presence would cost one payload rebuild and one +%% replicated cache write each, thousands per node, to correct a few dozen. +%% Filtering on "has an expiry" rather than "is expired" is deliberate: it also +%% arms the ongoing wakeup for statuses that have not expired yet. +-spec expiring_presence_pids([integer()]) -> [pid()]. +expiring_presence_pids(UserIds) -> + [ + Pid + || UserId <- UserIds, + has_expires_at(UserId), + Pid <- [presence_pid(UserId)], + is_pid(Pid) + ]. + +-spec has_expires_at(integer()) -> boolean(). +has_expires_at(UserId) -> + case catch presence_cache:get(UserId) of + {ok, Presence} when is_map(Presence) -> + presence_carries_expiry(Presence); + _ -> + false + end. + +%% presence_cache:get/1 returns {ok, map()} | not_found, never a bare map, and +%% expires_at_ms/1 takes the timestamp value rather than the custom_status map. +-spec presence_carries_expiry(map()) -> boolean(). +presence_carries_expiry(Presence) -> + case maps:get(<<"custom_status">>, Presence, null) of + CustomStatus when is_map(CustomStatus) -> + expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, null)) =/= none; + _ -> + false + end. + +-spec local_presence_user_ids() -> [user_id()]. +local_presence_user_ids() -> + try presence_manager_shards:determine_count() of + {ShardCount, _Source} -> shard_user_ids(ShardCount) + catch + _Class:_Reason -> [] + end. + +-spec shard_user_ids(pos_integer()) -> [user_id()]. +shard_user_ids(ShardCount) -> + lists:append([ + presence_manager:get_shard_user_ids(Index) + || Index <- lists:seq(0, ShardCount - 1) + ]). + +-spec presence_pid(user_id()) -> pid() | undefined. +presence_pid(UserId) -> + process_registry:registry_whereis(process_registry:build_process_key(presence, UserId)). + +-spec schedule_reconciles([pid()], non_neg_integer(), non_neg_integer()) -> ok. +schedule_reconciles(Pids, Total, WindowMs) -> + Message = reconcile_message(), + _ = lists:foldl( + fun(Pid, Index) -> + _ = schedule_one(Pid, spread_ms(Index, Total, WindowMs), Message), + Index + 1 + end, + 0, + Pids + ), + ok. + +%% A single unreachable destination must not abort the sweep and leave the +%% operator unable to tell how many nudges were actually scheduled. +-spec schedule_one(pid(), non_neg_integer(), term()) -> ok. +schedule_one(Pid, Delay, Message) -> + try erlang:send_after(Delay, Pid, Message) of + _Ref -> ok + catch + _Class:_Reason -> ok + end. + +-spec spread_ms(non_neg_integer(), non_neg_integer(), non_neg_integer()) -> non_neg_integer(). +spread_ms(_Index, 0, _WindowMs) -> 0; +spread_ms(_Index, _Total, 0) -> 0; +spread_ms(Index, Total, WindowMs) -> (Index * WindowMs) div Total. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +presence_carries_expiry_detects_an_expiry_test() -> + ?assert( + presence_carries_expiry(#{ + <<"custom_status">> => #{<<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>} + }) + ), + ?assertNot(presence_carries_expiry(#{<<"custom_status">> => #{<<"text">> => <<"hi">>}})), + ?assertNot(presence_carries_expiry(#{<<"custom_status">> => null})), + ?assertNot(presence_carries_expiry(#{})). + +-define(LIVE_EXPIRES_AT, <<"2026-05-13T13:02:27.497Z">>). + +epoch_ms(Iso) -> + calendar:rfc3339_to_system_time(binary_to_list(Iso), [{unit, millisecond}]). + +rfc3339(EpochMs) -> + list_to_binary( + calendar:system_time_to_rfc3339(EpochMs, [{unit, millisecond}, {offset, "Z"}]) + ). + +expiry_ms() -> + epoch_ms(?LIVE_EXPIRES_AT). + +status(ExpiresAt) -> + #{ + <<"emoji_animated">> => false, + <<"emoji_name">> => null, + <<"expires_at">> => ExpiresAt, + <<"text">> => <<"brb">> + }. + +status_without_expiry() -> + #{<<"emoji_animated">> => false, <<"emoji_name">> => null, <<"text">> => <<"brb">>}. + +parser_anchors_on_the_unix_epoch_test() -> + ?assertEqual(0, epoch_ms(<<"1970-01-01T00:00:00.000Z">>)), + ?assertEqual(1500, epoch_ms(<<"1970-01-01T00:00:01.500Z">>)). + +parser_keeps_milliseconds_test() -> + ?assertEqual(497, expiry_ms() rem 1000). + +not_expired_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 60000)). + +expires_one_second_in_the_future_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual({ok, 1000}, remaining_ms(Status, expiry_ms() - 1000)), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1000)). + +boundary_is_expired_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual({ok, 0}, remaining_ms(Status, expiry_ms())), + ?assertEqual(null, clear_if_expired(Status, expiry_ms())). + +one_millisecond_before_boundary_is_kept_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1)). + +expired_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 1)), + ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 194 * 86400000)). + +absent_expires_at_is_kept_test() -> + Status = status_without_expiry(), + ?assertEqual(none, remaining_ms(Status, expiry_ms())), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms())). + +null_expires_at_is_kept_test() -> + Status = status(null), + ?assertEqual(none, remaining_ms(Status, expiry_ms())), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms())). + +malformed_expires_at_is_kept_test() -> + lists:foreach( + fun(Value) -> + Status = status(Value), + ?assertEqual(none, remaining_ms(Status, expiry_ms())), + ?assertEqual(Status, clear_if_expired(Status, expiry_ms())) + end, + [<<"">>, <<"not-a-date">>, <<"2026-13-45T99:99:99Z">>, 1747141347, [], #{}, true] + ). + +custom_status_null_test() -> + ?assertEqual(none, remaining_ms(null, expiry_ms())), + ?assertEqual(null, clear_if_expired(null, expiry_ms())). + +custom_status_absent_test() -> + ?assertEqual(none, remaining_ms(undefined, expiry_ms())), + ?assertEqual(null, clear_if_expired(undefined, expiry_ms())). + +non_map_custom_status_test() -> + ?assertEqual(null, clear_if_expired(<<"garbage">>, expiry_ms())), + ?assertEqual(null, clear_if_expired(123, expiry_ms())). + +non_integer_reference_time_keeps_status_test() -> + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual(Status, clear_if_expired(Status, not_a_time)). + +wakeup_none_without_expiry_test() -> + ?assertEqual(none, wakeup_delay(remaining_ms(status_without_expiry(), expiry_ms()))), + ?assertEqual(none, wakeup_delay(remaining_ms(null, expiry_ms()))). + +wakeup_none_when_already_expired_test() -> + ?assertEqual(none, wakeup_delay({ok, 0})), + ?assertEqual(none, wakeup_delay({ok, -1000})). + +wakeup_is_never_early_test() -> + {ok, DelayMs} = wakeup_delay({ok, 1000}), + ?assert(DelayMs > 1000), + ?assert(DelayMs =< 1000 + ?WAKEUP_JITTER_MS). + +wakeup_clamps_to_the_timer_limit_test() -> + {ok, DelayMs} = wakeup_delay({ok, 400 * ?MAX_TIMER_MS}), + ?assert(DelayMs > ?MAX_TIMER_MS), + ?assert(DelayMs =< ?MAX_TIMER_MS + ?WAKEUP_JITTER_MS). + +disabled_by_default_test() -> + application:unset_env(fluxer_gateway, ?ENABLED_KEY), + Status = status(?LIVE_EXPIRES_AT), + ?assertEqual(false, enabled()), + ?assertEqual(Status, clear_if_expired(Status)), + ?assertEqual(none, next_wakeup_ms(future_status())). + +enabled_clears_expired_and_arms_future_test() -> + Live = future_status(), + with_expiry_enabled(fun() -> + ?assertEqual(true, enabled()), + ?assertEqual(null, clear_if_expired(status(?LIVE_EXPIRES_AT))), + ?assertEqual(Live, clear_if_expired(Live)), + ?assertMatch({ok, _}, next_wakeup_ms(Live)) + end). + +future_status() -> + status(rfc3339(erlang:system_time(millisecond) + 3600000)). + +reconcile_message_is_a_gen_server_cast_test() -> + ?assertEqual({'$gen_cast', reconcile_flattened_presence}, reconcile_message()). + +spread_ms_test() -> + ?assertEqual(0, spread_ms(0, 0, 60000)), + ?assertEqual(0, spread_ms(5, 10, 0)), + ?assertEqual(0, spread_ms(0, 10, 60000)), + ?assertEqual(30000, spread_ms(5, 10, 60000)), + ?assertEqual(54000, spread_ms(9, 10, 60000)). + +repair_rejects_a_bad_window_test() -> + ?assertEqual(#{error => invalid_window_seconds}, repair(-1)), + ?assertEqual(#{error => invalid_window_seconds}, repair(<<"900">>)). + +repair_reports_what_it_scheduled_test() -> + Result = repair(0), + ?assert(is_integer(maps:get(scheduled, Result))), + ?assert(maps:get(scheduled, Result) >= 0), + ?assertEqual(0, maps:get(window_seconds, Result)), + ?assertEqual(enabled(), maps:get(enabled, Result)). + +schedule_reconciles_delivers_the_cast_test() -> + flush_mailbox(), + ok = schedule_reconciles([self()], 1, 0), + receive + Message -> ?assertEqual(reconcile_message(), Message) + after 1000 -> ?assert(false) + end. + +flush_mailbox() -> + receive + _Any -> flush_mailbox() + after 0 -> ok + end. + +with_expiry_enabled(Fun) -> + Previous = application:get_env(fluxer_gateway, ?ENABLED_KEY), + application:set_env(fluxer_gateway, ?ENABLED_KEY, true), + try + Fun() + after + restore_expiry_env(Previous) + end. + +restore_expiry_env(undefined) -> + application:unset_env(fluxer_gateway, ?ENABLED_KEY); +restore_expiry_env({ok, Value}) -> + application:set_env(fluxer_gateway, ?ENABLED_KEY, Value). + +-endif. diff --git a/fluxer_gateway/test/guild_member_list_tests.erl b/fluxer_gateway/test/guild_member_list_tests.erl index 2d609390c..dfbb11d59 100644 --- a/fluxer_gateway/test/guild_member_list_tests.erl +++ b/fluxer_gateway/test/guild_member_list_tests.erl @@ -118,6 +118,7 @@ run_broadcast_channel_list_dispatches_sync_immediately() -> <<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>}, <<"roles">> => [] }, + ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []), OldState = channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [ Member ]), @@ -144,6 +145,9 @@ run_broadcast_channel_list_fans_out_repeated_syncs() -> #{<<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>}}, #{<<"user">> => #{<<"id">> => <<"2">>, <<"username">> => <<"two">>}} ], + ok = guild_member_list_engine:bulk_load( + Ref, [{1, <<"one">>, [], true}, {2, <<"two">>, [], true}], [] + ), OldState = channel_list_state( Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), Members ), @@ -408,6 +412,7 @@ presence_delta_state(Ref) -> <<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>}, <<"roles">> => [] }, + ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []), channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [Member]). presence_map(Status, Mobile, CustomStatus) -> diff --git a/fluxer_gateway/test/guild_voice_move_tests.erl b/fluxer_gateway/test/guild_voice_move_tests.erl index a25ef606d..81629c7f2 100644 --- a/fluxer_gateway/test/guild_voice_move_tests.erl +++ b/fluxer_gateway/test/guild_voice_move_tests.erl @@ -271,6 +271,9 @@ guild_server_stub(StateData, TestPid) -> guild_server_stub_loop(StateData, TestPid) -> receive + {'$gen_call', From, {get_voice_guild_state}} -> + gen:reply(From, undefined), + guild_server_stub_loop(StateData, TestPid); {'$gen_call', From, {get_sessions}} -> gen:reply(From, StateData), guild_server_stub_loop(StateData, TestPid); diff --git a/fluxer_gateway/test/session_lifecycle_tests.erl b/fluxer_gateway/test/session_lifecycle_tests.erl index ca825e4f5..2da65a101 100644 --- a/fluxer_gateway/test/session_lifecycle_tests.erl +++ b/fluxer_gateway/test/session_lifecycle_tests.erl @@ -171,8 +171,8 @@ handle_resume_clamps_truncated_gap_in_replay_buffer_test() -> #{seq => 5, event => message_create, data => #{}} ] }), - {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0), - ?assertEqual([4, 5], [maps:get(seq, Event) || Event <- Missed]), + {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0), + ?assertEqual([4, 5], collect_dispatched_seqs(2)), ?assertEqual(self(), maps:get(socket_pid, State1)). handle_resume_clamps_skipped_event_hole_test() -> @@ -183,8 +183,8 @@ handle_resume_clamps_skipped_event_hole_test() -> #{seq => 5, event => message_create, data => #{}} ] }), - {reply, {ok, Missed, 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0), - ?assertEqual([3, 5], [maps:get(seq, Event) || Event <- Missed]). + {reply, {ok, [], 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0), + ?assertEqual([3, 5], collect_dispatched_seqs(2)). handle_resume_rejects_seq_ahead_of_current_test() -> State0 = resume_test_state(#{ @@ -211,8 +211,8 @@ handle_resume_accepts_seq_at_ack_seq_test() -> #{seq => 10, event => message_create, data => #{}} ] }), - {reply, {ok, Missed, 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0), - ?assertEqual([9, 10], [maps:get(seq, Event) || Event <- Missed]). + {reply, {ok, [], 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0), + ?assertEqual([9, 10], collect_dispatched_seqs(2)). handle_resume_accepts_contiguous_replay_buffer_test() -> State0 = resume_test_state(#{ @@ -223,8 +223,8 @@ handle_resume_accepts_contiguous_replay_buffer_test() -> #{seq => 5, event => message_create, data => #{}} ] }), - {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0), - ?assertEqual([3, 4, 5], [maps:get(seq, Event) || Event <- Missed]), + {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0), + ?assertEqual([3, 4, 5], collect_dispatched_seqs(3)), ?assertEqual(self(), maps:get(socket_pid, State1)). handle_resume_cancels_pending_resume_timer_test() -> @@ -445,6 +445,19 @@ resume_test_state(Overrides) -> Overrides ). +%% Small replay buffers are delivered inline (dispatched straight to the socket) rather +%% than returned in the resume reply, so these tests read the missed events back off +%% self()'s mailbox instead of the (now empty) reply list. +collect_dispatched_seqs(Count) -> + lists:sort([collect_dispatched_seq() || _ <- lists:seq(1, Count)]). + +collect_dispatched_seq() -> + receive + {dispatch, _Event, _Payload, Seq} -> Seq + after 200 -> + ?assert(false) + end. + fenced_terminate_releases_the_user_session_count_test() -> ok = session_abuse_protection:ensure_tables(), UserId = 900201,