fix(gateway): wire presence grace, request limits and voice sync (#3329)

fluxer_gateway/src/gateway/gateway_cluster_membership.erl

@@ -140,8 +140,10 @@ handle_cast(_Msg, State) ->
 handle_info({cluster_peers_changed, Peers}, State) ->
     {noreply, apply_discovered(normalize_nodes(Peers), State)};
 handle_info({nodeup, Node}, State) when is_atom(Node) ->
+    ok = presence_cache:handle_nodeup(Node),
     {noreply, maybe_add_member(Node, State)};
 handle_info({nodedown, Node}, State) when is_atom(Node) ->
+    ok = maybe_start_presence_grace(Node, State),
     {noreply, maybe_remove_member(Node, State)};
 handle_info(refresh_roles, State) ->
     State1 = reconcile_against_connected_nodes(State),
@@ -211,6 +213,13 @@ force_discovery_refresh_safely() ->
         exit:_ -> ok
     end.
 
+-spec maybe_start_presence_grace(node(), state()) -> ok.
+maybe_start_presence_grace(Node, #{members := Members}) ->
+    case lists:member(Node, Members) andalso Node =/= node() of
+        true -> presence_cache:handle_nodedown(Node);
+        false -> ok
+    end.
+
 -spec maybe_remove_member(node(), state()) -> state().
 maybe_remove_member(Node, #{members := Members} = State) ->
     case lists:member(Node, Members) andalso Node =/= node() of

fluxer_gateway/src/guild/guild.erl

@@ -516,6 +516,12 @@ terminate(Reason, State) when is_map(State) ->
         end,
         "ets_cleanup"
     ),
+    safe_cleanup(
+        fun() ->
+            cleanup_passive_sync_registry(State)
+        end,
+        "passive_sync_cleanup"
+    ),
     safe_cleanup(
         fun() ->
             cleanup_voice_server(State)
@@ -578,6 +584,15 @@ cleanup_per_guild_ets(State) ->
     safe_delete_ets(maps:get(viewable_channels_cache, State, undefined)),
     ok.
 
+-spec cleanup_passive_sync_registry(guild_state()) -> ok.
+cleanup_passive_sync_registry(#{id := GuildId} = State) when is_integer(GuildId) ->
+    maps:foreach(
+        fun(SessionId, _Session) -> passive_sync_registry:delete(SessionId, GuildId) end,
+        maps:get(sessions, State, #{})
+    );
+cleanup_passive_sync_registry(_State) ->
+    ok.
+
 -spec cleanup_voice_server(guild_state()) -> ok.
 cleanup_voice_server(State) ->
     case maps:get(voice_server_pid, State, undefined) of

fluxer_gateway/src/guild/guild_request_members.erl

@@ -180,9 +180,16 @@ normalize_nonce(_) ->
 
 -spec process_request(map(), pid(), session_state()) -> ok | {error, atom()}.
 process_request(Request, SocketPid, SessionState) ->
+    UserId = snowflake_id:parse_optional(maps:get(user_id, SessionState)),
+    case guild_request_members_filter:check_request_rate_limit(UserId) of
+        ok -> process_admitted_request(Request, UserId, SocketPid, SessionState);
+        {error, Reason} -> {error, Reason}
+    end.
+
+-spec process_admitted_request(map(), integer() | undefined, pid(), session_state()) ->
+    ok | {error, atom()}.
+process_admitted_request(Request, UserId, SocketPid, SessionState) ->
     #{guild_ids := ReqGuildIds, query := Query, limit := Limit, user_ids := UserIds} = Request,
-    UserIdBin = maps:get(user_id, SessionState),
-    UserId = snowflake_id:parse_optional(UserIdBin),
     IsBot = truthy(maps:get(bot, SessionState, false)),
     case guild_request_members_filter:enforce_single_guild_for_bots(IsBot, ReqGuildIds) of
         ok ->
@@ -248,6 +255,22 @@ dispatch_guild_requests(GIds, UID, Bot, Q, L, UI, Req, SP, SS) ->
     session_state()
 ) -> ok | {error, atom()}.
 process_single_guild(GId, UID, Bot, Q, L, UI, Req, _SP, SS) ->
+    case guild_request_members_filter:check_guild_request_rate_limit(GId) of
+        ok -> process_admitted_guild(GId, UID, Bot, Q, L, UI, Req, SS);
+        {error, Reason} -> {error, Reason}
+    end.
+
+-spec process_admitted_guild(
+    integer(),
+    integer() | undefined,
+    boolean(),
+    binary(),
+    non_neg_integer(),
+    [integer()],
+    map(),
+    session_state()
+) -> ok | {error, atom()}.
+process_admitted_guild(GId, UID, Bot, Q, L, UI, Req, SS) ->
     IsFull = guild_request_members_filter:is_full_member_list(Q, L, UI),
     case guild_request_members_filter:check_full_list_bot_rate_limit(IsFull, Bot, UID, GId) of
         ok ->
@@ -320,6 +343,22 @@ try_guild_or_keep_processed(GId, UID, Bot, Q, L, UI, Req, SS, Acc) ->
     session_state()
 ) -> boolean().
 try_guild(GId, UID, Bot, Q, L, UI, Req, SS) ->
+    case guild_request_members_filter:check_guild_request_rate_limit(GId) of
+        ok -> try_admitted_guild(GId, UID, Bot, Q, L, UI, Req, SS);
+        {error, _Reason} -> false
+    end.
+
+-spec try_admitted_guild(
+    integer(),
+    integer() | undefined,
+    boolean(),
+    binary(),
+    non_neg_integer(),
+    [integer()],
+    map(),
+    session_state()
+) -> boolean().
+try_admitted_guild(GId, UID, Bot, Q, L, UI, Req, SS) ->
     IsFull = guild_request_members_filter:is_full_member_list(Q, L, UI),
     case guild_request_members_filter:check_full_list_bot_rate_limit(IsFull, Bot, UID, GId) of
         ok ->

fluxer_gateway/src/guild/guild_request_members_filter.erl

@@ -16,6 +16,10 @@
 
 -export_type([session_state/0]).
 
+-ifdef(TEST).
+-export([prune_expired_entries/2]).
+-endif.
+
 -define(REQUEST_MEMBERS_RATE_LIMIT_TABLE, guild_request_members_rate_limit).
 -define(REQUEST_MEMBERS_RATE_LIMIT_WINDOW_MS, 10000).
 -define(REQUEST_MEMBERS_RATE_LIMIT_MAX_EVENTS, 12).
@@ -24,6 +28,7 @@
 -define(REQUEST_MEMBERS_GUILD_RATE_LIMIT_MAX_EVENTS, 40).
 -define(FULL_LIST_BOT_RATE_LIMIT_TABLE, guild_request_members_bot_full_list_rate_limit).
 -define(FULL_LIST_BOT_RATE_LIMIT_WINDOW_MS, 30000).
+-define(RATE_LIMIT_CLEANUP_INTERVAL_MS, 60000).
 
 -type session_state() :: map().
 
@@ -106,7 +111,7 @@ check_full_list_bot_rate_limit(true, true, _UserId, _GuildId) ->
 -spec check_bot_rate_limit_window(integer(), integer()) ->
     ok | {rate_limited, non_neg_integer()}.
 check_bot_rate_limit_window(UserId, GuildId) ->
-    ensure_ets_table(?FULL_LIST_BOT_RATE_LIMIT_TABLE),
+    ensure_ets_table(?FULL_LIST_BOT_RATE_LIMIT_TABLE, ?FULL_LIST_BOT_RATE_LIMIT_WINDOW_MS),
     Now = erlang:system_time(millisecond),
     Key = {UserId, GuildId},
     Window = ?FULL_LIST_BOT_RATE_LIMIT_WINDOW_MS,
@@ -120,25 +125,40 @@ check_bot_rate_limit_window(UserId, GuildId) ->
 
 -spec check_request_rate_limit(integer() | undefined) -> ok | {error, atom()}.
 check_request_rate_limit(UserId) when is_integer(UserId), UserId > 0 ->
-    ensure_ets_table(?REQUEST_MEMBERS_RATE_LIMIT_TABLE),
-    check_sliding_window(
-        ?REQUEST_MEMBERS_RATE_LIMIT_TABLE,
-        UserId,
-        ?REQUEST_MEMBERS_RATE_LIMIT_WINDOW_MS,
-        ?REQUEST_MEMBERS_RATE_LIMIT_MAX_EVENTS
-    );
+    case gateway_handler_rate_limit:rate_limits_disabled() of
+        true ->
+            ok;
+        false ->
+            ensure_ets_table(
+                ?REQUEST_MEMBERS_RATE_LIMIT_TABLE, ?REQUEST_MEMBERS_RATE_LIMIT_WINDOW_MS
+            ),
+            check_sliding_window(
+                ?REQUEST_MEMBERS_RATE_LIMIT_TABLE,
+                UserId,
+                ?REQUEST_MEMBERS_RATE_LIMIT_WINDOW_MS,
+                ?REQUEST_MEMBERS_RATE_LIMIT_MAX_EVENTS
+            )
+    end;
 check_request_rate_limit(_) ->
     {error, invalid_session}.
 
 -spec check_guild_request_rate_limit(integer()) -> ok | {error, atom()}.
 check_guild_request_rate_limit(GuildId) when is_integer(GuildId), GuildId > 0 ->
-    ensure_ets_table(?REQUEST_MEMBERS_GUILD_RATE_LIMIT_TABLE),
-    check_sliding_window(
-        ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_TABLE,
-        GuildId,
-        ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_WINDOW_MS,
-        ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_MAX_EVENTS
-    );
+    case gateway_handler_rate_limit:rate_limits_disabled() of
+        true ->
+            ok;
+        false ->
+            ensure_ets_table(
+                ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_TABLE,
+                ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_WINDOW_MS
+            ),
+            check_sliding_window(
+                ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_TABLE,
+                GuildId,
+                ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_WINDOW_MS,
+                ?REQUEST_MEMBERS_GUILD_RATE_LIMIT_MAX_EVENTS
+            )
+    end;
 check_guild_request_rate_limit(_) ->
     {error, invalid_guild_id}.
 
@@ -166,30 +186,56 @@ check_window_count(Table, Key, Now, Recent, _MaxEvents) ->
     ets:insert(Table, {Key, [Now | Recent]}),
     ok.
 
--spec ensure_ets_table(atom()) -> ok.
-ensure_ets_table(Name) ->
+-spec ensure_ets_table(atom(), pos_integer()) -> ok.
+ensure_ets_table(Name, WindowMs) ->
     case ets:whereis(Name) of
         undefined ->
-            create_ets_table(Name);
+            create_ets_table(Name, WindowMs);
         _ ->
             ok
     end.
 
--spec create_ets_table(atom()) -> ok.
-create_ets_table(Name) ->
+-spec create_ets_table(atom(), pos_integer()) -> ok.
+create_ets_table(Name, WindowMs) ->
     try
-        _ = ets:new(Name, [
-            named_table,
-            public,
-            set,
-            {read_concurrency, true},
-            {write_concurrency, true}
-        ]),
+        Tid = ets:new(
+            Name,
+            [
+                named_table,
+                public,
+                set,
+                {read_concurrency, true},
+                {write_concurrency, true}
+            ] ++ guild_ets_utils:heir_options()
+        ),
+        _ = spawn(fun() -> cleanup_loop(Tid, WindowMs) end),
         ok
     catch
         error:badarg -> ok
     end.
 
+-spec cleanup_loop(ets:table(), pos_integer()) -> ok.
+cleanup_loop(Table, WindowMs) ->
+    ok = gateway_retry_timer:wait(?RATE_LIMIT_CLEANUP_INTERVAL_MS),
+    case prune_expired_entries(Table, WindowMs) of
+        ok -> cleanup_loop(Table, WindowMs);
+        gone -> ok
+    end.
+
+-spec prune_expired_entries(ets:table(), pos_integer()) -> ok | gone.
+prune_expired_entries(Table, WindowMs) ->
+    Cutoff = erlang:system_time(millisecond) - WindowMs,
+    try
+        _ = ets:select_delete(Table, [
+            {{'_', []}, [], [true]},
+            {{'_', '$1'}, [{is_list, '$1'}, {'=<', {hd, '$1'}, Cutoff}], [true]},
+            {{'_', '$1'}, [{is_integer, '$1'}, {'=<', '$1', Cutoff}], [true]}
+        ]),
+        ok
+    catch
+        error:badarg -> gone
+    end.
+
 -spec dispatch_full_list_rate_limited(
     session_state(), integer(), map(), non_neg_integer()
 ) -> ok.

fluxer_gateway/src/guild/guild_state.erl

@@ -155,10 +155,7 @@ post_update_role(guild_role_delete, EventData, OldState, NewState) ->
     guild_state().
 resync_roles_after_permission_change(RoleIds, OldState, NewState) ->
     Recomputed = guild_state_roles:recompute_visibility_for_roles(RoleIds, OldState, NewState),
-    lists:foreach(
-        fun(RoleId) -> guild_voice_permission_sync:sync_users_with_role(RoleId, Recomputed) end,
-        RoleIds
-    ),
+    ok = guild_voice_permission_sync:sync_roles_after_change(RoleIds, OldState, Recomputed),
     Recomputed.
 
 -spec post_update_channel(event(), event_data(), guild_state(), guild_state()) -> guild_state().

fluxer_gateway/src/guild/passive_sync_registry.erl

@@ -7,8 +7,7 @@
     init/0,
     store/3,
     lookup/2,
-    delete/2,
-    delete_all_for_session/1
+    delete/2
 ]).
 
 -ifdef(TEST).
@@ -72,12 +71,6 @@ delete(SessionId, GuildId) ->
     ets:delete(?TABLE, Key),
     ok.
 
--spec delete_all_for_session(session_id()) -> ok.
-delete_all_for_session(SessionId) ->
-    ensure_table(),
-    ets:match_delete(?TABLE, {{SessionId, '_'}, '_'}),
-    ok.
-
 -spec ensure_table() -> ok.
 ensure_table() ->
     case ets:whereis(?TABLE) of
@@ -152,38 +145,6 @@ delete_removes_entry_test() ->
     ),
     cleanup_table().
 
-delete_all_for_session_removes_all_guilds_test() ->
-    cleanup_table(),
-    ok = init(),
-    SessionId = <<"session_1">>,
-    OtherSessionId = <<"session_2">>,
-    State1 = make_passive_state(<<"ch1">>, <<"msg1">>),
-    State2 = make_passive_state(<<"ch2">>, <<"msg2">>),
-    OtherState = make_passive_state(<<"ch3">>, <<"msg3">>),
-    ok = store(SessionId, 100, State1),
-    ok = store(SessionId, 200, State2),
-    ok = store(OtherSessionId, 100, OtherState),
-    ok = delete_all_for_session(SessionId),
-    Empty = empty_passive_state(),
-    ?assertEqual(Empty, lookup(SessionId, 100)),
-    ?assertEqual(Empty, lookup(SessionId, 200)),
-    ?assertEqual(OtherState, lookup(OtherSessionId, 100)),
-    cleanup_table().
-
-make_passive_state(Ch, Msg) ->
-    #{
-        previous_passive_updates => #{Ch => Msg},
-        previous_passive_channel_versions => #{},
-        previous_passive_voice_states => #{}
-    }.
-
-empty_passive_state() ->
-    #{
-        previous_passive_updates => #{},
-        previous_passive_channel_versions => #{},
-        previous_passive_voice_states => #{}
-    }.
-
 store_overwrites_existing_test() ->
     cleanup_table(),
     ok = init(),

fluxer_gateway/src/guild/voice/guild_voice_permission_sync.erl

@@ -6,8 +6,7 @@
 -export([
     sync_user_voice_permissions/2,
     sync_all_voice_permissions_for_channels/2,
-    sync_users_with_role/2,
-    maybe_sync_permissions_on_role_update/2,
+    sync_roles_after_change/3,
     maybe_sync_permissions_on_member_update/2
 ]).
 
@@ -96,14 +95,6 @@ sync_channel_voice_state_permissions(GuildId, VoiceState, State) ->
         UserId -> sync_voice_state_permissions(GuildId, UserId, VoiceState, State)
     end.
 
--spec maybe_sync_permissions_on_role_update(map(), guild_state()) -> ok.
-maybe_sync_permissions_on_role_update(RoleUpdate, State) ->
-    RoleId = snowflake_id:parse_optional(maps:get(<<"id">>, RoleUpdate, undefined)),
-    case RoleId of
-        undefined -> ok;
-        RoleIdInt -> sync_users_with_role(RoleIdInt, State)
-    end.
-
 -spec maybe_sync_permissions_on_member_update(map(), guild_state()) -> ok.
 maybe_sync_permissions_on_member_update(MemberUpdate, State) ->
     UserId = get_member_user_id(MemberUpdate),
@@ -241,54 +232,66 @@ enforce_voice_permissions_in_livekit(
             ok
     end.
 
--spec sync_users_with_role(integer(), guild_state()) -> ok.
-sync_users_with_role(RoleId, State) ->
-    VoiceStates = guild_voice_lifecycle:authoritative_voice_states(State),
+-spec sync_roles_after_change([integer()], guild_state(), guild_state()) -> ok.
+sync_roles_after_change([], _OldState, _State) ->
+    ok;
+sync_roles_after_change(RoleIds, OldState, State) ->
     case state_guild_id(State) of
         undefined ->
             ok;
         GuildId ->
-            do_sync_users_with_role(GuildId, RoleId, VoiceStates, State)
+            Affected = affected_role_users(GuildId, RoleIds, OldState, State),
+            sync_affected_voice_states(GuildId, Affected, State)
     end.
 
--spec do_sync_users_with_role(integer(), integer(), map(), guild_state()) -> ok.
-do_sync_users_with_role(GuildId, RoleId, VoiceStates, State) ->
-    RoleUsers = user_ids_with_role(RoleId, State),
+-spec sync_affected_voice_states(integer(), all | sets:set(user_id()), guild_state()) -> ok.
+sync_affected_voice_states(GuildId, Affected, State) ->
     maps:foreach(
         fun(_ConnId, VoiceState) ->
-            maybe_sync_role_voice_state(GuildId, RoleUsers, VoiceState, State)
+            maybe_sync_affected_voice_state(GuildId, Affected, VoiceState, State)
         end,
-        VoiceStates
-    ),
-    ok.
+        guild_voice_lifecycle:authoritative_voice_states(State)
+    ).
 
--spec maybe_sync_role_voice_state(integer(), sets:set(user_id()), voice_state(), guild_state()) ->
-    ok.
-maybe_sync_role_voice_state(GuildId, RoleUsers, VoiceState, State) ->
-    UserId = voice_state_utils:voice_state_user_id(VoiceState),
-    case UserId of
-        undefined ->
-            ok;
-        _ ->
-            sync_voice_state_permissions_if_role_matches(
-                GuildId, RoleUsers, UserId, VoiceState, State
+-spec affected_role_users(integer(), [integer()], guild_state(), guild_state()) ->
+    all | sets:set(user_id()).
+affected_role_users(GuildId, RoleIds, OldState, State) ->
+    case lists:member(GuildId, RoleIds) of
+        true ->
+            all;
+        false ->
+            OldIndex = guild_data_index:member_role_index(maps:get(data, OldState, #{})),
+            NewIndex = guild_data_index:member_role_index(maps:get(data, State, #{})),
+            sets:from_list(
+                lists:append([
+                    maps:keys(maps:get(RoleId, OldIndex, #{})) ++
+                        maps:keys(maps:get(RoleId, NewIndex, #{}))
+                 || RoleId <- RoleIds
+                ])
             )
     end.
 
--spec sync_voice_state_permissions_if_role_matches(
-    integer(), sets:set(user_id()), user_id(), voice_state(), guild_state()
+-spec maybe_sync_affected_voice_state(
+    integer(), all | sets:set(user_id()), voice_state(), guild_state()
 ) -> ok.
-sync_voice_state_permissions_if_role_matches(GuildId, RoleUsers, UserId, VoiceState, State) ->
-    case sets:is_element(UserId, RoleUsers) of
+maybe_sync_affected_voice_state(GuildId, Affected, VoiceState, State) ->
+    case voice_state_utils:voice_state_user_id(VoiceState) of
+        undefined -> ok;
+        UserId -> sync_if_affected(GuildId, UserId, Affected, VoiceState, State)
+    end.
+
+-spec sync_if_affected(
+    integer(), user_id(), all | sets:set(user_id()), voice_state(), guild_state()
+) -> ok.
+sync_if_affected(GuildId, UserId, Affected, VoiceState, State) ->
+    case is_affected(UserId, Affected) of
         true -> sync_voice_state_permissions(GuildId, UserId, VoiceState, State);
         false -> ok
     end.
 
--spec user_ids_with_role(integer(), guild_state()) -> sets:set(user_id()).
-user_ids_with_role(RoleId, State) ->
-    Data = maps:get(data, State, #{}),
-    RoleIndex = guild_data_index:member_role_index(Data),
-    sets:from_list(maps:keys(maps:get(RoleId, RoleIndex, #{}))).
+-spec is_affected(user_id(), all | sets:set(user_id())) -> boolean().
+is_affected(_UserId, all) -> true;
+is_affected(UserId, Affected) -> sets:is_element(UserId, Affected).
 
 -spec get_member_user_id(map()) -> user_id() | undefined.
 get_member_user_id(MemberUpdate) ->
@@ -377,7 +380,7 @@ maybe_sync_permissions_on_member_update_no_role_change_test() ->
     },
     ?assertEqual(ok, maybe_sync_permissions_on_member_update(MemberUpdate, State)).
 
-maybe_sync_permissions_on_role_update_uses_role_index_test() ->
+sync_roles_after_change_uses_role_index_test() ->
     TestFun = make_sync_test_fun(),
     GuildId = 42,
     RoleId = 999,
@@ -408,9 +411,7 @@ maybe_sync_permissions_on_role_update_uses_role_index_test() ->
             GuildId, RoleId, OtherRoleId, UserId, OtherUserId, ChannelId, Permissions
         )
     },
-    ok = maybe_sync_permissions_on_role_update(
-        #{<<"id">> => integer_to_binary(RoleId)}, State
-    ),
+    ok = sync_roles_after_change([RoleId], State, State),
     receive
         {synced, GuildId, ChannelId, UserId, <<"conn1">>, _Perms} -> ok
     after 200 ->