pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-07 19:22:14 +09:00
Code Issues Packages Projects Releases Wiki Activity

perf(gateway): speed up reconnects and pin guilds to nodes (#3107)

Browse Source
This commit is contained in:
Hampus
2026-10-02 14:02:22 +02:00
committed by GitHub
parent ee2d11ee0a
commit a9265cbb39
48 changed files with 5459 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_gateway/scripts/docker_entrypoint.sh
+15
View File
@@ -54,6 +54,21 @@ is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MIN:-}" || FLUXER_ERLANG_SCHEDULERS_
is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MAX:-}" || FLUXER_ERLANG_SCHEDULERS_MAX=16
: "${FLUXER_ERLANG_NODE_NAME:[email protected]}"
: "${FLUXER_ERLANG_DIST_PORT:=8081}"
node_name_file=/opt/fluxer_gateway/node_name
if [ -n "${FLUXER_ERLANG_NODE_NAME_PREFIX:-}" ]; then
case "$FLUXER_ERLANG_NODE_NAME_PREFIX" in
*[!A-Za-z0-9_]*)
echo 'FLUXER_ERLANG_NODE_NAME_PREFIX may only contain letters, digits and underscores.' >&2
exit 1
;;
esac
node_nonce="$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n')"
FLUXER_ERLANG_NODE_NAME="${FLUXER_ERLANG_NODE_NAME_PREFIX}_${node_nonce}@${FLUXER_ERLANG_NODE_NAME#*@}"
printf '%s\n' "$FLUXER_ERLANG_NODE_NAME" >"$node_name_file"
else
rm -f "$node_name_file"
fi
if [ -z "${FLUXER_ERLANG_COOKIE:-}" ]; then
echo 'FLUXER_ERLANG_COOKIE is required.' >&2
fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh
+5
View File
@@ -13,4 +13,9 @@ eval | console | remote_console | remote | remsh | console_clean | console_boot
esac
script_dir="$(CDPATH='' cd "$(dirname "$0")" && pwd -P)"
node_name_file="$script_dir/../node_name"
if [ -r "$node_name_file" ]; then
FLUXER_ERLANG_NODE_NAME="$(cat "$node_name_file")"
export FLUXER_ERLANG_NODE_NAME
fi
exec "$script_dir/fluxer_gateway.real" "$@"
fluxer_gateway/src/gateway/fluxer_gateway_app.erl
+1
View File
@@ -16,6 +16,7 @@ start(_StartType, _StartArgs) ->
-spec init_subsystems() -> ok.
init_subsystems() ->
_ = fluxer_gateway_env:load(),
ok = gateway_guild_pin_keeper:apply_boot_pins(),
gateway_compress:init(),
gateway_cluster_metrics:init(),
process_registry:init(),
fluxer_gateway/src/gateway/fluxer_gateway_config.erl
+46 -1
View File
@@ -114,6 +114,17 @@ env_gateway_base_config() ->
<<"gateway_nats_rpc_max_handlers">> => env_int(
"FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS", 512
),
<<"nats_rpc_enabled">> => env_bool("FLUXER_GATEWAY_NATS_RPC_ENABLED", true),
<<"pinned_guild_ids">> => env_optional_binary("FLUXER_GATEWAY_PINNED_GUILD_IDS"),
<<"guild_pin_keeper_beam">> => env_optional_binary(
"FLUXER_GATEWAY_GUILD_PIN_KEEPER_BEAM"
),
<<"guild_pin_keeper_beam_md5">> => env_optional_binary(
"FLUXER_GATEWAY_GUILD_PIN_KEEPER_BEAM_MD5"
),
<<"guild_pin_keeper_base_md5s">> => env_optional_binary(
"FLUXER_GATEWAY_GUILD_PIN_KEEPER_BASE_MD5S"
),
<<"gateway_http_failure_threshold">> => env_int(
"FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD", 6
),
@@ -142,6 +153,7 @@ build_config(RawConfig) ->
build_sharding_config(Service),
build_http_config(Service),
build_cluster_config(Service, Public),
build_pinned_node_config(Service),
build_misc_config(Service)
]).
@@ -195,7 +207,7 @@ build_sharding_config(Service) ->
guild_counts_cache_shards => get_optional_int(Service, <<"guild_counts_cache_shards">>),
guild_shards => get_optional_int(Service, <<"guild_shards">>),
session_shards => get_optional_int(Service, <<"session_shards">>),
session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 1024)
session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 8192)
}.
-spec build_http_config(map()) -> config().
@@ -233,6 +245,39 @@ build_cluster_config(Service, Public) ->
)
}.
-spec build_pinned_node_config(map()) -> config().
build_pinned_node_config(Service) ->
#{
nats_rpc_enabled => get_bool(Service, <<"nats_rpc_enabled">>, true),
pinned_guild_ids => parse_guild_id_list(
get_optional_binary(Service, <<"pinned_guild_ids">>)
),
guild_pin_keeper_beam => optional_string(
get_optional_binary(Service, <<"guild_pin_keeper_beam">>)
),
guild_pin_keeper_beam_md5 => get_optional_binary(
Service, <<"guild_pin_keeper_beam_md5">>
),
guild_pin_keeper_base_md5s => get_optional_binary(
Service, <<"guild_pin_keeper_base_md5s">>
)
}.
-spec parse_guild_id_list(binary() | undefined) -> [pos_integer()].
parse_guild_id_list(undefined) ->
[];
parse_guild_id_list(Bin) when is_binary(Bin) ->
lists:usort([parse_guild_id(Token) || Token <- string:lexemes(binary_to_list(Bin), ", ")]).
-spec parse_guild_id(string()) -> pos_integer().
parse_guild_id(Token) ->
try list_to_integer(Token) of
Id when Id > 0 -> Id;
_ -> erlang:error({invalid_pinned_guild_id, Token})
catch
error:badarg -> erlang:error({invalid_pinned_guild_id, Token})
end.
-spec build_misc_config(map()) -> config().
build_misc_config(Service) ->
#{
fluxer_gateway/src/gateway/fluxer_gateway_sup.erl
+3 -2
View File
@@ -43,7 +43,8 @@ common_children() ->
[
child_spec(gateway_dispatch_relay, gateway_dispatch_relay),
child_spec(gateway_periodic_gc, gateway_periodic_gc),
child_spec(process_health_watchdog, process_health_watchdog)
child_spec(process_health_watchdog, process_health_watchdog),
child_spec(gateway_stall_monitor, gateway_stall_monitor)
].
-spec role_children(atom()) -> [supervisor:child_spec()].
@@ -95,7 +96,7 @@ role_specs(guilds, _Role) ->
child_spec(guild_counts_cache, guild_counts_cache),
child_spec(guild_manager, guild_manager),
child_spec(voice_state_counts_sync, voice_state_counts_sync)
];
] ++ gateway_guild_pin_keeper:child_specs();
role_specs(calls, Role) ->
[child_spec(call_manager, call_manager)] ++ calls_voice_state_counts_sync_children(Role);
role_specs(push, _Role) ->
fluxer_gateway/src/gateway/gateway_guild_pin_keeper.erl
+1143
View File
File diff suppressed because it is too large Load Diff
fluxer_gateway/src/gateway/gateway_nats_pool.erl
+23 -1
View File
@@ -112,13 +112,17 @@ init([]) ->
slots => #{},
monitors => #{},
connecting => #{},
rpc_enabled => true,
rpc_enabled => nats_rpc_enabled(),
subs => #{},
handler_count => 0,
handler_refs => #{},
max_handlers => gateway_nats_pool_conn:max_handlers()
}}.
-spec nats_rpc_enabled() -> boolean().
nats_rpc_enabled() ->
fluxer_gateway_env:get(nats_rpc_enabled) =/= false.
-spec handle_call(term(), gen_server:from(), map()) -> {reply, term(), map()}.
handle_call(get_pool_status, _From, State) ->
{reply, build_pool_status(State), State};
@@ -571,6 +575,24 @@ legacy_connect_timeout_ignores_tokened_slot_worker_test() ->
exit(Pid, kill)
end.
pool_keeps_rpc_unsubscribed_when_switched_off_test() ->
persistent_term:put({fluxer_gateway, runtime_config}, #{nats_rpc_enabled => false}),
Parent = self(),
try
Pid = spawn(fun() -> Parent ! {self(), init([])} end),
{ok, State} =
receive
{Pid, Reply} -> Reply
after 5000 -> timeout
end,
?assertEqual(false, maps:get(rpc_enabled, State)),
?assertEqual(State, handle_ready(self(), false, #{0 => self()}, State))
after
persistent_term:erase(?PERSISTENT_TERM_KEY),
persistent_term:erase(?REPLY_FAILURE_KEY),
persistent_term:erase({fluxer_gateway, runtime_config})
end.
wait_forever() ->
receive
stop -> ok
fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl
+1 -1
View File
@@ -169,7 +169,7 @@ do_subscribe(#{conn := Conn, rpc_enabled := RpcEnabled} = State) when
false ->
logger:info(
"Gateway NATS RPC connected but subscription disabled"
" (GATEWAY_NATS_RPC_ENABLED=false)"
" (FLUXER_GATEWAY_NATS_RPC_ENABLED=false)"
),
State;
_ ->
fluxer_gateway/src/gateway/gateway_node_router.erl
+152 -4
View File
@@ -7,6 +7,8 @@
owner_node/1, owner_node/2,
owner_node_result/2,
select_owner_node/2,
select_guild_owner_node/2,
pinned_guild_owner/1,
active_nodes/0, active_nodes/1,
is_ready/0,
is_draining/0
@@ -30,11 +32,50 @@ owner_node(Key, Role) ->
-spec owner_node_result(term(), atom()) -> {ok, node()} | {error, {no_active_nodes, atom()}}.
owner_node_result(Key, Role) ->
case pinned_owner(Key, Role) of
{ok, Pinned} -> {ok, Pinned};
none -> hashed_owner_node_result(Key, Role)
end.
-spec hashed_owner_node_result(term(), atom()) ->
{ok, node()} | {error, {no_active_nodes, atom()}}.
hashed_owner_node_result(Key, Role) ->
case active_nodes(Role) of
[] -> fallback_owner_for_role(Role);
Nodes -> {ok, select_owner_node(Key, Nodes)}
end.
-spec select_guild_owner_node(term(), [node()]) -> node().
select_guild_owner_node(GuildId, Nodes) ->
case pinned_guild_owner(GuildId) of
{ok, Pinned} -> Pinned;
none -> select_owner_node(GuildId, Nodes)
end.
-spec pinned_owner(term(), atom()) -> {ok, node()} | none.
pinned_owner(Key, guilds) ->
pinned_guild_owner(Key);
pinned_owner(_Key, _Role) ->
none.
-spec pinned_guild_owner(term()) -> {ok, node()} | none.
pinned_guild_owner(GuildId) when is_integer(GuildId) ->
case application:get_env(fluxer_gateway, guild_owner_pins) of
{ok, #{GuildId := Node}} when is_atom(Node) -> live_pin(Node, nodes());
_ -> none
end;
pinned_guild_owner(_GuildId) ->
none.
-spec live_pin(node(), [node()]) -> {ok, node()} | none.
live_pin(Node, _Connected) when Node =:= node() ->
{ok, Node};
live_pin(Node, Connected) ->
case lists:member(Node, Connected) of
true -> {ok, Node};
false -> none
end.
-spec select_owner_node(term(), [node()]) -> node().
select_owner_node(_Key, []) ->
node();
@@ -67,13 +108,27 @@ role_active_nodes(Role, RoleMembers) when is_map(RoleMembers) ->
Monoliths0 = maps:get(all, RoleMembers, []),
Specific = filter_atoms(Specific0),
Monoliths = filter_atoms(Monoliths0),
intersect_current_members(lists:usort(Specific ++ Monoliths));
Active = intersect_current_members(lists:usort(Specific ++ Monoliths)),
exclude_pinned_only_nodes(Role, Active);
role_active_nodes(Role, _RoleMembers) ->
exclude_pinned_only_nodes(Role, local_role_nodes(Role)).
-spec local_role_nodes(atom()) -> [node()].
local_role_nodes(Role) ->
case fluxer_gateway_sup:role_enabled(Role) of
true -> [node()];
false -> []
end.
-spec exclude_pinned_only_nodes(atom(), [node()]) -> [node()].
exclude_pinned_only_nodes(guilds, Nodes) ->
case application:get_env(fluxer_gateway, guild_pinned_only_nodes) of
{ok, [_ | _] = Excluded} -> [Node || Node <- Nodes, not lists:member(Node, Excluded)];
_ -> Nodes
end;
exclude_pinned_only_nodes(_Role, Nodes) ->
Nodes.
-spec filter_atoms(term()) -> [atom()].
filter_atoms(List) when is_list(List) ->
[X || X <- List, is_atom(X)];
@@ -91,9 +146,9 @@ intersect_current_members(RoleNodes) ->
-spec fallback_owner_for_role(atom()) -> {ok, node()} | {error, {no_active_nodes, atom()}}.
fallback_owner_for_role(Role) ->
case fluxer_gateway_sup:role_enabled(Role) of
true -> {ok, node()};
false -> {error, {no_active_nodes, Role}}
case exclude_pinned_only_nodes(Role, local_role_nodes(Role)) of
[Self] -> {ok, Self};
_ -> {error, {no_active_nodes, Role}}
end.
-spec is_ready() -> boolean().
@@ -246,6 +301,99 @@ is_draining_reads_persistent_term_test() ->
?assert(is_draining()),
persistent_term:erase({fluxer_gateway, draining}).
-define(PINNED_GUILD, 1100000000000000001).
with_guild_members(Fun) ->
persistent_term:put(
{gateway_cluster_membership, members}, [node(), 'g1@h', 'g2@h', 'g3@h']
),
persistent_term:put({gateway_cluster_membership, members_by_role}, #{
guilds => [node(), 'g1@h', 'g2@h', 'g3@h'],
calls => ['g1@h', 'g2@h']
}),
try
Fun()
after
application:unset_env(fluxer_gateway, guild_owner_pins),
application:unset_env(fluxer_gateway, guild_pinned_only_nodes),
persistent_term:erase({gateway_cluster_membership, members}),
persistent_term:erase({gateway_cluster_membership, members_by_role})
end.
hashed_owners(Keys, Role) ->
Nodes = active_nodes(Role),
[{ok, select_owner_node(Key, Nodes)} || Key <- Keys].
unset_pins_keep_hashed_owners_test() ->
with_guild_members(fun() ->
Keys = [?PINNED_GUILD | lists:seq(1, 500)],
?assertEqual(hashed_owners(Keys, guilds), [owner_node_result(K, guilds) || K <- Keys]),
?assertEqual(hashed_owners(Keys, calls), [owner_node_result(K, calls) || K <- Keys])
end).
pin_to_self_applies_to_guilds_role_only_test() ->
with_guild_members(fun() ->
[Hashed] = hashed_owners([?PINNED_GUILD], calls),
Nodes = active_nodes(guilds),
Release = select_owner_node(?PINNED_GUILD, Nodes),
application:set_env(fluxer_gateway, guild_owner_pins, #{?PINNED_GUILD => node()}),
?assertEqual({ok, node()}, owner_node_result(?PINNED_GUILD, guilds)),
?assertEqual(Hashed, owner_node_result(?PINNED_GUILD, calls)),
?assertEqual(Release, select_owner_node(?PINNED_GUILD, Nodes)),
?assertEqual(hashed_owners([7], guilds), [owner_node_result(7, guilds)])
end).
pin_to_disconnected_node_falls_back_to_hashing_test() ->
with_guild_members(fun() ->
[Hashed] = hashed_owners([?PINNED_GUILD], guilds),
application:set_env(fluxer_gateway, guild_owner_pins, #{?PINNED_GUILD => 'pin_x@h'}),
?assertEqual(Hashed, owner_node_result(?PINNED_GUILD, guilds)),
?assertEqual(none, pinned_guild_owner(?PINNED_GUILD))
end).
live_pin_requires_a_connected_node_test() ->
?assertEqual({ok, 'pin_x@h'}, live_pin('pin_x@h', ['g1@h', 'pin_x@h'])),
?assertEqual(none, live_pin('pin_x@h', ['g1@h'])),
?assertEqual({ok, node()}, live_pin(node(), [])).
pinned_only_node_owns_no_hashed_guilds_test() ->
with_guild_members(fun() ->
Keys = lists:seq(1, 2000),
Before = [K || K <- Keys, owner_node_result(K, guilds) =:= {ok, node()}],
application:set_env(fluxer_gateway, guild_pinned_only_nodes, [node()]),
After = [K || K <- Keys, owner_node_result(K, guilds) =:= {ok, node()}],
?assertNotEqual([], Before),
?assertEqual([], After),
?assertEqual(['g1@h', 'g2@h', 'g3@h'], active_nodes(guilds)),
?assertEqual(['g1@h', 'g2@h'], active_nodes(calls))
end).
pinned_only_node_without_peers_owns_only_its_pins_test() ->
persistent_term:put({fluxer_gateway, runtime_config}, #{gateway_role => guilds}),
persistent_term:erase({gateway_cluster_membership, members_by_role}),
application:set_env(fluxer_gateway, guild_pinned_only_nodes, [node()]),
application:set_env(fluxer_gateway, guild_owner_pins, #{?PINNED_GUILD => node()}),
try
?assertEqual([], active_nodes(guilds)),
?assertEqual({error, {no_active_nodes, guilds}}, owner_node_result(5, guilds)),
?assertEqual({ok, node()}, owner_node_result(?PINNED_GUILD, guilds))
after
application:unset_env(fluxer_gateway, guild_owner_pins),
application:unset_env(fluxer_gateway, guild_pinned_only_nodes),
persistent_term:erase({fluxer_gateway, runtime_config})
end.
select_guild_owner_node_honours_live_pins_test() ->
with_guild_members(fun() ->
Targets = ['g1@h', 'g2@h'],
Hashed = select_owner_node(?PINNED_GUILD, Targets),
?assertEqual(Hashed, select_guild_owner_node(?PINNED_GUILD, Targets)),
application:set_env(fluxer_gateway, guild_owner_pins, #{?PINNED_GUILD => node()}),
?assertEqual(node(), select_guild_owner_node(?PINNED_GUILD, Targets)),
application:set_env(fluxer_gateway, guild_owner_pins, #{?PINNED_GUILD => 'pin_x@h'}),
?assertEqual(Hashed, select_guild_owner_node(?PINNED_GUILD, Targets))
end).
is_ready_inverts_is_draining_test() ->
persistent_term:erase({fluxer_gateway, draining}),
?assert(is_ready()),
fluxer_gateway/src/gateway/gateway_periodic_gc.erl
+3 -3
View File
@@ -89,9 +89,9 @@ gc_if_large_and_idle(Pid) ->
-spec gc_if_known_type(pid(), non_neg_integer()) -> ok.
gc_if_known_type(Pid, Memory) ->
try erlang:process_info(Pid, dictionary) of
{dictionary, Dict} ->
force_gc_by_type(Pid, Memory, proplists:get_value('$initial_call', Dict));
try erlang:process_info(Pid, {dictionary, '$initial_call'}) of
{{dictionary, '$initial_call'}, InitialCall} ->
force_gc_by_type(Pid, Memory, InitialCall);
_ ->
ok
catch
fluxer_gateway/src/gateway/gateway_rpc_guild_infra.erl
+3 -2
View File
@@ -67,7 +67,7 @@ with_guild_unchecked(GuildId, Fun) ->
-spec ensure_responsive(pid()) -> ok.
ensure_responsive(Pid) ->
case guild_health:is_degraded(Pid) of
case guild_health:is_overloaded(Pid) of
true -> gateway_rpc_error:raise(<<"guild_overloaded">>);
false -> ok
end.
@@ -433,7 +433,8 @@ overloaded_guild_is_rejected_before_enqueuing_work_test() ->
stop -> ok
end
end),
true = ets:insert(guild_health_status, {Pid, 42, true, undefined, undefined}),
Pending = {make_ref(), erlang:monotonic_time(millisecond), 2500},
true = ets:insert(guild_health_status, {Pid, 42, true, undefined, Pending}),
try
?assertError(
{gateway_rpc_error, <<"guild_overloaded">>},
fluxer_gateway/src/guild/guild.erl
+10
View File
@@ -49,18 +49,24 @@ init(GuildState) ->
-spec handle_call(term(), gen_server:from(), guild_state()) -> call_reply().
handle_call(Msg, From, State) ->
ok = guild_mailbox_age:note(),
Result = handle_call_internal(Msg, From, State),
ok = publish_read_model(Result, State),
Result.
-spec handle_cast(term(), guild_state()) -> cast_reply().
handle_cast(Msg, State) ->
ok = guild_mailbox_age:note(),
Result = handle_cast_internal(Msg, State),
ok = publish_read_model(Result, State),
Result.
-spec handle_info(term(), guild_state()) -> info_reply().
handle_info({guild_mailbox_age, Seq}, State) when is_integer(Seq), Seq >= 0 ->
ok = guild_mailbox_age:handle_mark(Seq),
{noreply, State};
handle_info(Msg, State) ->
ok = guild_mailbox_age:note(),
Result = handle_info_internal(Msg, State),
ok = publish_read_model(Result, State),
Result.
@@ -188,6 +194,10 @@ handle_cast_internal(
{session_connect_worker_done, SessionId, Attempt, Result0, Computed}, State
) ->
handle_session_connect_worker_done_cast(SessionId, Attempt, Result0, Computed, State);
handle_cast_internal({session_connect_worker_batch_done, Results}, State) when
is_list(Results)
->
{noreply, guild_connect_async:finalize_session_connect_batch(Results, State)};
handle_cast_internal({set_session_active, SessionId}, State) ->
handle_set_session_active_cast(SessionId, State);
handle_cast_internal({set_session_passive, SessionId}, State) ->
fluxer_gateway/src/guild/guild_connect_async.erl
+209 -85
View File
@@ -6,6 +6,7 @@
-export([
enqueue_session_connect_async/5,
finalize_session_connect_async/5,
finalize_session_connect_batch/2,
maybe_start_session_connect_workers/1,
decrement_session_connect_inflight/1,
ensure_session_connect_queue/1
@@ -13,22 +14,23 @@
-type session_id() :: binary().
-type guild_id() :: integer().
-export_type([session_id/0]).
-type user_id() :: integer().
-type connect_result() :: {session_id() | undefined, non_neg_integer(), term(), map()}.
-type joined() :: {#{user_id() => map()}, [user_id()]}.
-export_type([session_id/0, connect_result/0]).
-define(SESSION_CONNECT_MAX_WORKERS, 8).
-define(SESSION_CONNECT_DEFAULT_MAX_QUEUE, 1024).
-define(SESSION_CONNECT_MAX_BATCH, 8).
-define(SESSION_CONNECT_DEFAULT_MAX_QUEUE, 8192).
-spec ensure_session_connect_queue(term()) -> queue:queue().
ensure_session_connect_queue(Value) when is_list(Value) ->
queue:from_list(Value);
ensure_session_connect_queue(Value) ->
safe_queue_cast(Value).
-spec safe_queue_cast(term()) -> queue:queue().
safe_queue_cast({R, F} = _Value) when is_list(R), is_list(F) ->
queue:from_list(F ++ lists:reverse(R));
safe_queue_cast(_) ->
queue:new().
case queue:is_queue(Value) of
true -> Value;
false -> queue:new()
end.
-spec enqueue_session_connect_async(integer(), non_neg_integer(), map(), map(), map()) -> map().
enqueue_session_connect_async(GuildId, Attempt, Request, Msg, State) ->
@@ -70,60 +72,114 @@ enqueue_session_connect_async_for_session(GuildId, Attempt, Request, Msg, Sessio
map(),
map()
) -> map().
finalize_session_connect_async(undefined, _Attempt, _Result0, _Computed, State) ->
maybe_start_session_connect_workers(decrement_session_connect_inflight(State));
finalize_session_connect_async(SessionId, Attempt, Result0, Computed, State) ->
State1 = decrement_session_connect_inflight(State),
Pending0 = maps:get(session_connect_pending, State1, #{}),
case maps:find(SessionId, Pending0) of
{ok, Attempt} ->
State2 = State1#{session_connect_pending => maps:remove(SessionId, Pending0)},
finalize_matched(SessionId, Attempt, Result0, Computed, State2);
_ ->
maybe_start_session_connect_workers(State1)
end.
finalize_session_connect_batch([{SessionId, Attempt, Result0, Computed}], State).
-spec finalize_matched(session_id(), non_neg_integer(), term(), map(), map()) -> map().
finalize_matched(SessionId, Attempt, Result0, Computed, State) ->
-spec finalize_session_connect_batch([connect_result()], map()) -> map().
finalize_session_connect_batch(Results, State) ->
State1 = decrement_session_connect_inflight(State),
{State2, Joined} = lists:foldl(fun finalize_result/2, {State1, {#{}, []}}, Results),
maybe_start_session_connect_workers(finish_joined_users(Joined, State1, State2)).
-spec finalize_result(term(), {map(), joined()}) -> {map(), joined()}.
finalize_result({SessionId, Attempt, Result0, Computed}, {State, Joined}) when
is_binary(SessionId), is_integer(Attempt), is_map(Computed)
->
Pending0 = maps:get(session_connect_pending, State, #{}),
case {maps:find(SessionId, Pending0), valid_result(Result0)} of
{{ok, Attempt}, true} ->
State1 = State#{session_connect_pending => maps:remove(SessionId, Pending0)},
finalize_matched(SessionId, Attempt, Result0, Computed, State1, Joined);
_ ->
{State, Joined}
end;
finalize_result(_Result, Acc) ->
Acc.
-spec valid_result(term()) -> boolean().
valid_result({ok, Result}) -> is_map(Result);
valid_result({ok_unavailable, Result}) -> is_map(Result);
valid_result({error, _Reason}) -> true;
valid_result(_) -> false.
-spec finalize_matched(session_id(), non_neg_integer(), term(), map(), map(), joined()) ->
{map(), joined()}.
finalize_matched(SessionId, Attempt, Result0, Computed, State, Joined) ->
Request = maps:get(request, Computed, #{}),
SessionPid = maps:get(session_pid, Request, undefined),
case is_pid(SessionPid) of
false ->
maybe_start_session_connect_workers(State);
{State, Joined};
true ->
finalize_with_pid(SessionId, Attempt, Result0, Computed, Request, SessionPid, State)
finalize_with_pid(
SessionId, Attempt, Result0, Computed, Request, SessionPid, State, Joined
)
end.
-spec finalize_with_pid(session_id(), non_neg_integer(), term(), map(), map(), pid(), map()) ->
map().
finalize_with_pid(SessionId, Attempt, Result0, Computed, Request, SessionPid, State) ->
-spec finalize_with_pid(
session_id(), non_neg_integer(), term(), map(), map(), pid(), map(), joined()
) -> {map(), joined()}.
finalize_with_pid(SessionId, Attempt, Result0, Computed, Request, SessionPid, State, Joined) ->
case resolve_guild_id(State, Computed) of
GuildId when is_integer(GuildId), GuildId > 0 ->
finalize_resolved(
GuildId, SessionId, Attempt, Result0, Computed, Request, SessionPid, State
GuildId,
SessionId,
Attempt,
Result0,
Computed,
Request,
SessionPid,
State,
Joined
);
_ ->
maybe_start_session_connect_workers(State)
{State, Joined}
end.
-spec finalize_resolved(
integer(), session_id(), non_neg_integer(), term(), map(), map(), pid(), map()
) -> map().
integer(), session_id(), non_neg_integer(), term(), map(), map(), pid(), map(), joined()
) -> {map(), joined()}.
finalize_resolved(
GuildId, SessionId, Attempt, {error, _} = Result0, _Computed, _Request, SessionPid, State
GuildId,
SessionId,
Attempt,
{error, Reason} = Result0,
_Computed,
_Request,
SessionPid,
State,
Joined
) ->
State1 = discard_pending_session(SessionId, State),
State1 = discard_pending_session(SessionId, Reason, State),
send_result(GuildId, Attempt, Result0, SessionPid),
maybe_start_session_connect_workers(State1);
finalize_resolved(GuildId, SessionId, Attempt, Result0, Computed, Request, SessionPid, State) ->
State1 = upsert_session(SessionId, SessionPid, Request, Computed, State),
{State1, Joined};
finalize_resolved(
GuildId, SessionId, Attempt, Result0, Computed, Request, SessionPid, State, Joined
) ->
Acc = upsert_session(SessionId, SessionPid, Request, Computed, State, Joined),
send_result(GuildId, Attempt, Result0, SessionPid),
maybe_start_session_connect_workers(State1).
Acc.
-spec discard_pending_session(session_id() | undefined, map()) -> map().
discard_pending_session(SessionId, State) when is_binary(SessionId) ->
-spec finish_joined_users(joined(), map(), map()) -> map().
finish_joined_users({Joined, Fresh}, StartState, State) ->
State1 = guild_presence:apply_connect_presences(lists:reverse(Fresh), State),
maps:fold(
fun(UserId, PresenceBefore, Acc) ->
guild_sessions_connect:resection_connected_user(
UserId, PresenceBefore, StartState, Acc
)
end,
State1,
Joined
).
-spec discard_pending_session(session_id(), term(), map()) -> map().
discard_pending_session(SessionId, Reason, State) ->
Sessions0 = maps:get(sessions, State, #{}),
case maps:find(SessionId, Sessions0) of
{ok, #{pending_connect := true, owns_connected_tracking := true} = Entry} ->
discard_reconnecting_session(SessionId, Reason, Entry, State);
{ok, #{pending_connect := true} = Entry} ->
demonitor_pending_session(Entry),
guild_sessions_connect:remove_session_ref(
@@ -132,8 +188,18 @@ discard_pending_session(SessionId, State) when is_binary(SessionId) ->
);
_ ->
State
end;
discard_pending_session(_SessionId, State) ->
end.
-spec discard_reconnecting_session(session_id(), term(), map(), map()) -> map().
discard_reconnecting_session(SessionId, not_member, #{mref := MRef} = Entry, State) when
is_reference(MRef)
->
demonitor(MRef, [flush]),
{noreply, State1} = guild_sessions_connect:handle_session_down(
MRef, {SessionId, Entry}, State
),
State1;
discard_reconnecting_session(_SessionId, _Reason, _Entry, State) ->
State.
-spec demonitor_pending_session(map()) -> ok.
@@ -158,19 +224,31 @@ maybe_start_session_connect_workers(State) ->
-spec dequeue_and_start(queue:queue(), non_neg_integer(), map()) -> map().
dequeue_and_start(Queue0, Inflight0, State) ->
case queue:out(Queue0) of
{{value, Next}, Rest} when is_map(Next) ->
case take_batch(batch_size(Queue0, Inflight0), Queue0, []) of
{[], Rest} ->
State#{session_connect_queue => Rest};
{Items, Rest} ->
State1 = State#{
session_connect_queue => Rest,
session_connect_inflight => Inflight0 + 1
},
maybe_start_session_connect_workers(start_worker(Next, State1));
{{value, _Next}, Rest} ->
maybe_start_session_connect_workers(
State#{session_connect_queue => Rest}
);
{empty, _} ->
State#{session_connect_queue => Queue0}
maybe_start_session_connect_workers(start_worker(Items, State1))
end.
-spec batch_size(queue:queue(), non_neg_integer()) -> pos_integer().
batch_size(Queue, Inflight) ->
FreeWorkers = max(1, ?SESSION_CONNECT_MAX_WORKERS - Inflight),
Spread = (queue:len(Queue) + FreeWorkers - 1) div FreeWorkers,
max(1, min(?SESSION_CONNECT_MAX_BATCH, Spread)).
-spec take_batch(non_neg_integer(), queue:queue(), [map()]) -> {[map()], queue:queue()}.
take_batch(0, Queue, Acc) ->
{lists:reverse(Acc), Queue};
take_batch(N, Queue, Acc) ->
case queue:out(Queue) of
{{value, Item}, Rest} when is_map(Item) -> take_batch(N - 1, Rest, [Item | Acc]);
{{value, _Item}, Rest} -> take_batch(N, Rest, Acc);
{empty, Rest} -> {lists:reverse(Acc), Rest}
end.
-spec decrement_session_connect_inflight(map()) -> map().
@@ -280,7 +358,7 @@ cleanup_dropped_session_connect(Item, State) ->
case queued_session_id(Item) of
SessionId when is_binary(SessionId) ->
State1 = remove_pending_session_connect(SessionId, State),
discard_pending_session(SessionId, State1);
discard_pending_session(SessionId, overloaded, State1);
_ ->
State
end.
@@ -316,14 +394,22 @@ upsert_pending_session(S, U, P, Request, State) ->
active_guilds => maps:get(active_guilds, Request, sets:new()),
bot => maps:get(bot, Request, false),
is_staff => maps:get(is_staff, Request, false),
pending_connect => true,
viewable_channels => #{}
pending_connect => true
},
guild_sessions_connect:put_session_ref(S, MRef, State#{
sessions => Sessions0#{S => Entry}
});
{ok, Existing} ->
State#{sessions => Sessions0#{S => Existing#{pending_connect => true}}}
State#{sessions => Sessions0#{S => mark_pending(Existing)}}
end.
-spec mark_pending(map()) -> map().
mark_pending(#{pending_connect := true} = Existing) ->
Existing;
mark_pending(Existing) ->
case connected_user_id_from_existing(Existing) of
undefined -> Existing#{pending_connect => true};
_UserId -> Existing#{pending_connect => true, owns_connected_tracking => true}
end.
-spec drop_queued(session_id(), queue:queue()) -> queue:queue().
@@ -364,24 +450,30 @@ queued_session_pid(#{reply_via_pid := SessionPid}) when is_pid(SessionPid) ->
queued_session_pid(_) ->
undefined.
-spec start_worker(map(), map()) -> map().
start_worker(Item, State) ->
-spec start_worker([map()], map()) -> map().
start_worker(Items, State) ->
Self = self(),
Snapshot = guild_data:build_connect_snapshot(Item, State),
{_Pid, Ref} = spawn_monitor(fun() -> compute_and_send_done(Item, Self, Snapshot) end),
Snapshot = guild_data:build_connect_snapshot(Items, State),
{_Pid, Ref} = spawn_monitor(fun() -> compute_and_send_done(Items, Self, Snapshot) end),
WorkerRefs = maps:get(session_connect_worker_refs, State, #{}),
State#{session_connect_worker_refs => WorkerRefs#{Ref => true}}.
-spec compute_and_send_done(map(), pid(), map()) -> ok.
compute_and_send_done(Item, GuildPid, Snapshot) ->
-spec compute_and_send_done([map()], pid(), map()) -> ok.
compute_and_send_done(Items, GuildPid, Snapshot) ->
Results = [compute_item(Item, Snapshot) || Item <- Items],
gen_server:cast(GuildPid, {session_connect_worker_batch_done, Results}),
ok.
-spec compute_item(map(), map()) -> connect_result().
compute_item(Item, Snapshot) ->
Request = maps:get(request, Item, #{}),
{Result0, Computed0} = compute_connect_result(Snapshot, Item, Request),
gen_server:cast(
GuildPid,
{session_connect_worker_done, maps:get(session_id, Request, undefined),
maps:get(attempt, Item, 0), Result0, maps:merge(Item, Computed0)}
),
ok.
{
maps:get(session_id, Request, undefined),
maps:get(attempt, Item, 0),
Result0,
maps:merge(Item, Computed0)
}.
-spec compute_connect_result(map(), map(), map()) ->
{{ok, map()} | {ok_unavailable, map()} | {error, term()}, map()}.
@@ -471,16 +563,21 @@ resolve_guild_id(State, Fallback) ->
snowflake_id:parse_optional(maps:get(guild_id, Fallback, undefined))
end.
-spec upsert_session(session_id(), pid(), map(), map(), map()) -> map().
upsert_session(SessionId, SessionPid, Request, Computed, State) ->
-spec upsert_session(session_id(), pid(), map(), map(), map(), joined()) -> {map(), joined()}.
upsert_session(SessionId, SessionPid, Request, Computed, State, Joined) ->
UserId = maps:get(user_id, Request, undefined),
case is_integer(UserId) of
false -> State;
true -> upsert_session_valid(SessionId, SessionPid, UserId, Request, Computed, State)
false ->
{State, Joined};
true ->
upsert_session_valid(
SessionId, SessionPid, UserId, Request, Computed, State, Joined
)
end.
-spec upsert_session_valid(session_id(), pid(), integer(), map(), map(), map()) -> map().
upsert_session_valid(SessionId, SessionPid, UserId, Request, Computed, State) ->
-spec upsert_session_valid(session_id(), pid(), integer(), map(), map(), map(), joined()) ->
{map(), joined()}.
upsert_session_valid(SessionId, SessionPid, UserId, Request, Computed, State, Joined) ->
Sessions0 = maps:get(sessions, State, #{}),
Existing = maps:get(SessionId, Sessions0, undefined),
{MRef, Existing1} = resolve_monitor(Existing, SessionPid),
@@ -491,9 +588,21 @@ upsert_session_valid(SessionId, SessionPid, UserId, Request, Computed, State) ->
Sessions = merge_session(SessionId, FinalSD, Existing1, Sessions0),
State1 = reindex_session_ref(SessionId, Existing, MRef, State#{sessions => Sessions}),
State2 = update_connected_tracking(UserId, Existing, State1),
PresenceBefore = guild_member_list_connected:resolve_presence_for_user(State2, UserId),
State3 = update_presence_subscription(UserId, Existing, State2),
guild_sessions_connect:resection_connected_user(UserId, PresenceBefore, State, State3).
update_presence_subscription(
UserId, Existing, State2, note_joined_user(UserId, State2, Joined)
).
-spec note_joined_user(integer(), map(), joined()) -> joined().
note_joined_user(UserId, State, {Joined, Fresh}) ->
case maps:is_key(UserId, Joined) of
true ->
{Joined, Fresh};
false ->
PresenceBefore = guild_member_list_connected:resolve_presence_for_user(
State, UserId
),
{Joined#{UserId => PresenceBefore}, Fresh}
end.
-spec build_session_data(session_id(), integer(), pid(), reference(), map(), map()) -> map().
build_session_data(SessionId, UserId, SessionPid, MRef, Request, Computed) ->
@@ -530,7 +639,9 @@ maybe_mark_synced(GuildId, Computed, SessionData) ->
merge_session(SessionId, FinalSD, undefined, Sessions0) ->
Sessions0#{SessionId => FinalSD};
merge_session(SessionId, FinalSD, Existing, Sessions0) ->
Sessions0#{SessionId => maps:merge(Existing, FinalSD)}.
Sessions0#{
SessionId => maps:merge(maps:remove(owns_connected_tracking, Existing), FinalSD)
}.
-spec reindex_session_ref(session_id(), map() | undefined, reference(), map()) -> map().
reindex_session_ref(SessionId, #{mref := OldRef}, MRef, State) when OldRef =/= MRef ->
@@ -563,26 +674,37 @@ update_connected_tracking(UserId, Existing, State) ->
add_connected_user(UserId, remove_connected_user(ExistingUserId, State))
end.
-spec update_presence_subscription(integer(), map() | undefined, map()) -> map().
update_presence_subscription(UserId, Existing, State) ->
-spec update_presence_subscription(integer(), map() | undefined, map(), joined()) ->
{map(), joined()}.
update_presence_subscription(UserId, Existing, State, Joined) ->
case connected_user_id_from_existing(Existing) of
UserId ->
State;
{State, Joined};
undefined ->
guild_sessions:subscribe_connected_user_presence(UserId, State);
subscribe_joined_user(UserId, State, Joined);
ExistingUserId ->
State1 = guild_sessions:unsubscribe_from_user_presence(ExistingUserId, State),
guild_sessions:subscribe_connected_user_presence(UserId, State1)
subscribe_joined_user(UserId, State1, Joined)
end.
-spec subscribe_joined_user(integer(), map(), joined()) -> {map(), joined()}.
subscribe_joined_user(UserId, State, {Joined, Fresh}) ->
case guild_sessions_presence:subscribe_without_cached_presence(UserId, State) of
{fresh, State1} -> {State1, {Joined, [UserId | Fresh]}};
{existing, State1} -> {State1, {Joined, Fresh}}
end.
-spec connected_user_id_from_existing(map() | undefined) -> integer() | undefined.
connected_user_id_from_existing(undefined) ->
undefined;
connected_user_id_from_existing(#{pending_connect := true}) ->
undefined;
connected_user_id_from_existing(Existing) ->
case maps:get(user_id, Existing, undefined) of
UserId when is_integer(UserId), UserId > 0 ->
case
{
guild_sessions_connect:counts_as_connected(Existing),
maps:get(user_id, Existing, undefined)
}
of
{true, UserId} when is_integer(UserId), UserId > 0 ->
UserId;
_ ->
undefined
@@ -703,6 +825,7 @@ enqueue_marks_existing_session_pending_connect_test() ->
SessionId, UserId, pending_connect_state(#{SessionId => Existing})
),
?assertEqual(true, maps:get(pending_connect, Entry)),
?assertEqual(true, maps:get(owns_connected_tracking, Entry)),
?assertEqual(MRef, maps:get(mref, Entry)),
?assertEqual(UserId, maps:get(user_id, Entry)).
@@ -712,6 +835,7 @@ enqueue_creates_pending_session_entry_test() ->
Entry = enqueued_session_entry(SessionId, UserId, pending_connect_state(#{})),
demonitor(maps:get(mref, Entry), [flush]),
?assertEqual(true, maps:get(pending_connect, Entry)),
?assertNot(maps:is_key(owns_connected_tracking, Entry)),
?assertEqual(UserId, maps:get(user_id, Entry)).
-endif.
fluxer_gateway/src/guild/guild_cutover.erl
+342
View File
@@ -0,0 +1,342 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(guild_cutover).
-feature(maybe_expr, enable).
-typing([eqwalizer]).
-export([run/1, reverse/1, move/4, routes/2]).
-define(PIN_KEY, guild_owner_pins).
-define(DEFAULT_ROUTE_TIMEOUT, 5000).
-define(DEFAULT_RELOAD_TIMEOUT, 120000).
-define(DEFAULT_MAX_MAILBOX, 100).
-define(PROBE_KEY, 0).
-define(KEEPER_KEYS, [guild_pin_keeper_armed, guild_pin_keeper_ever_armed]).
-type guild_id() :: integer().
-type saved_pins() :: #{node() => {ok, term()} | undefined}.
-type report() :: map().
-export_type([report/0]).
-spec run(map()) -> {ok, report()} | {error, report()}.
run(#{guild_id := GuildId, source_node := SourceNode} = Opts) when
is_integer(GuildId), is_atom(SourceNode)
->
move(GuildId, SourceNode, node(), maps:merge(#{transfer_sessions => true}, Opts)).
-spec reverse(map()) -> {ok, report()} | {error, report()}.
reverse(#{guild_id := GuildId, target_node := TargetNode} = Opts) when
is_integer(GuildId), is_atom(TargetNode)
->
move(GuildId, node(), TargetNode, maps:merge(#{transfer_sessions => false}, Opts)).
-spec routes(guild_id(), [node()]) -> #{node() => term()}.
routes(GuildId, Nodes) ->
Args = [GuildId, guilds],
Results = erpc:multicall(
Nodes, gateway_node_router, owner_node_result, Args, ?DEFAULT_ROUTE_TIMEOUT
),
maps:from_list(lists:zip(Nodes, Results)).
-spec move(guild_id(), node(), node(), map()) -> {ok, report()} | {error, report()}.
move(GuildId, From, To, Opts) ->
Nodes = route_nodes(From, To, Opts),
case preflight(GuildId, From, To, Nodes, Opts) of
{ok, SrcPid, SrcShard, Saved} ->
TransferOpts = transfer_opts(GuildId, From, To, Nodes, Saved, Opts),
Result = guild_handoff_freeze:transfer(GuildId, SrcPid, SrcShard, To, TransferOpts),
finish(GuildId, SrcPid, {From, To, Nodes}, Result, Opts);
{error, Reason} ->
{error, #{phase => preflight, reason => Reason, from => From, to => To}}
end.
-spec route_nodes(node(), node(), map()) -> [node()].
route_nodes(From, To, Opts) ->
Nodes = maps:get(route_nodes, Opts, [node() | nodes()]),
lists:usort([From, To | [N || N <- Nodes, is_atom(N)]]).
-spec preflight(guild_id(), node(), node(), [node()], map()) ->
{ok, pid(), pid(), saved_pins()} | {error, term()}.
preflight(_GuildId, Node, Node, _Nodes, _Opts) ->
{error, same_node};
preflight(GuildId, From, To, Nodes, Opts) ->
maybe
ok ?= keeper_idle(lists:usort([node(), From, To])),
ok ?= connected([From, To]),
ok ?= expect_routes(GuildId, Nodes, From),
{ok, SrcShard} ?= guild_handoff_freeze:shard_pid(GuildId, From),
{ok, SrcPid} ?= lookup(SrcShard, GuildId),
ok ?= absent_on(GuildId, To),
ok ?= mailbox_below(SrcPid, maps:get(max_mailbox, Opts, ?DEFAULT_MAX_MAILBOX)),
false ?= guild_handoff_freeze:is_frozen(SrcPid),
{ok, Saved} ?= read_pins(Nodes),
ok ?= probe_pins(Nodes, To, Saved),
{ok, SrcPid, SrcShard, Saved}
else
true -> {error, source_already_frozen};
{error, _} = Error -> Error
end.
-spec transfer_opts(guild_id(), node(), node(), [node()], saved_pins(), map()) ->
guild_handoff_freeze:opts().
transfer_opts(GuildId, From, To, Nodes, Saved, Opts) ->
Base = maps:with(
[
suspend_timeout,
export_timeout,
start_timeout,
stop_timeout,
barrier_timeout,
rpc_timeout,
guard_timeout,
freeze_budget,
commit_timeout,
forward_rounds,
transfer_sessions,
max_heap_words,
measure
],
Opts
),
Others = [N || N <- Nodes, N =/= To],
Base#{
before_start => fun() -> pin_and_verify([To], GuildId, To, Saved) end,
after_start => fun(_NewPid) -> pin_and_verify(Others, GuildId, To, Saved) end,
on_abort => fun() -> restore_and_verify(Nodes, GuildId, From, Saved) end
}.
-spec finish(
guild_id(), pid(), {node(), node(), [node()]}, guild_handoff_freeze:result(), map()
) ->
{ok, report()} | {error, report()}.
finish(GuildId, _SrcPid, {From, To, Nodes}, {ok, Report}, Opts) ->
Reload = maybe_reload(GuildId, To, Opts),
{ok, Report#{
from => From,
to => To,
reload => Reload,
routes_after => routes(GuildId, Nodes)
}};
finish(GuildId, SrcPid, {From, To, Nodes}, {error, Report}, Opts) ->
Repair = repair_source(GuildId, SrcPid, Report, Opts),
{error, Report#{
from => From,
to => To,
source_reload => Repair,
routes_after => routes(GuildId, Nodes)
}}.
-spec repair_source(guild_id(), pid(), report(), map()) -> skipped | term().
repair_source(GuildId, SrcPid, #{abort := #{routes := Routes} = Abort}, Opts) when
Routes =/= skipped
->
case maps:get(exposed, Abort, true) of
true -> guild_handoff_freeze:repair(GuildId, SrcPid, repair_opts(Opts));
false -> skipped
end;
repair_source(_GuildId, _SrcPid, _Report, _Opts) ->
skipped.
-spec repair_opts(map()) -> guild_handoff_freeze:opts().
repair_opts(Opts) ->
maps:with([suspend_timeout, guard_timeout, repair_timeout], Opts).
-spec maybe_reload(guild_id(), node(), map()) -> skipped | term().
maybe_reload(GuildId, To, #{reload := true} = Opts) ->
reload(GuildId, To, Opts);
maybe_reload(_GuildId, _To, _Opts) ->
skipped.
-spec reload(guild_id(), node(), map()) -> term().
reload(GuildId, Node, Opts) ->
Timeout = maps:get(reload_timeout, Opts, ?DEFAULT_RELOAD_TIMEOUT),
Args = [guild_manager, {reload_guild, GuildId}, Timeout],
try erpc:call(Node, gen_server, call, Args, Timeout + 5000) of
Reply -> Reply
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec keeper_idle([node()]) -> ok | {error, term()}.
keeper_idle(Nodes) ->
Answers = [
{Node, Key, Answer}
|| Key <- ?KEEPER_KEYS,
{Node, Answer} <- lists:zip(
Nodes,
erpc:multicall(
Nodes, application, get_env, [fluxer_gateway, Key], ?DEFAULT_ROUTE_TIMEOUT
)
)
],
Active = [{Node, Key} || {Node, Key, {ok, {ok, true}}} <- Answers],
Unreadable = [{Node, Key, A} || {Node, Key, A} <- Answers, not keeper_answer(A)],
case {Active, Unreadable} of
{[], []} -> ok;
{[], _} -> {error, {pin_keeper_unreadable, Unreadable}};
_ -> {error, {pin_keeper_active, Active}}
end.
-spec keeper_answer(term()) -> boolean().
keeper_answer({ok, undefined}) -> true;
keeper_answer({ok, {ok, _Value}}) -> true;
keeper_answer(_Answer) -> false.
-spec connected([node()]) -> ok | {error, term()}.
connected(Nodes) ->
case [N || N <- Nodes, N =/= node(), not lists:member(N, nodes())] of
[] -> ok;
Missing -> {error, {not_connected, Missing}}
end.
-spec expect_routes(guild_id(), [node()], node()) -> ok | {error, term()}.
expect_routes(GuildId, Nodes, Owner) ->
case [{N, R} || {N, R} <- maps:to_list(routes(GuildId, Nodes)), R =/= {ok, {ok, Owner}}] of
[] -> ok;
Wrong -> {error, {routes_disagree, Owner, Wrong}}
end.
-spec lookup(pid(), guild_id()) -> {ok, pid()} | {error, term()}.
lookup(Shard, GuildId) ->
try gen_server:call(Shard, {lookup, GuildId}, ?DEFAULT_ROUTE_TIMEOUT) of
{ok, Pid} when is_pid(Pid) -> {ok, Pid};
Other -> {error, {source_lookup, Other}}
catch
exit:Reason -> {error, {source_lookup, Reason}}
end.
-spec absent_on(guild_id(), node()) -> ok | {error, term()}.
absent_on(GuildId, Node) ->
case guild_handoff_freeze:shard_pid(GuildId, Node) of
{ok, Shard} ->
try gen_server:call(Shard, {lookup, GuildId}, ?DEFAULT_ROUTE_TIMEOUT) of
{error, not_found} -> ok;
Other -> {error, {target_not_empty, Other}}
catch
exit:Reason -> {error, {target_lookup, Reason}}
end;
{error, _} = Error ->
Error
end.
-spec mailbox_below(pid(), non_neg_integer()) -> ok | {error, term()}.
mailbox_below(Pid, Max) ->
Args = [Pid, message_queue_len],
try erpc:call(node(Pid), erlang, process_info, Args, ?DEFAULT_ROUTE_TIMEOUT) of
{message_queue_len, Len} when Len =< Max -> ok;
Other -> {error, {source_mailbox, Other}}
catch
Class:Reason -> {error, {source_mailbox, Class, Reason}}
end.
-spec read_pins([node()]) -> {ok, saved_pins()} | {error, term()}.
read_pins(Nodes) ->
Results = erpc:multicall(
Nodes, application, get_env, [fluxer_gateway, ?PIN_KEY], ?DEFAULT_ROUTE_TIMEOUT
),
case [{N, R} || {N, R} <- lists:zip(Nodes, Results), not readable_pin(R)] of
[] -> {ok, maps:from_list([{N, V} || {N, {ok, V}} <- lists:zip(Nodes, Results)])};
Failed -> {error, {pins_unreadable, Failed}}
end.
-spec readable_pin(term()) -> boolean().
readable_pin({ok, undefined}) -> true;
readable_pin({ok, {ok, Pins}}) when is_map(Pins) -> true;
readable_pin(_) -> false.
-spec probe_pins([node()], node(), saved_pins()) -> ok | {error, term()}.
probe_pins(Nodes, Owner, Saved) ->
Probe = pin_and_verify(Nodes, ?PROBE_KEY, Owner, Saved),
Restore = write_pins([{N, maps:get(N, Saved, undefined)} || N <- Nodes]),
case {Probe, Restore} of
{ok, ok} -> ok;
_ -> {error, {pin_probe_failed, Probe, Restore}}
end.
-spec pin_and_verify([node()], guild_id(), node(), saved_pins()) -> ok | {error, term()}.
pin_and_verify([], _GuildId, _Owner, _Saved) ->
ok;
pin_and_verify(Nodes, GuildId, Owner, Saved) ->
Values = [{N, {ok, pinned(maps:get(N, Saved, undefined), GuildId, Owner)}} || N <- Nodes],
maybe
ok ?= write_pins(Values),
expect_routes(GuildId, Nodes, Owner)
end.
-spec restore_and_verify([node()], guild_id(), node(), saved_pins()) -> ok | {error, term()}.
restore_and_verify(Nodes, GuildId, Owner, Saved) ->
Values = [{N, maps:get(N, Saved, undefined)} || N <- Nodes],
maybe
ok ?= write_pins(Values),
expect_routes(GuildId, Nodes, Owner)
end.
-spec pinned({ok, term()} | undefined, guild_id(), node()) -> map().
pinned({ok, Pins}, GuildId, Owner) when is_map(Pins) ->
Pins#{GuildId => Owner};
pinned(_Previous, GuildId, Owner) ->
#{GuildId => Owner}.
-spec write_pins([{node(), {ok, term()} | undefined}]) -> ok | {error, term()}.
write_pins(Values) ->
Groups = maps:groups_from_list(fun({_N, V}) -> V end, fun({N, _V}) -> N end, Values),
Failed = maps:fold(fun write_group/3, [], Groups),
case Failed of
[] -> ok;
_ -> {error, {pin_write_failed, Failed}}
end.
-spec write_group({ok, term()} | undefined, [node()], list()) -> list().
write_group({ok, Value}, Nodes, Acc) ->
Args = [fluxer_gateway, ?PIN_KEY, Value],
collect_failures(
Nodes, erpc:multicall(Nodes, application, set_env, Args, ?DEFAULT_ROUTE_TIMEOUT), Acc
);
write_group(undefined, Nodes, Acc) ->
Args = [fluxer_gateway, ?PIN_KEY],
collect_failures(
Nodes, erpc:multicall(Nodes, application, unset_env, Args, ?DEFAULT_ROUTE_TIMEOUT), Acc
).
-spec collect_failures([node()], list(), list()) -> list().
collect_failures(Nodes, Results, Acc) ->
[{N, R} || {N, R} <- lists:zip(Nodes, Results), R =/= {ok, ok}] ++ Acc.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
cutover_refuses_once_the_pin_keeper_has_armed_test() ->
Opts = #{guild_id => 42, target_node => '[email protected]'},
application:set_env(fluxer_gateway, guild_pin_keeper_ever_armed, true),
try
?assertMatch(
{error, #{
phase := preflight,
reason := {pin_keeper_active, [{_, guild_pin_keeper_ever_armed}]}
}},
reverse(Opts)
)
after
application:unset_env(fluxer_gateway, guild_pin_keeper_ever_armed)
end,
?assertMatch(
{error, #{phase := preflight, reason := {pin_keeper_unreadable, _}}}, reverse(Opts)
).
exposed_abort_repairs_the_source_test() ->
Gone = spawn(fun() -> ok end),
Ref = erlang:monitor(process, Gone),
receive
{'DOWN', Ref, process, Gone, _} -> ok
end,
Exposed = #{abort => #{routes => ok, exposed => true}},
?assertEqual({error, noproc}, repair_source(42, Gone, Exposed, #{})),
Crashed = #{abort => #{routes => ok, target => ok}},
?assertEqual({error, noproc}, repair_source(42, Gone, Crashed, #{})),
Unexposed = #{abort => #{routes => ok, exposed => false}},
?assertEqual(skipped, repair_source(42, Gone, Unexposed, #{})),
Unrouted = #{abort => #{routes => skipped, exposed => true}},
?assertEqual(skipped, repair_source(42, Gone, Unrouted, #{})).
-endif.
fluxer_gateway/src/guild/guild_data.erl
+50 -19
View File
@@ -157,8 +157,10 @@ get_guild_state(UserId, State) ->
JoinedAt
).
-spec build_connect_snapshot(map(), guild_state()) -> map().
build_connect_snapshot(Item, State) ->
-spec build_connect_snapshot(map() | [map()], guild_state()) -> map().
build_connect_snapshot(Item, State) when is_map(Item) ->
build_connect_snapshot([Item], State);
build_connect_snapshot(Items, State) ->
Base = maps:with(
[
id,
@@ -172,28 +174,38 @@ build_connect_snapshot(Item, State) ->
],
State
),
project_snapshot_sessions(maybe_trim_connect_snapshot(Item, Base, State)).
project_snapshot_sessions(maybe_trim_connect_snapshot(Items, Base, State)).
-spec project_snapshot_sessions(map()) -> map().
project_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->
Projected = maps:map(
fun(_SessionId, SessionData) -> project_snapshot_session(SessionData) end,
Sessions
),
Snapshot#{sessions => Projected};
case snapshot_reads_sessions(Snapshot) of
true ->
Projected = maps:map(
fun(_SessionId, SessionData) -> project_snapshot_session(SessionData) end,
Sessions
),
Snapshot#{sessions => Projected};
false ->
Snapshot#{sessions => #{}}
end;
project_snapshot_sessions(Snapshot) ->
Snapshot.
-spec snapshot_reads_sessions(map()) -> boolean().
snapshot_reads_sessions(Snapshot) ->
guild_availability_check:get_unavailability_mode_from_state(Snapshot) =:=
unavailable_for_everyone_but_staff.
-spec project_snapshot_session(term()) -> term().
project_snapshot_session(SessionData) when is_map(SessionData) ->
maps:without(?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, SessionData);
project_snapshot_session(SessionData) ->
SessionData.
-spec maybe_trim_connect_snapshot(map(), map(), guild_state()) -> map().
maybe_trim_connect_snapshot(Item, Base, State) ->
-spec maybe_trim_connect_snapshot([map()], map(), guild_state()) -> map().
maybe_trim_connect_snapshot(Items, Base, State) ->
case should_trim_connect_snapshot(State) of
true -> trim_connect_snapshot(Item, Base);
true -> trim_connect_snapshot(Items, Base);
false -> Base
end.
@@ -205,9 +217,9 @@ should_trim_connect_snapshot(State) ->
has_members_ets(#{data := #{members_ets := Tab}}) -> is_reference(Tab);
has_members_ets(_) -> false.
-spec trim_connect_snapshot(map(), map()) -> map().
trim_connect_snapshot(Item, #{data := Data} = Base) when is_map(Data) ->
Retained = retained_member_map(Item, Base, Data),
-spec trim_connect_snapshot([map()], map()) -> map().
trim_connect_snapshot(Items, #{data := Data} = Base) when is_map(Data) ->
Retained = retained_member_map(Items, Base, Data),
Trimmed = maps:without(?CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS, Data),
Base#{
data => Trimmed#{
@@ -217,12 +229,12 @@ trim_connect_snapshot(Item, #{data := Data} = Base) when is_map(Data) ->
guild_data_index_members:build_member_role_index(Retained)
}
};
trim_connect_snapshot(_Item, Base) ->
trim_connect_snapshot(_Items, Base) ->
Base.
-spec retained_member_map(map(), map(), map()) -> #{integer() => map()}.
retained_member_map(Item, Base, Data) ->
UserIds = [connect_user_id(Item) | voice_state_user_ids(Base)],
-spec retained_member_map([map()], map(), map()) -> #{integer() => map()}.
retained_member_map(Items, Base, Data) ->
UserIds = [connect_user_id(Item) || Item <- Items] ++ voice_state_user_ids(Base),
lists:foldl(
fun(UserId, Acc) -> retain_member(UserId, Data, Acc) end,
#{},
@@ -480,8 +492,12 @@ guild_id_wire_value(GuildId) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
staff_only_data() ->
#{<<"guild">> => #{<<"features">> => [<<"UNAVAILABLE_FOR_EVERYONE_BUT_STAFF">>]}}.
projection_snapshot() ->
#{
data => staff_only_data(),
sessions => #{
<<"s1">> => #{
session_id => <<"s1">>,
@@ -520,9 +536,24 @@ projection_leaves_snapshot_data_untouched_test() ->
?assertEqual(Data, maps:get(data, project_snapshot_sessions(Snapshot))).
projection_keeps_non_map_session_entries_test() ->
Snapshot = #{sessions => #{<<"broken">> => not_a_map}},
Snapshot = #{data => staff_only_data(), sessions => #{<<"broken">> => not_a_map}},
?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).
projection_drops_sessions_unless_staff_only_test() ->
Snapshot = projection_snapshot(),
lists:foreach(
fun(Data) ->
Projected = project_snapshot_sessions(Snapshot#{data => Data}),
?assertEqual(#{}, maps:get(sessions, Projected)),
?assertEqual(Data, maps:get(data, Projected))
end,
[
#{},
#{<<"guild">> => #{<<"features">> => []}},
#{<<"guild">> => #{<<"features">> => [<<"UNAVAILABLE_FOR_EVERYONE">>]}}
]
).
projection_keeps_snapshot_without_sessions_test() ->
Snapshot = #{id => 42, member_count => 3},
?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).
fluxer_gateway/src/guild/guild_dm_partners.erl
+40 -14
View File
@@ -21,6 +21,8 @@
-define(MAX_PARTNERS, 1000).
-define(PRUNE_SLACK, 64).
-define(VIEW_MEMO_KEY, dm_partner_view_memo).
-define(VIEW_MEMO_LIMIT, 4096).
-spec handle_cast(term(), guild_state()) -> {noreply, guild_state()}.
handle_cast({update_dm_partners, SessionId, PartnerIds}, State) when
@@ -84,21 +86,21 @@ channel_inputs(Channels) ->
-spec update(session_id(), [term()], guild_state()) -> guild_state().
update(SessionId, PartnerIds, State) ->
Registrations = registrations(State),
Next =
{Next, State1} =
case session_owner(SessionId, State) of
{ok, UserId, Pid} ->
update_owned(SessionId, UserId, Pid, PartnerIds, Registrations, State);
error ->
maps:remove(SessionId, Registrations)
{maps:remove(SessionId, Registrations), State}
end,
State#{dm_partners => maybe_prune(Next, State)}.
State1#{dm_partners => maybe_prune(Next, State1)}.
-spec update_owned(session_id(), user_id(), pid(), [term()], registrations(), guild_state()) ->
registrations().
{registrations(), guild_state()}.
update_owned(SessionId, UserId, Pid, PartnerIds, Registrations, State) ->
case presence_targets:dm_partner_presence_enabled(UserId) of
false ->
maps:remove(SessionId, Registrations);
{maps:remove(SessionId, Registrations), State};
true ->
Entry = #{
user_id => UserId,
@@ -108,14 +110,16 @@ update_owned(SessionId, UserId, Pid, PartnerIds, Registrations, State) ->
maps:get(SessionId, Registrations, undefined), Pid
)
},
put_all(evaluate(#{SessionId => Entry}, State), Registrations)
{Evaluated, State1} = evaluate(#{SessionId => Entry}, State),
{put_all(Evaluated, Registrations), State1}
end.
-spec reevaluate(scope(), registrations(), guild_state()) -> guild_state().
reevaluate(none, _Registrations, State) ->
State;
reevaluate(all, Registrations, State) ->
State#{dm_partners => evaluate(live_registrations(Registrations, State), State)};
{Evaluated, State1} = evaluate(live_registrations(Registrations, State), State),
State1#{dm_partners => Evaluated};
reevaluate({user, UserId}, Registrations, State) ->
Affected = maps:fold(
fun(SessionId, Entry, Acc) ->
@@ -135,7 +139,8 @@ reevaluate({user, UserId}, Registrations, State) ->
Kept = maps:without(
maps:keys(maps:without(maps:keys(Live), Affected)), Registrations
),
State#{dm_partners => maybe_prune(put_all(evaluate(Live, State), Kept), State)}
{Evaluated, State1} = evaluate(Live, State),
State1#{dm_partners => maybe_prune(put_all(Evaluated, Kept), State1)}
end.
-spec put_all(registrations(), registrations()) -> registrations().
@@ -156,22 +161,43 @@ maybe_prune(Registrations, State) ->
Registrations
end.
-spec evaluate(registrations(), guild_state()) -> registrations().
evaluate(Entries, _State) when map_size(Entries) =:= 0 ->
Entries;
-spec evaluate(registrations(), guild_state()) -> {registrations(), guild_state()}.
evaluate(Entries, State) when map_size(Entries) =:= 0 ->
{Entries, State};
evaluate(Entries, State) ->
Requests = [
{SessionId, UserId, maps:keys(Partners)}
|| {SessionId, #{user_id := UserId, partners := Partners}} <- maps:to_list(Entries)
],
Results = guild_subscription_mutual_channels:filter_session_member_ids(Requests, State),
{Memo0, Inputs} = view_memo(State),
{Results, Memo} = guild_subscription_mutual_channels:filter_session_member_ids(
Requests, Memo0, State
),
GuildId = maps:get(id, State),
maps:map(
Evaluated = maps:map(
fun(SessionId, Entry) ->
apply_result(GuildId, maps:get(SessionId, Results, []), Entry)
end,
Entries
).
),
{Evaluated, store_view_memo(Inputs, Memo, State)}.
-spec view_memo(guild_state()) ->
{guild_subscription_mutual_channels:view_memo(), term()}.
view_memo(State) ->
Inputs = visibility_inputs(State),
case maps:get(?VIEW_MEMO_KEY, State, undefined) of
#{inputs := Inputs, memo := Memo} when is_map(Memo) -> {Memo, Inputs};
_ -> {#{}, Inputs}
end.
-spec store_view_memo(term(), guild_subscription_mutual_channels:view_memo(), guild_state()) ->
guild_state().
store_view_memo(Inputs, Memo, State) ->
case map_size(maps:get(views, Memo, #{})) > ?VIEW_MEMO_LIMIT of
true -> maps:remove(?VIEW_MEMO_KEY, State);
false -> State#{?VIEW_MEMO_KEY => #{inputs => Inputs, memo => Memo}}
end.
-spec apply_result(integer(), [user_id()], registration()) -> registration().
apply_result(GuildId, EligibleIds, #{pid := Pid, eligible := Previous} = Entry) ->
fluxer_gateway/src/guild/guild_handoff.erl
+23 -5
View File
@@ -5,6 +5,7 @@
-export([
export_handoff_state/1,
derived_data_keys/0,
validate_handoff_state/1,
remonitor_transferred_sessions/1,
restore_transferred_session_state/1
@@ -18,7 +19,7 @@
export_handoff_state(State) ->
#{
id => maps:get(id, State),
data => maps:get(data, State, #{}),
data => maps:without(derived_data_keys(), maps:get(data, State, #{})),
sessions => export_handoff_sessions(maps:get(sessions, State, #{})),
voice_states => maps:get(voice_states, State, #{}),
virtual_channel_access => maps:get(virtual_channel_access, State, #{}),
@@ -30,6 +31,19 @@ export_handoff_state(State) ->
maps:get(virtual_channel_access_move_pending, State, #{})
}.
-spec derived_data_keys() -> [atom() | binary()].
derived_data_keys() ->
[
members_normalized,
members_sorted_ids,
member_list_revision,
members_ets,
role_perms_cache,
overwrite_perms_cache,
<<"role_index">>,
<<"member_role_index">>
].
-spec validate_handoff_state(term()) -> ok | {error, [atom()]}.
validate_handoff_state(Exported) when is_map(Exported) ->
RequiredKeys = [id, data, sessions, voice_states],
@@ -140,11 +154,15 @@ restore_transferred_session(_SessionId, _SessionData, State) ->
State.
-spec active_session_user_id(map()) -> integer() | undefined.
active_session_user_id(#{pending_connect := true}) ->
undefined;
active_session_user_id(SessionData) ->
case {maps:get(user_id, SessionData, undefined), maps:get(pid, SessionData, undefined)} of
{UserId, Pid} when is_integer(UserId), UserId > 0, is_pid(Pid) ->
case
{
guild_sessions_connect:counts_as_connected(SessionData),
maps:get(user_id, SessionData, undefined),
maps:get(pid, SessionData, undefined)
}
of
{true, UserId, Pid} when is_integer(UserId), UserId > 0, is_pid(Pid) ->
UserId;
_ ->
undefined
fluxer_gateway/src/guild/guild_handoff_freeze.erl
+941
View File
@@ -0,0 +1,941 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(guild_handoff_freeze).
-typing([eqwalizer]).
-export([
transfer/5,
repair/3,
default_opts/0,
is_frozen/1,
shard_pid/2,
ensure_absent/3
]).
-define(SHARD_TABLE, guild_manager_shard_table).
-define(GUARD_EXPRS,
"R = erlang:monitor(process, C),"
" C ! {freeze_guard_ready, self()},"
" receive"
" {freeze_guard_release, C} -> erlang:demonitor(R, [flush]), released;"
" {'DOWN', R, process, C, _} ->"
" try sys:resume(P, 60000) of ok -> resumed catch _:Reason -> {resume_failed, Reason} end"
" end."
).
-define(CAPTURE_EXPRS,
"F = fun({'$gen_call', _, _}) -> true;"
" ({'$gen_cast', _}) -> true;"
" ({presence, U, _}) when is_integer(U) -> true;"
" ({reconcile_user_presence, _}) -> true;"
" ({clear_stale_cached_voice_states, _}) -> true;"
" (_) -> false end,"
" case erlang:process_info(P, messages) of"
" {messages, Ms} ->"
" R = [M || M <- Ms, F(M)], L = length(R),"
" {L, case L >= S of true -> lists:nthtail(S, R); false -> [] end};"
" undefined -> undefined"
" end."
).
-define(DRAIN_POLL_MS, 10).
-define(REPAIR_EXPRS,
"D = erlang:monotonic_time(millisecond) + T,"
" case M:fetch_guild_data(G) of"
" {ok, Data} ->"
" case erlang:monotonic_time(millisecond) < D of"
" true ->"
" R = fun(S) -> case A:handle_call({reload, Data}, {self(), make_ref()}, S) of"
" {reply, _, S1} -> S1;"
" {reply, _, S1, _} -> S1"
" end end,"
" _ = sys:replace_state(P, R, max(1, D - erlang:monotonic_time(millisecond))),"
" reloaded;"
" false -> late"
" end;"
" Other -> {fetch_failed, Other}"
" end."
).
-define(EXPORT_EXPRS,
"maps:merge(maps:update_with(data, fun(D) -> maps:without(K, D) end,"
" guild_handoff:export_handoff_state(sys:get_state(P, T))), X)."
).
-type guild_id() :: integer().
-type hook() :: fun(() -> ok | {error, term()}).
-type started_hook() :: fun((pid()) -> ok | {error, term()}).
-type opts() :: #{
suspend_timeout => pos_integer(),
export_timeout => pos_integer(),
start_timeout => pos_integer(),
stop_timeout => pos_integer(),
barrier_timeout => pos_integer(),
rpc_timeout => pos_integer(),
guard_timeout => pos_integer(),
freeze_budget => pos_integer(),
commit_timeout => pos_integer(),
repair_timeout => pos_integer(),
forward_rounds => non_neg_integer(),
transfer_sessions => boolean(),
max_heap_words => non_neg_integer(),
measure => boolean(),
before_start => hook(),
after_start => started_hook(),
on_abort => hook()
}.
-type report() :: map().
-type result() :: {ok, report()} | {error, report()}.
-type freeze() :: #{pid := pid(), guard := pid(), guard_ref := reference()}.
-export_type([opts/0, report/0, result/0]).
-spec default_opts() -> opts().
default_opts() ->
#{
suspend_timeout => 30000,
export_timeout => 120000,
start_timeout => 120000,
stop_timeout => 60000,
barrier_timeout => 30000,
rpc_timeout => 10000,
guard_timeout => 10000,
freeze_budget => 90000,
commit_timeout => 300000,
repair_timeout => 30000,
forward_rounds => 3,
transfer_sessions => true,
max_heap_words => 0,
measure => false,
before_start => fun() -> ok end,
after_start => fun(_Pid) -> ok end,
on_abort => fun() -> ok end
}.
-spec transfer(guild_id(), pid(), pid(), node(), opts()) -> result().
transfer(GuildId, SrcPid, SrcShard, TargetNode, Opts0) ->
Opts = maps:merge(default_opts(), Opts0),
Parent = self(),
Tag = make_ref(),
Notify = fun(Event) ->
Parent ! {Tag, Event},
ok
end,
{Pid, Ref} = spawn_monitor(fun() ->
process_flag(trap_exit, true),
ok = maybe_limit_heap(maps:get(max_heap_words, Opts)),
Parent ! {Tag, run(GuildId, SrcPid, SrcShard, TargetNode, Opts, Notify)}
end),
await(GuildId, TargetNode, {Pid, Ref, Tag}, false, Opts).
-spec await(guild_id(), node(), {pid(), reference(), reference()}, boolean(), opts()) ->
result().
await(GuildId, TargetNode, {Pid, Ref, Tag} = Run, Committed, Opts) ->
receive
{Tag, committed} ->
await(GuildId, TargetNode, Run, true, Opts);
{Tag, Result} ->
erlang:demonitor(Ref, [flush]),
Result;
{'DOWN', Ref, process, Pid, Reason} ->
crashed(GuildId, TargetNode, Reason, Committed, Opts)
end.
-spec crashed(guild_id(), node(), term(), boolean(), opts()) -> result().
crashed(_GuildId, _TargetNode, Reason, true, _Opts) ->
{error, #{phase => crashed, reason => Reason, committed => true}};
crashed(GuildId, TargetNode, Reason, false, Opts) ->
Routes = safe_hook(maps:get(on_abort, Opts)),
Target = ensure_absent(GuildId, TargetNode, Opts),
{error, #{
phase => crashed, reason => Reason, abort => #{routes => Routes, target => Target}
}}.
-spec repair(guild_id(), pid(), opts()) -> ok | {error, term()}.
repair(GuildId, SrcPid, Opts0) ->
Opts = maps:merge(default_opts(), Opts0),
maybe
ok ?= await_thawed(SrcPid, now_ms() + maps:get(guard_timeout, Opts)),
{ok, Freeze} ?= freeze(SrcPid, Opts),
Reload = reload_frozen(GuildId, SrcPid, maps:get(repair_timeout, Opts)),
repaired(Reload, thaw(Freeze, Opts))
end.
-spec await_thawed(pid(), integer()) -> ok | {error, term()}.
await_thawed(SrcPid, Deadline) ->
case is_frozen(SrcPid) of
false ->
ok;
true ->
case now_ms() < Deadline of
true ->
timer:sleep(?DRAIN_POLL_MS),
await_thawed(SrcPid, Deadline);
false ->
{error, still_frozen}
end;
{error, _} = Error ->
Error
end.
-spec reload_frozen(guild_id(), pid(), pos_integer()) -> ok | {error, term()}.
reload_frozen(GuildId, SrcPid, Timeout) ->
Bindings = bindings([
{'G', GuildId},
{'P', SrcPid},
{'T', Timeout},
{'M', guild_manager_shard_fetch},
{'A', guild}
]),
Args = [parse(?REPAIR_EXPRS), Bindings],
case rpc_call(node(SrcPid), erl_eval, exprs, Args, Timeout + 5000) of
{ok, {value, reloaded, _}} -> ok;
{ok, {value, Other, _}} -> {error, Other};
{error, _} = Error -> Error
end.
-spec repaired(ok | {error, term()}, ok | {error, term()}) -> ok | {error, term()}.
repaired(ok, ok) ->
ok;
repaired(Reload, Thaw) ->
{error, #{reload => Reload, thaw => Thaw}}.
-spec is_frozen(pid()) -> boolean() | {error, term()}.
is_frozen(Pid) ->
case rpc_call(node(Pid), erlang, process_info, [Pid, current_function], 10000) of
{ok, {current_function, {sys, _, _}}} -> true;
{ok, {current_function, _}} -> false;
{ok, undefined} -> {error, noproc};
{ok, Other} -> {error, {unexpected, Other}};
{error, _} = Error -> Error
end.
-spec shard_pid(guild_id(), node()) -> {ok, pid()} | {error, term()}.
shard_pid(GuildId, Node) ->
case rpc_call(Node, ets, lookup, [?SHARD_TABLE, shard_count], 10000) of
{ok, [{shard_count, Count}]} when is_integer(Count), Count > 0 ->
Index = guild_manager_shards:select_shard(GuildId, Count),
indexed_shard_pid(Node, Index);
{ok, _} ->
{error, shard_table_unavailable};
{error, _} = Error ->
Error
end.
-spec ensure_absent(guild_id(), node(), opts()) -> ok | {error, term()}.
ensure_absent(GuildId, Node, Opts0) ->
Opts = maps:merge(default_opts(), Opts0),
StopTimeout = maps:get(start_timeout, Opts) + maps:get(stop_timeout, Opts),
case shard_pid(GuildId, Node) of
{ok, Shard} ->
_ = safe_call(Shard, {stop_guild, GuildId}, StopTimeout),
case safe_call(Shard, {lookup, GuildId}, maps:get(rpc_timeout, Opts)) of
{error, not_found} -> ok;
{ok, Pid} when is_pid(Pid) -> {error, {still_running, Pid}};
Other -> {error, {unverified, Other}}
end;
{error, _} = Error ->
Error
end.
-spec run(guild_id(), pid(), pid(), node(), opts(), fun((committed) -> ok)) -> result().
run(GuildId, SrcPid, SrcShard, TargetNode, Opts, Notify) ->
Ctx = #{
notify => Notify,
guild_id => GuildId,
src => SrcPid,
src_shard => SrcShard,
target => TargetNode,
opts => Opts,
t0 => now_ms(),
report => #{source => SrcPid, target_node => TargetNode}
},
case is_frozen(SrcPid) of
false -> run_freeze(Ctx);
true -> fail(preflight, already_frozen, Ctx);
{error, Reason} -> fail(preflight, Reason, Ctx)
end.
-spec run_freeze(map()) -> result().
run_freeze(#{src := SrcPid, opts := Opts} = Ctx) ->
T0 = now_ms(),
case freeze(SrcPid, Opts) of
{ok, Freeze} ->
Ctx1 = put_ms(suspend_ms, T0, Ctx#{freeze => Freeze, frozen_at => now_ms()}),
run_export(Ctx1);
{error, Reason} ->
fail(freeze, Reason, Ctx)
end.
-spec run_export(map()) -> result().
run_export(#{src := SrcPid, opts := Opts} = Ctx) ->
T0 = now_ms(),
case export(SrcPid, Opts) of
{ok, Export} ->
Ctx1 = put_ms(export_ms, T0, Ctx),
Ctx2 = put_report(export_measure, measure_export(Export, Opts), Ctx1),
run_before_start(Export, Ctx2);
{error, Reason} ->
abort(export, Reason, Ctx)
end.
-spec run_before_start(map(), map()) -> result().
run_before_start(Export, #{opts := Opts} = Ctx) ->
case {within_budget(Ctx), guard_held(Ctx)} of
{true, ok} -> run_hook(before_start, maps:get(before_start, Opts), Export, Ctx);
{false, _} -> abort(before_start, freeze_budget_exceeded, Ctx);
{true, {error, Reason}} -> abort(before_start, Reason, Ctx)
end.
-spec run_hook(before_start, hook(), map(), map()) -> result().
run_hook(Phase, Hook, Export, Ctx) ->
case safe_hook(Hook) of
ok -> run_start(Export, Ctx#{routed => true});
{error, Reason} -> abort(Phase, Reason, Ctx#{routed => true})
end.
-spec run_start(map(), map()) -> result().
run_start(Export, #{guild_id := GuildId, target := Target, opts := Opts} = Ctx) ->
T0 = now_ms(),
Ctx1 = Ctx#{start_sent => true},
case start_on(GuildId, Target, Export, maps:get(start_timeout, Opts)) of
{ok, NewPid} ->
run_after_start(NewPid, put_ms(start_ms, T0, Ctx1#{new_pid => NewPid}));
{error, Reason} ->
abort(start, Reason, put_ms(start_ms, T0, Ctx1))
end.
-spec run_after_start(pid(), map()) -> result().
run_after_start(NewPid, #{opts := Opts} = Ctx) ->
case guard_held(Ctx) of
ok -> run_route(NewPid, maps:get(after_start, Opts), Ctx#{exposed => true});
{error, Reason} -> abort(after_start, Reason, Ctx)
end.
-spec run_route(pid(), started_hook(), map()) -> result().
run_route(NewPid, Hook, Ctx) ->
T0 = now_ms(),
case safe_started_hook(Hook, NewPid) of
ok -> run_commit(NewPid, put_ms(route_ms, T0, Ctx));
{error, Reason} -> abort(after_start, Reason, put_ms(route_ms, T0, Ctx))
end.
-spec run_commit(pid(), map()) -> result().
run_commit(NewPid, #{src := SrcPid, opts := Opts} = Ctx) ->
case forward_rounds(SrcPid, NewPid, maps:get(forward_rounds, Opts), 0, [], Opts) of
{ok, Seen, Rounds} ->
commit_final(NewPid, Seen, rounds_report(Rounds, Ctx));
{error, {capture_failed, noproc}, _Seen, Rounds} ->
source_gone(NewPid, rounds_report(Rounds, Ctx));
{error, {capture_failed, _} = Reason, Seen, Rounds} ->
Ctx1 = put_report(rounds_stopped, Reason, rounds_report(Rounds, Ctx)),
commit_final(NewPid, Seen, Ctx1);
{error, Reason, _Seen, Rounds} ->
abort(forward, Reason, rounds_report(Rounds, Ctx))
end.
-spec rounds_report([map()], map()) -> map().
rounds_report(Rounds, Ctx) ->
Forwarded = lists:foldl(fun(Round, Acc) -> Acc + forwarded_count(Round) end, 0, Rounds),
add_forwarded(Forwarded, put_report(rounds, lists:reverse(Rounds), Ctx)).
-spec add_forwarded(non_neg_integer(), map()) -> map().
add_forwarded(Count, Ctx) ->
Ctx#{forwarded => maps:get(forwarded, Ctx, 0) + Count}.
-spec forwarded_count(map()) -> non_neg_integer().
forwarded_count(Stats) ->
lists:foldl(
fun(Kind, Acc) ->
case maps:get(Kind, Stats, 0) of
Count when is_integer(Count), Count > 0 -> Acc + Count;
_ -> Acc
end
end,
0,
[call, cast, presence_repair, voice_cleanup]
).
-spec commit_final(pid(), non_neg_integer(), map()) -> result().
commit_final(NewPid, Seen, #{src := SrcPid} = Ctx) ->
case {target_exit(NewPid), guard_held(Ctx), is_frozen(SrcPid)} of
{{exited, Reason}, _, _} ->
abort(target_died, Reason, Ctx);
{alive, {error, Reason}, _} ->
abort(final_capture, Reason, Ctx);
{alive, ok, true} ->
stop_and_drain(NewPid, Seen, Ctx);
{alive, ok, {error, noproc}} ->
source_gone(NewPid, Ctx);
{alive, ok, Frozen} ->
abort(final_capture, {not_frozen, Frozen}, Ctx)
end.
-spec guard_held(map()) -> ok | {error, term()}.
guard_held(#{freeze := #{guard := Guard, guard_ref := Ref}}) ->
receive
{'DOWN', Ref, process, Guard, Reason} -> {error, {guard_lost, Reason}}
after 0 -> ok
end.
-spec target_exit(pid()) -> alive | {exited, term()}.
target_exit(NewPid) ->
receive
{'EXIT', NewPid, Reason} -> {exited, Reason}
after 0 -> alive
end.
-spec source_gone(pid(), map()) -> result().
source_gone(NewPid, #{src := SrcPid, opts := Opts} = Ctx) ->
case {target_exit(NewPid), source_alive(SrcPid, Opts)} of
{alive, false} -> finish(NewPid, put_report(source_gone, true, Ctx));
Other -> abort(source_gone, {source_state, Other}, Ctx)
end.
-spec stop_and_drain(pid(), non_neg_integer(), map()) -> result().
stop_and_drain(NewPid, Seen, #{src := SrcPid, opts := Opts} = Ctx) ->
case capture(SrcPid, Seen, Opts) of
{ok, _Len, New} ->
Stats = forward(New, NewPid, final),
ok = commit(NewPid, Ctx),
Parent = self(),
Tag = make_ref(),
{Pid, Ref} = spawn_monitor(fun() -> Parent ! {Tag, stop_source(Ctx)} end),
Drain = #{
stopper => {Pid, Ref, Tag},
stop => pending,
stats => Stats,
captures => 1,
t0 => now_ms()
},
drain(NewPid, Seen + length(New), Drain, Ctx);
{error, noproc} ->
source_gone(NewPid, Ctx);
{error, Reason} ->
abort(final_capture, Reason, Ctx)
end.
-spec commit(pid(), map()) -> ok.
commit(NewPid, #{freeze := Freeze, notify := Notify}) ->
release_guard(Freeze),
unlink(NewPid),
receive
{'EXIT', NewPid, _Reason} -> ok
after 0 -> ok
end,
Notify(committed).
-spec drain(pid(), non_neg_integer(), map(), map()) -> result().
drain(NewPid, Seen, Drain, #{src := SrcPid, opts := Opts} = Ctx) ->
case capture(SrcPid, Seen, Opts) of
{ok, _Len, New} ->
Stats = merge_counts(maps:get(stats, Drain), forward(New, NewPid, final)),
Drain1 = Drain#{stats => Stats, captures => maps:get(captures, Drain) + 1},
drain_alive(NewPid, Seen + length(New), poll_stopper(Drain1, ?DRAIN_POLL_MS), Ctx);
{error, noproc} ->
drained(NewPid, poll_stopper(Drain, maps:get(stop_timeout, Opts)), Ctx);
{error, Reason} ->
Polled = poll_stopper(Drain, maps:get(stop_timeout, Opts)),
drain_unknown(NewPid, Seen, Reason, Polled, Ctx)
end.
-spec drain_alive(pid(), non_neg_integer(), map(), map()) -> result().
drain_alive(NewPid, Seen, #{stop := pending} = Drain, Ctx) ->
drain(NewPid, Seen, Drain, Ctx);
drain_alive(NewPid, Seen, #{stop := ok} = Drain, #{src := SrcPid, opts := Opts} = Ctx) ->
case now_ms() - maps:get(t0, Drain) > maps:get(stop_timeout, Opts) of
true -> exit(SrcPid, kill);
false -> timer:sleep(?DRAIN_POLL_MS)
end,
drain(NewPid, Seen, Drain, Ctx);
drain_alive(NewPid, Seen, #{stop := Reason} = Drain, Ctx) ->
await_source(NewPid, Seen, Reason, Drain, Ctx).
-spec drain_unknown(pid(), non_neg_integer(), term(), map(), map()) -> result().
drain_unknown(NewPid, Seen, Reason, Drain, #{src := SrcPid, opts := Opts} = Ctx) ->
case source_alive(SrcPid, Opts) of
false ->
drained(NewPid, Drain, Ctx);
_ ->
await_source(
NewPid, Seen, {capture_failed, Reason, maps:get(stop, Drain)}, Drain, Ctx
)
end.
-spec await_source(pid(), non_neg_integer(), term(), map(), map()) -> result().
await_source(NewPid, Seen, Reason, Drain, #{opts := Opts} = Ctx) ->
case now_ms() - maps:get(t0, Drain) > maps:get(commit_timeout, Opts) of
true ->
stuck(NewPid, Reason, Drain, Ctx);
false ->
timer:sleep(?DRAIN_POLL_MS),
drain(NewPid, Seen, Drain, Ctx)
end.
-spec stuck(pid(), term(), map(), map()) -> result().
stuck(NewPid, Reason, Drain, Ctx) ->
Forwarded = maps:get(forwarded, Ctx, 0) + forwarded_count(maps:get(stats, Drain)),
Report = maps:get(report, Ctx),
{error, Report#{
phase => stop,
reason => Reason,
committed => true,
new_pid => NewPid,
forwarded => Forwarded,
total_ms => now_ms() - maps:get(t0, Ctx)
}}.
-spec drained(pid(), map(), map()) -> result().
drained(NewPid, Drain, Ctx) ->
Final = (maps:get(stats, Drain))#{
captures => maps:get(captures, Drain),
stop => maps:get(stop, Drain)
},
Ctx1 = put_report(stop_ms, now_ms() - maps:get(t0, Drain), Ctx),
finish(NewPid, put_report(final, Final, Ctx1)).
-spec poll_stopper(map(), non_neg_integer()) -> map().
poll_stopper(#{stop := pending, stopper := {Pid, Ref, Tag}} = Drain, Timeout) ->
receive
{Tag, Result} ->
erlang:demonitor(Ref, [flush]),
Drain#{stop => Result};
{'DOWN', Ref, process, Pid, Reason} ->
Drain#{stop => {error, {stopper_crashed, Reason}}}
after Timeout ->
Drain
end;
poll_stopper(Drain, _Timeout) ->
Drain.
-spec merge_counts(map(), map()) -> map().
merge_counts(Left, Right) ->
maps:merge_with(fun(_Key, A, B) -> A + B end, Left, Right).
-spec finish(pid(), map()) -> result().
finish(NewPid, #{freeze := Freeze, frozen_at := FrozenAt} = Ctx) ->
release_guard(Freeze),
unlink(NewPid),
Report = maps:get(report, Ctx),
{ok, Report#{
new_pid => NewPid,
frozen_ms => now_ms() - FrozenAt,
total_ms => now_ms() - maps:get(t0, Ctx)
}}.
-spec fail(atom(), term(), map()) -> result().
fail(Phase, Reason, Ctx) ->
Report = maps:get(report, Ctx),
{error, Report#{phase => Phase, reason => Reason, total_ms => now_ms() - maps:get(t0, Ctx)}}.
-spec abort(atom(), term(), map()) -> result().
abort(Phase, Reason, #{opts := Opts} = Ctx) ->
Routes = abort_routes(Ctx, maps:get(on_abort, Opts)),
Target = abort_target(Ctx),
Thaw = thaw(maps:get(freeze, Ctx), Opts),
Report = maps:get(report, Ctx),
{error, Report#{
phase => Phase,
reason => Reason,
abort => #{
routes => Routes,
target => Target,
thaw => Thaw,
exposed => maps:get(exposed, Ctx, false),
forwarded => maps:get(forwarded, Ctx, 0)
},
total_ms => now_ms() - maps:get(t0, Ctx)
}}.
-spec abort_routes(map(), hook()) -> ok | {error, term()} | skipped.
abort_routes(#{routed := true}, Hook) ->
safe_hook(Hook);
abort_routes(_Ctx, _Hook) ->
skipped.
-spec abort_target(map()) -> ok | {error, term()} | skipped.
abort_target(#{start_sent := true, guild_id := GuildId, target := Target, opts := Opts} = Ctx) ->
Result = ensure_absent(GuildId, Target, Opts),
case maps:get(new_pid, Ctx, undefined) of
Pid when is_pid(Pid) -> unlink_and_kill_if(Result, Pid);
_ -> Result
end;
abort_target(_Ctx) ->
skipped.
-spec unlink_and_kill_if(ok | {error, term()}, pid()) -> ok | {error, term()}.
unlink_and_kill_if(ok, Pid) ->
unlink(Pid),
ok;
unlink_and_kill_if({error, _} = Error, Pid) ->
unlink(Pid),
exit(Pid, kill),
Error.
-spec freeze(pid(), opts()) -> {ok, freeze()} | {error, term()}.
freeze(SrcPid, Opts) ->
case spawn_guard(SrcPid, maps:get(guard_timeout, Opts)) of
{ok, Guard, GuardRef} ->
Freeze = #{pid => SrcPid, guard => Guard, guard_ref => GuardRef},
case suspend(SrcPid, maps:get(suspend_timeout, Opts)) of
ok ->
{ok, Freeze};
{error, Reason} ->
_ = thaw(Freeze, Opts),
{error, {suspend_failed, Reason}}
end;
{error, Reason} ->
{error, {guard_failed, Reason}}
end.
-spec thaw(freeze(), opts()) -> ok | {error, term()}.
thaw(#{pid := SrcPid} = Freeze, Opts) ->
Result =
try sys:resume(SrcPid, maps:get(suspend_timeout, Opts)) of
ok -> ok
catch
exit:Reason -> {error, Reason}
end,
release_guard(Freeze),
Result.
-spec suspend(pid(), pos_integer()) -> ok | {error, term()}.
suspend(SrcPid, Timeout) ->
try sys:suspend(SrcPid, Timeout) of
ok -> ok
catch
exit:Reason -> {error, Reason}
end.
-spec spawn_guard(pid(), pos_integer()) -> {ok, pid(), reference()} | {error, term()}.
spawn_guard(SrcPid, Timeout) ->
Bindings = bindings([{'C', self()}, {'P', SrcPid}]),
Guard = erlang:spawn(node(SrcPid), erl_eval, exprs, [parse(?GUARD_EXPRS), Bindings]),
Ref = erlang:monitor(process, Guard),
receive
{freeze_guard_ready, Guard} -> {ok, Guard, Ref};
{'DOWN', Ref, process, Guard, Reason} -> {error, Reason}
after Timeout ->
erlang:demonitor(Ref, [flush]),
exit(Guard, kill),
{error, guard_timeout}
end.
-spec release_guard(freeze()) -> ok.
release_guard(#{guard := Guard, guard_ref := Ref}) ->
erlang:demonitor(Ref, [flush]),
Guard ! {freeze_guard_release, self()},
ok.
-spec export(pid(), opts()) -> {ok, map()} | {error, term()}.
export(SrcPid, Opts) ->
Timeout = maps:get(export_timeout, Opts),
Bindings = bindings([
{'P', SrcPid},
{'T', Timeout},
{'K', guild_handoff:derived_data_keys()},
{'X', export_overrides(Opts)}
]),
Args = [parse(?EXPORT_EXPRS), Bindings],
case rpc_call(node(SrcPid), erl_eval, exprs, Args, Timeout + 5000) of
{ok, {value, Export, _}} -> validate_export(Export);
{ok, Other} -> {error, {unexpected_export, Other}};
{error, _} = Error -> Error
end.
-spec export_overrides(opts()) -> map().
export_overrides(#{transfer_sessions := false}) ->
#{sessions => #{}};
export_overrides(_Opts) ->
#{}.
-spec validate_export(term()) -> {ok, map()} | {error, term()}.
validate_export(Export) when is_map(Export) ->
case guild_handoff:validate_handoff_state(Export) of
ok -> {ok, Export};
{error, Missing} -> {error, {invalid_export, Missing}}
end;
validate_export(Other) ->
{error, {invalid_export, Other}}.
-spec measure_export(map(), opts()) -> map() | skipped.
measure_export(Export, #{measure := true}) ->
T0 = now_ms(),
Bytes = erlang:external_size(Export),
#{external_bytes => Bytes, measure_ms => now_ms() - T0};
measure_export(_Export, _Opts) ->
skipped.
-spec start_on(guild_id(), node(), map(), pos_integer()) -> {ok, pid()} | {error, term()}.
start_on(GuildId, Target, Export, Timeout) ->
case shard_pid(GuildId, Target) of
{ok, Shard} ->
case safe_call(Shard, {start_transferred, GuildId, Export}, Timeout) of
{ok, NewPid} when is_pid(NewPid) ->
link(NewPid),
{ok, NewPid};
{error, Reason} ->
{error, Reason};
Other ->
{error, {unexpected_start_reply, Other}}
end;
{error, _} = Error ->
Error
end.
-spec forward_rounds(pid(), pid(), non_neg_integer(), non_neg_integer(), [map()], opts()) ->
{ok, non_neg_integer(), [map()]} | {error, term(), non_neg_integer(), [map()]}.
forward_rounds(_SrcPid, _NewPid, 0, Seen, Rounds, _Opts) ->
{ok, Seen, Rounds};
forward_rounds(SrcPid, NewPid, Left, Seen, Rounds, Opts) ->
case capture(SrcPid, Seen, Opts) of
{ok, Len, _New} when Len < Seen ->
{error, mailbox_shrank, Seen, Rounds};
{ok, _Len, []} ->
{ok, Seen, Rounds};
{ok, _Len, New} ->
Stats = forward(New, NewPid, rounds),
T0 = now_ms(),
Barrier = barrier(NewPid, maps:get(barrier_timeout, Opts)),
Round = Stats#{
captured => length(New), barrier => Barrier, barrier_ms => now_ms() - T0
},
forward_rounds(
SrcPid, NewPid, Left - 1, Seen + length(New), [Round | Rounds], Opts
);
{error, Reason} ->
{error, {capture_failed, Reason}, Seen, Rounds}
end.
-spec forward([term()], pid(), rounds | final) -> map().
forward(Msgs, NewPid, Phase) ->
lists:foldl(
fun(Msg, Acc) ->
Kind = forward_msg(Msg, NewPid, Phase),
maps:update_with(Kind, fun(N) -> N + 1 end, 1, Acc)
end,
#{},
Msgs
).
-spec forward_msg(term(), pid(), rounds | final) -> atom().
forward_msg({'$gen_call', _From, _Request} = Msg, NewPid, rounds) ->
NewPid ! Msg,
call;
forward_msg({'$gen_call', _From, _Request}, _NewPid, final) ->
dropped_call;
forward_msg({'$gen_cast', {session_connect_worker_done, _, _, _, _}}, _NewPid, _Phase) ->
dropped_worker_result;
forward_msg({'$gen_cast', {session_connect_worker_batch_done, _}}, _NewPid, _Phase) ->
dropped_worker_result;
forward_msg({'$gen_cast', _Request} = Msg, NewPid, _Phase) ->
NewPid ! Msg,
cast;
forward_msg({presence, UserId, _Payload}, NewPid, _Phase) ->
NewPid ! {reconcile_user_presence, UserId},
presence_repair;
forward_msg({reconcile_user_presence, _UserId} = Msg, NewPid, _Phase) ->
NewPid ! Msg,
presence_repair;
forward_msg({clear_stale_cached_voice_states, _Ids} = Msg, NewPid, _Phase) ->
NewPid ! Msg,
voice_cleanup.
-spec barrier(pid(), pos_integer()) -> ok | {error, term()}.
barrier(NewPid, Timeout) ->
case safe_call(NewPid, {get_guild_id}, Timeout) of
Id when is_integer(Id) -> ok;
Other -> {error, Other}
end.
-spec capture(pid(), non_neg_integer(), opts()) ->
{ok, non_neg_integer(), [term()]} | {error, term()}.
capture(SrcPid, Seen, Opts) ->
Args = [parse(?CAPTURE_EXPRS), bindings([{'P', SrcPid}, {'S', Seen}])],
case rpc_call(node(SrcPid), erl_eval, exprs, Args, maps:get(rpc_timeout, Opts)) of
{ok, {value, {Len, New}, _}} when is_integer(Len), is_list(New) -> {ok, Len, New};
{ok, {value, undefined, _}} -> {error, noproc};
{ok, Other} -> {error, {unexpected, Other}};
{error, _} = Error -> Error
end.
-spec stop_source(map()) -> ok | {error, term()}.
stop_source(#{guild_id := GuildId, src_shard := Shard, opts := Opts}) ->
Request = {stop_guild, GuildId, {shutdown, handoff}},
case safe_call(Shard, Request, maps:get(stop_timeout, Opts)) of
ok -> ok;
Other -> {error, Other}
end.
-spec source_alive(pid(), opts()) -> boolean() | {error, term()}.
source_alive(SrcPid, Opts) ->
Args = [SrcPid],
case rpc_call(node(SrcPid), erlang, is_process_alive, Args, maps:get(rpc_timeout, Opts)) of
{ok, Alive} when is_boolean(Alive) -> Alive;
{ok, Other} -> {error, {unexpected, Other}};
{error, _} = Error -> Error
end.
-spec within_budget(map()) -> boolean().
within_budget(#{frozen_at := FrozenAt, opts := Opts}) ->
now_ms() - FrozenAt =< maps:get(freeze_budget, Opts).
-spec indexed_shard_pid(node(), non_neg_integer()) -> {ok, pid()} | {error, term()}.
indexed_shard_pid(Node, Index) ->
case rpc_call(Node, ets, lookup, [?SHARD_TABLE, {shard_pid, Index}], 10000) of
{ok, [{{shard_pid, Index}, Pid}]} when is_pid(Pid) -> {ok, Pid};
{ok, _} -> {error, shard_unavailable};
{error, _} = Error -> Error
end.
-spec safe_hook(hook()) -> ok | {error, term()}.
safe_hook(Hook) ->
try Hook() of
ok -> ok;
{error, _} = Error -> Error
catch
Class:Reason -> {error, {hook_crashed, Class, Reason}}
end.
-spec safe_started_hook(started_hook(), pid()) -> ok | {error, term()}.
safe_started_hook(Hook, NewPid) ->
try Hook(NewPid) of
ok -> ok;
{error, _} = Error -> Error
catch
Class:Reason -> {error, {hook_crashed, Class, Reason}}
end.
-spec safe_call(pid(), term(), pos_integer()) -> term().
safe_call(Server, Request, Timeout) ->
try
gen_server:call(Server, Request, Timeout)
catch
exit:{timeout, _} -> {error, timeout};
exit:Reason -> {error, {exit, Reason}}
end.
-spec rpc_call(node(), module(), atom(), [term()], pos_integer()) ->
{ok, term()} | {error, term()}.
rpc_call(Node, Module, Function, Args, Timeout) ->
try erpc:call(Node, Module, Function, Args, Timeout) of
Result -> {ok, Result}
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec parse(string()) -> [erl_parse:abstract_expr()].
parse(Source) ->
{ok, Tokens, _} = erl_scan:string(Source),
{ok, Exprs} = erl_parse:parse_exprs(Tokens),
Exprs.
-spec bindings([{atom(), term()}]) -> erl_eval:binding_struct().
bindings(Pairs) ->
lists:foldl(
fun({Name, Value}, Acc) -> erl_eval:add_binding(Name, Value, Acc) end,
erl_eval:new_bindings(),
Pairs
).
-spec maybe_limit_heap(non_neg_integer()) -> ok.
maybe_limit_heap(0) ->
ok;
maybe_limit_heap(Words) ->
_ = process_flag(max_heap_size, #{size => Words, kill => true, error_logger => true}),
ok.
-spec put_ms(atom(), integer(), map()) -> map().
put_ms(Key, T0, Ctx) ->
put_report(Key, now_ms() - T0, Ctx).
-spec put_report(atom(), term(), map()) -> map().
put_report(Key, Value, #{report := Report} = Ctx) ->
Ctx#{report => Report#{Key => Value}}.
-spec now_ms() -> integer().
now_ms() ->
erlang:monotonic_time(millisecond).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
capture_returns_only_new_forwardable_messages_test() ->
Pid = spawn(fun() ->
receive
stop -> ok
end
end),
Msgs = [
{'$gen_cast', a},
{'DOWN', make_ref(), process, self(), normal},
{'$gen_call', {self(), make_ref()}, b},
{presence, not_a_user, x},
{presence, 7, x},
{reconcile_user_presence, 7},
{clear_stale_cached_voice_states, []}
],
[Pid ! Msg || Msg <- Msgs],
Opts = default_opts(),
try
?assertMatch({ok, 5, [_, _, _, _, _]}, capture(Pid, 0, Opts)),
?assertEqual(
{ok, 5, [
{presence, 7, x},
{reconcile_user_presence, 7},
{clear_stale_cached_voice_states, []}
]},
capture(Pid, 2, Opts)
),
?assertEqual({ok, 5, []}, capture(Pid, 5, Opts)),
?assertEqual({ok, 5, []}, capture(Pid, 9, Opts))
after
Pid ! stop
end,
ok = wait_exit(Pid),
?assertEqual({error, noproc}, capture(Pid, 0, Opts)).
repair_applies_the_reload_before_dispatches_queued_during_the_fetch_test() ->
Test = self(),
meck:new(guild_repair_stub, [non_strict]),
meck:expect(guild_repair_stub, init, fun(State) -> {ok, State} end),
meck:expect(guild_repair_stub, handle_cast, fun({set, Value}, _State) ->
{noreply, Value}
end),
meck:new(guild_manager_shard_fetch, [passthrough]),
meck:expect(guild_manager_shard_fetch, fetch_guild_data, fun(42) ->
Test ! {fetching, self()},
receive
go -> {ok, fetched}
end
end),
meck:new(guild, [passthrough]),
meck:expect(guild, handle_call, fun({reload, Data}, _From, _State) -> {reply, ok, Data} end),
{ok, Pid} = gen_server:start(guild_repair_stub, before, []),
try
spawn(fun() -> Test ! {repaired, repair(42, Pid, #{})} end),
receive
{fetching, Fetcher} ->
?assert(is_frozen(Pid)),
gen_server:cast(Pid, {set, dispatched}),
Fetcher ! go
after 5000 -> error(no_fetch)
end,
?assertEqual(
ok,
receive
{repaired, Result} -> Result
after 5000 -> timeout
end
),
?assertEqual(dispatched, sys:get_state(Pid))
after
exit(Pid, kill),
meck:unload([guild, guild_manager_shard_fetch, guild_repair_stub])
end.
wait_exit(Pid) ->
Ref = erlang:monitor(process, Pid),
receive
{'DOWN', Ref, process, Pid, _} -> ok
after 5000 -> timeout
end.
-endif.
fluxer_gateway/src/guild/guild_health.erl
+111 -34
View File
@@ -6,7 +6,7 @@
-export([
start_link/0,
is_degraded/1,
is_overloaded/1,
register_guild/1,
put_session/2,
remove_session/2,
@@ -19,19 +19,23 @@
-define(DEGRADED_MS, 2000).
-define(RECOVERED_MS, 500).
-define(REMOTE_LOOKUP_TIMEOUT_MS, 250).
-define(PROBE_TIMEOUT_MS, 5000).
-define(STALE_PROBE_MS, (?PROBE_TIMEOUT_MS + 4 * ?INTERVAL_MS)).
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-spec is_degraded(pid()) -> boolean().
is_degraded(Pid) when node(Pid) =:= node() ->
-spec is_overloaded(pid()) -> boolean().
is_overloaded(Pid) when node(Pid) =:= node() ->
case lookup(Pid) of
{Pid, _GuildId, Degraded, _Targets, _Pending} -> Degraded;
undefined -> false
{Pid, _GuildId, _Degraded, _Targets, Pending} ->
overloaded(Pending, erlang:monotonic_time(millisecond));
undefined ->
false
end;
is_degraded(Pid) ->
try erpc:call(node(Pid), ?MODULE, is_degraded, [Pid], ?REMOTE_LOOKUP_TIMEOUT_MS) of
is_overloaded(Pid) ->
try erpc:call(node(Pid), ?MODULE, is_overloaded, [Pid], ?REMOTE_LOOKUP_TIMEOUT_MS) of
true -> true;
_ -> false
catch
@@ -119,14 +123,8 @@ handle_call(_, _, State) ->
handle_cast({register, Pid}, State) when is_pid(Pid), node(Pid) =:= node() ->
{noreply, track_guild(Pid, State)};
handle_cast({pong, Pid, Ref, HandledAt}, State) when is_pid(Pid), is_integer(HandledAt) ->
case lookup(Pid) of
{Pid, GuildId, Degraded, Targets, {Ref, SentAt}} ->
NewDegraded = next_degraded(Degraded, max(0, HandledAt - SentAt)),
update({Pid, GuildId, Degraded, Targets, undefined}, NewDegraded),
{noreply, State};
_ ->
{noreply, State}
end;
ok = pong(Pid, Ref, HandledAt, erlang:monotonic_time(millisecond)),
{noreply, State};
handle_cast({current, Pid, SessionPid}, State) when is_pid(Pid), is_pid(SessionPid) ->
case lookup(Pid) of
{Pid, GuildId, Degraded, _Targets, _Pending} ->
@@ -186,20 +184,49 @@ check_guild(Pid, Now) ->
case lookup(Pid) of
{Pid, GuildId, Degraded, Targets, undefined} ->
case should_probe(Pid, Degraded) of
true ->
Ref = make_ref(),
ets:insert(?TABLE, {Pid, GuildId, Degraded, Targets, {Ref, Now}}),
Pid ! {guild_health_probe, Ref},
ok;
false ->
ok
true -> update({Pid, GuildId, Degraded, Targets, probe(Pid, Now, 0)}, Degraded);
false -> ok
end;
{Pid, _, Degraded, _, {_Ref, SentAt}} = Entry ->
{Pid, GuildId, Degraded, Targets, {_Ref, SentAt, Lag}} when
Now - SentAt >= ?PROBE_TIMEOUT_MS
->
Pending = probe(Pid, Now, max(Lag, Now - SentAt)),
update({Pid, GuildId, Degraded, Targets, Pending}, true);
{Pid, _, Degraded, _, {_Ref, SentAt, _Lag}} = Entry ->
update(Entry, Degraded orelse Now - SentAt > ?DEGRADED_MS);
_ ->
ok
end.
-spec pong(pid(), reference(), integer(), integer()) -> ok.
pong(Pid, Ref, HandledAt, Now) ->
case lookup(Pid) of
{Pid, GuildId, Degraded, Targets, {Ref, SentAt, _Lag}} ->
Delay = max(0, HandledAt - SentAt),
NewDegraded = next_degraded(Degraded, Delay),
Pending = next_pending(Pid, NewDegraded, Delay, Now),
update({Pid, GuildId, Degraded, Targets, Pending}, NewDegraded);
_ ->
ok
end.
-spec next_pending(pid(), boolean(), non_neg_integer(), integer()) ->
{reference(), integer(), non_neg_integer()} | undefined.
next_pending(Pid, true, Delay, Now) -> probe(Pid, Now, Delay);
next_pending(_Pid, false, _Delay, _Now) -> undefined.
-spec probe(pid(), integer(), non_neg_integer()) -> {reference(), integer(), non_neg_integer()}.
probe(Pid, Now, Lag) ->
Ref = make_ref(),
Pid ! {guild_health_probe, Ref},
{Ref, Now, Lag}.
-spec overloaded(term(), integer()) -> boolean().
overloaded({_Ref, SentAt, Lag}, Now) when Now - SentAt =< ?STALE_PROBE_MS ->
max(Lag, Now - SentAt) > ?DEGRADED_MS;
overloaded(_Pending, _Now) ->
false.
-spec should_probe(pid(), boolean()) -> boolean().
should_probe(_Pid, true) ->
true;
@@ -273,7 +300,7 @@ hysteresis_test() ->
unreachable_remote_pid_is_unknown_test() ->
Pid = binary_to_term(<<131, 88, 119, 12, "fake@nowhere", 1:32, 0:32, 1:32>>),
?assertNotEqual(node(), node(Pid)),
{ElapsedUs, Result} = timer:tc(?MODULE, is_degraded, [Pid]),
{ElapsedUs, Result} = timer:tc(?MODULE, is_overloaded, [Pid]),
?assertNot(Result),
?assert(ElapsedUs < 1000000).
@@ -291,23 +318,22 @@ pending_probe_is_bounded_and_recovers_only_after_fresh_reply_test() ->
try
Pid ! queued_work,
check_guild(Pid, 100),
{Pid, 42, false, Targets, {Ref, 100}} = lookup(Pid),
{Pid, 42, false, Targets, {Ref, 100, 0}} = lookup(Pid),
check_guild(Pid, 2201),
?assert(is_degraded(Pid)),
{Pid, 42, true, Targets, {Ref, 100, 0}} = lookup(Pid),
receive
{'$gen_cast', {guild_health, 42, Pid, true}} -> ok
after 100 -> ?assert(false)
end,
check_guild(Pid, 5000),
?assertEqual({message_queue_len, 2}, process_info(Pid, message_queue_len)),
{noreply, #{}} = handle_cast({pong, Pid, Ref, 5000}, #{}),
?assert(is_degraded(Pid)),
check_guild(Pid, 5100),
{Pid, 42, true, Targets, {Ref2, 5100}} = lookup(Pid),
{noreply, #{}} = handle_cast({pong, Pid, Ref, 5110}, #{}),
?assert(is_degraded(Pid)),
{noreply, #{}} = handle_cast({pong, Pid, Ref2, 5110}, #{}),
?assertNot(is_degraded(Pid)),
ok = pong(Pid, Ref, 5000, 5000),
{Pid, 42, true, Targets, {Ref2, 5000, 4900}} = lookup(Pid),
?assertEqual({message_queue_len, 3}, process_info(Pid, message_queue_len)),
ok = pong(Pid, Ref, 5110, 5110),
{Pid, 42, true, Targets, {Ref2, 5000, 4900}} = lookup(Pid),
ok = pong(Pid, Ref2, 5110, 5110),
?assertEqual({Pid, 42, false, Targets, undefined}, lookup(Pid)),
receive
{'$gen_cast', {guild_health, 42, Pid, false}} -> ok
after 100 -> ?assert(false)
@@ -318,6 +344,57 @@ pending_probe_is_bounded_and_recovers_only_after_fresh_reply_test() ->
ets:delete(Targets)
end.
lost_pong_is_reprobed_and_the_guild_recovers_test() ->
ok = guild_ets_owner:ensure_table(?TABLE, [named_table, public, set]),
Targets = ets:new(health_lost_pong_targets, [set]),
ets:insert(Targets, {<<"session">>, self()}),
Pid = spawn(fun() ->
receive
stop -> ok
end
end),
ets:insert(?TABLE, {Pid, 44, false, Targets, undefined}),
try
Pid ! queued_work,
check_guild(Pid, 0),
{Pid, 44, false, Targets, {Lost, 0, 0}} = lookup(Pid),
check_guild(Pid, 2001),
receive
{'$gen_cast', {guild_health, 44, Pid, true}} -> ok
after 100 -> ?assert(false)
end,
check_guild(Pid, 4999),
?assertEqual({message_queue_len, 2}, process_info(Pid, message_queue_len)),
check_guild(Pid, 5000),
{Pid, 44, true, Targets, {Fresh, 5000, 5000} = Pending} = lookup(Pid),
?assertNotEqual(Lost, Fresh),
?assertEqual({message_queue_len, 3}, process_info(Pid, message_queue_len)),
?assert(overloaded(Pending, 5001)),
ok = pong(Pid, Fresh, 5020, 5020),
?assertEqual({Pid, 44, false, Targets, undefined}, lookup(Pid)),
?assertNot(is_overloaded(Pid)),
receive
{'$gen_cast', {guild_health, 44, Pid, false}} -> ok
after 100 -> ?assert(false)
end,
ok = pong(Pid, Lost, 5030, 5030),
?assertEqual({Pid, 44, false, Targets, undefined}, lookup(Pid))
after
Pid ! stop,
ets:delete(?TABLE, Pid),
ets:delete(Targets)
end.
shedding_needs_fresh_evidence_of_lag_test() ->
Ref = make_ref(),
?assertNot(overloaded(undefined, 10000)),
?assertNot(overloaded({Ref, 10000, 0}, 12000)),
?assert(overloaded({Ref, 10000, 0}, 12001)),
?assertNot(overloaded({Ref, 10000, 1500}, 10100)),
?assert(overloaded({Ref, 10000, 2500}, 10100)),
?assert(overloaded({Ref, 10000, 5000}, 10000 + ?STALE_PROBE_MS)),
?assertNot(overloaded({Ref, 10000, 5000}, 10001 + ?STALE_PROBE_MS)).
session_targets_follow_replacement_and_removal_test() ->
Tab = ets:new(health_test_targets, [set]),
State = #{guild_health_sessions => Tab, sessions => #{<<"s">> => #{pid => self()}}},
fluxer_gateway/src/guild/guild_mailbox_age.erl
+168
View File
@@ -0,0 +1,168 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(guild_mailbox_age).
-typing([eqwalizer]).
-export([note/0, handle_mark/1, min_age_ms/0]).
-define(KEY, {?MODULE, marks}).
-define(MARK_INTERVAL_MS, 50).
-type seq() :: non_neg_integer().
-type marks() :: {seq(), integer() | undefined, queue:queue({seq(), integer()})}.
-spec note() -> ok.
note() ->
Now = erlang:monotonic_time(millisecond),
{Seq, Checked, Pending} = marks(),
case is_integer(Checked) andalso Now - Checked < ?MARK_INTERVAL_MS of
true -> ok;
false -> maybe_mark(Now, Seq, Pending)
end.
-spec maybe_mark(integer(), seq(), queue:queue({seq(), integer()})) -> ok.
maybe_mark(Now, Seq, Pending) ->
case erlang:process_info(self(), message_queue_len) of
{message_queue_len, Len} when Len > 0 ->
self() ! {?MODULE, Seq},
SentBy = erlang:monotonic_time(millisecond) + 1,
put_marks({Seq + 1, Now, queue:in({Seq, SentBy}, Pending)});
_ ->
put_marks({Seq, Now, Pending})
end.
-spec handle_mark(seq()) -> ok.
handle_mark(Seq) ->
{Next, Checked, Pending} = marks(),
put_marks({Next, Checked, drop_through(Seq, Pending)}).
-spec min_age_ms() -> non_neg_integer() | undefined.
min_age_ms() ->
{_, _, Pending} = marks(),
case queue:peek(Pending) of
{value, {_, SentBy}} when is_integer(SentBy) ->
max(0, erlang:monotonic_time(millisecond) - SentBy);
_ ->
undefined
end.
-spec drop_through(seq(), queue:queue({seq(), integer()})) -> queue:queue({seq(), integer()}).
drop_through(Seq, Pending) ->
case queue:peek(Pending) of
{value, {S, _}} when S =< Seq -> drop_through(Seq, queue:drop(Pending));
_ -> Pending
end.
-spec marks() -> marks().
marks() ->
case erlang:get(?KEY) of
{Seq, _, _} = Marks when is_integer(Seq) ->
Marks;
_ ->
{0, undefined, queue:new()}
end.
-spec put_marks(marks()) -> ok.
put_marks(Marks) ->
_ = erlang:put(?KEY, Marks),
ok.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
in_fresh_process(Fun) ->
Self = self(),
Ref = make_ref(),
{Pid, MRef} = spawn_monitor(fun() -> Self ! {Ref, Fun()} end),
receive
{Ref, Result} ->
erlang:demonitor(MRef, [flush]),
Result;
{'DOWN', MRef, process, Pid, Reason} ->
erlang:error(Reason)
after 5000 ->
erlang:error(timeout)
end.
next_mark() ->
receive
{?MODULE, Seq} -> Seq
after 0 -> none
end.
no_age_without_a_backlog_test() ->
?assertEqual(
{undefined, none},
in_fresh_process(fun() ->
ok = note(),
{min_age_ms(), next_mark()}
end)
).
a_message_queued_before_a_mark_is_at_least_as_old_as_the_mark_test() ->
{Age, Seq} = in_fresh_process(fun() ->
self() ! queued_request,
ok = note(),
timer:sleep(120),
receive
queued_request -> ok
end,
{min_age_ms(), next_mark()}
end),
?assertEqual(0, Seq),
?assert(Age >= 119),
?assert(Age =< 1000).
handled_marks_stop_bounding_later_messages_test() ->
?assertEqual(
undefined,
in_fresh_process(fun() ->
self() ! queued_request,
ok = note(),
receive
queued_request -> ok
end,
ok = handle_mark(next_mark()),
min_age_ms()
end)
).
marks_are_rate_limited_while_busy_test() ->
Seqs = in_fresh_process(fun() ->
self() ! queued_request,
ok = note(),
ok = note(),
timer:sleep(60),
ok = note(),
[next_mark(), next_mark(), next_mark()]
end),
?assertEqual([0, 1, none], Seqs).
the_oldest_unhandled_mark_bounds_the_age_test() ->
{Age, Seqs} = in_fresh_process(fun() ->
self() ! queued_request,
ok = note(),
timer:sleep(200),
ok = note(),
receive
queued_request -> ok
end,
First = next_mark(),
Second = next_mark(),
{min_age_ms(), [First, Second]}
end),
?assertEqual([0, 1], Seqs),
?assert(Age >= 199).
a_forged_mark_only_loosens_the_bound_test() ->
?assertEqual(
undefined,
in_fresh_process(fun() ->
self() ! queued_request,
ok = note(),
ok = handle_mark(1000),
min_age_ms()
end)
).
-endif.
fluxer_gateway/src/guild/guild_manager_handoff.erl
+29 -4
View File
@@ -151,12 +151,9 @@ continue_handoff_to_target(TargetNode, GuildIds, State, Iteration, AccResult) ->
continue_handoff_to_topology(_TargetNodes, [], State, _Iteration, AccResult) ->
{AccResult, State};
continue_handoff_to_topology(TargetNodes, GuildIds, State, Iteration, AccResult) ->
Resolver = fun(GId) ->
gateway_node_router:select_owner_node(GId, TargetNodes)
end,
run_and_continue(
GuildIds,
Resolver,
topology_owner_resolver(),
AccResult,
State,
fun(S1, I, M, B) ->
@@ -165,6 +162,12 @@ continue_handoff_to_topology(TargetNodes, GuildIds, State, Iteration, AccResult)
Iteration
).
-spec topology_owner_resolver() -> fun((guild_id()) -> term()).
topology_owner_resolver() ->
fun(GId) ->
gateway_node_router:owner_node_result(GId, guilds)
end.
-spec run_and_continue(
[guild_id()],
fun((guild_id()) -> term()),
@@ -291,3 +294,25 @@ mark_handoff_attempt(#{attempted := Attempted, handed_off := HandedOff}) ->
-spec mark_handoff_success(handoff_result()) -> handoff_result().
mark_handoff_success(#{attempted := Attempted, handed_off := HandedOff}) ->
#{attempted => Attempted + 1, handed_off => HandedOff + 1}.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
topology_handoff_keeps_a_guild_pinned_to_this_node_test() ->
GuildId = 1100000000000000001,
Peers = ['guilds_a@h', 'guilds_b@h'],
Resolver = topology_owner_resolver(),
persistent_term:put({gateway_cluster_membership, members}, [node() | Peers]),
persistent_term:put({gateway_cluster_membership, members_by_role}, #{guilds => Peers}),
try
?assertMatch({handoff, _}, resolve_handoff_target(GuildId, node(), Resolver)),
application:set_env(fluxer_gateway, guild_owner_pins, #{GuildId => node()}),
?assertEqual(skip, resolve_handoff_target(GuildId, node(), Resolver)),
?assertMatch({handoff, _}, resolve_handoff_target(GuildId + 1, node(), Resolver))
after
application:unset_env(fluxer_gateway, guild_owner_pins),
persistent_term:erase({gateway_cluster_membership, members}),
persistent_term:erase({gateway_cluster_membership, members_by_role})
end.
-endif.
fluxer_gateway/src/guild/guild_manager_handoff_transfer.erl
+20 -89
View File
@@ -13,103 +13,34 @@
-export_type([guild_id/0, state/0]).
-define(SHARD_TABLE, guild_manager_shard_table).
-spec handoff_guild_to_owner(guild_id(), node(), state()) -> {boolean(), state()}.
handoff_guild_to_owner(GuildId, TargetNode, State) ->
case export_local_guild_state(GuildId, State) of
{ok, TransferState} ->
start_transferred_guild_on_owner(
GuildId, TargetNode, TransferState, State
);
{error, _Reason} ->
{false, State}
end.
-spec start_transferred_guild_on_owner(guild_id(), node(), map(), state()) ->
{boolean(), state()}.
start_transferred_guild_on_owner(GuildId, TargetNode, TransferState, State) ->
Request = {start_transferred, GuildId, TransferState},
case call_target_shard(GuildId, TargetNode, Request) of
{ok, Pid} when is_pid(Pid) -> stop_local_guild_after_handoff(GuildId, State);
_Other -> {false, State}
end.
-spec call_target_shard(guild_id(), node(), term()) -> term().
call_target_shard(GuildId, TargetNode, Request) ->
case target_shard_pid(GuildId, TargetNode) of
{ok, Pid} ->
shard_utils:safe_gen_call_remote(Pid, Request, ?DEFAULT_GEN_SERVER_TIMEOUT);
{error, _Reason} = Error ->
Error
end.
-spec target_shard_pid(guild_id(), node()) -> {ok, pid()} | {error, term()}.
target_shard_pid(GuildId, TargetNode) ->
case target_shard_count(TargetNode) of
{ok, Count} -> lookup_target_shard_pid(GuildId, TargetNode, Count);
{error, _Reason} = Error -> Error
end.
-spec target_shard_count(node()) -> {ok, pos_integer()} | {error, term()}.
target_shard_count(TargetNode) ->
Request = [?SHARD_TABLE, shard_count],
case rpc:call(TargetNode, ets, lookup, Request, ?DEFAULT_GEN_SERVER_TIMEOUT) of
[{shard_count, Count}] when is_integer(Count), Count > 0 -> {ok, Count};
{badrpc, Reason} -> {error, {badrpc, Reason}};
_Other -> {error, shard_table_unavailable}
end.
-spec lookup_target_shard_pid(guild_id(), node(), pos_integer()) ->
{ok, pid()} | {error, term()}.
lookup_target_shard_pid(GuildId, TargetNode, Count) ->
Index = guild_manager_shards:select_shard(GuildId, Count),
Request = [?SHARD_TABLE, {shard_pid, Index}],
case rpc:call(TargetNode, ets, lookup, Request, ?DEFAULT_GEN_SERVER_TIMEOUT) of
[{{shard_pid, Index}, Pid}] when is_pid(Pid) -> {ok, Pid};
{badrpc, Reason} -> {error, {badrpc, Reason}};
_Other -> {error, shard_unavailable}
end.
-spec export_local_guild_state(guild_id(), state()) -> {ok, map()} | {error, term()}.
export_local_guild_state(GuildId, State) ->
case local_guild_pid(GuildId, State) of
{ok, Pid} -> export_guild_state(Pid);
{error, _Reason} = Error -> Error
end.
-spec local_guild_pid(guild_id(), state()) -> {ok, pid()} | {error, term()}.
local_guild_pid(GuildId, State) ->
{Index, State1} = guild_manager_shards:ensure_shard(GuildId, State),
Shards = maps:get(shards, State1),
#{pid := Pid} = maps:get(Index, Shards),
#{pid := Shard} = maps:get(Index, maps:get(shards, State1)),
case local_guild_pid(Shard, GuildId) of
{ok, GuildPid} ->
Result = guild_handoff_freeze:transfer(GuildId, GuildPid, Shard, TargetNode, #{}),
{handed_off(GuildId, TargetNode, Result), State1};
{error, _Reason} ->
{false, State1}
end.
-spec local_guild_pid(pid(), guild_id()) -> {ok, pid()} | {error, term()}.
local_guild_pid(Shard, GuildId) ->
case
shard_utils:safe_gen_call_remote(Pid, {lookup, GuildId}, ?DEFAULT_GEN_SERVER_TIMEOUT)
shard_utils:safe_gen_call_remote(Shard, {lookup, GuildId}, ?DEFAULT_GEN_SERVER_TIMEOUT)
of
{ok, GuildPid} when is_pid(GuildPid) -> {ok, GuildPid};
{error, Reason} -> {error, Reason};
Other -> {error, {unexpected_lookup_reply, Other}}
end.
-spec stop_local_guild_after_handoff(guild_id(), state()) -> {boolean(), state()}.
stop_local_guild_after_handoff(GuildId, State) ->
{Index, State1} = guild_manager_shards:ensure_shard(GuildId, State),
Shards = maps:get(shards, State1),
#{pid := Pid} = maps:get(Index, Shards),
Reply = shard_utils:safe_gen_call_remote(
Pid, {stop_guild, GuildId, {shutdown, handoff}}, ?DEFAULT_GEN_SERVER_TIMEOUT
-spec handed_off(guild_id(), node(), guild_handoff_freeze:result()) -> boolean().
handed_off(_GuildId, _TargetNode, {ok, _Report}) ->
true;
handed_off(GuildId, TargetNode, {error, Report}) ->
logger:warning(
"guild_handoff_failed: guild_id=~p target=~p report=~p",
[GuildId, TargetNode, Report]
),
case Reply of
ok -> {true, State1};
_Other -> {false, State1}
end.
-spec export_guild_state(pid()) -> {ok, map()} | {error, term()}.
export_guild_state(Pid) ->
case
shard_utils:safe_gen_call_remote(Pid, export_handoff_state, ?DEFAULT_GEN_SERVER_TIMEOUT)
of
{ok, TransferState} when is_map(TransferState) -> {ok, TransferState};
{error, Reason} -> {error, Reason};
Other -> {error, {unexpected_export_reply, Other}}
end.
false.
fluxer_gateway/src/guild/guild_member_list_channel_engine.erl
+9 -1
View File
@@ -230,7 +230,7 @@ build(ListId, State) ->
undefined ->
State;
ChannelId ->
Ref = load_engine(ChannelId, State),
Ref = load_or_clone_engine(ListId, ChannelId, State),
guild_member_list_engine_inputs:record(
ListId,
ChannelId,
@@ -239,6 +239,14 @@ build(ListId, State) ->
)
end.
-spec load_or_clone_engine(list_id(), pos_integer(), guild_state()) -> engine_ref().
load_or_clone_engine(ListId, ChannelId, State) ->
Engines = maps:remove(ListId, engines(State)),
case guild_member_list_engine_inputs:current_twin(ChannelId, maps:keys(Engines), State) of
{ok, TwinListId} -> guild_member_list_engine:clone(maps:get(TwinListId, Engines));
none -> load_engine(ChannelId, State)
end.
-spec load_engine(pos_integer(), guild_state()) -> engine_ref().
load_engine(ChannelId, State) ->
Ref = guild_member_list_engine:new(),
fluxer_gateway/src/guild/guild_member_list_engine.erl
+12
View File
@@ -5,6 +5,7 @@
-export([
new/0,
clone/1,
destroy/1,
bulk_load/3,
add_member/5,
@@ -49,6 +50,17 @@ new() ->
]),
Ref.
-spec clone(ets:table()) -> ets:table().
clone(Source) ->
{SourceOSet, SourceITab} = lookup_tabs(Source),
Ref = new(),
{OSet, ITab} = lookup_tabs(Ref),
true = ets:insert(ITab, ets:tab2list(SourceITab)),
ok = guild_member_list_oset:from_sorted(OSet, guild_member_list_oset:to_list(SourceOSet)),
true = ets:insert(Ref, [Row || Row <- ets:tab2list(Source), element(1, Row) =/= tabs]),
ok = bump_version(Ref),
Ref.
-spec destroy(term()) -> ok.
destroy(Ref) ->
try ets:lookup(eqwalizer:dynamic_cast(Ref), tabs) of
fluxer_gateway/src/guild/guild_member_list_engine_inputs.erl
+27
View File
@@ -5,6 +5,7 @@
-export([
is_stale/2,
current_twin/3,
record/4,
forget/2,
forget_all/1,
@@ -30,6 +31,32 @@ is_stale(ListId, State) ->
true
end.
-spec current_twin(channel_id(), [list_id()], guild_state()) -> {ok, list_id()} | none.
current_twin(ChannelId, ListIds, State) ->
AccessByChannel = virtual_access_by_channel(State),
find_current_twin(
inputs(ChannelId, State, AccessByChannel),
ListIds,
recorded(State),
AccessByChannel,
State
).
-spec find_current_twin(inputs(), [list_id()], map(), #{term() => [term()]}, guild_state()) ->
{ok, list_id()} | none.
find_current_twin(_Inputs, [], _Recorded, _AccessByChannel, _State) ->
none;
find_current_twin(Inputs, [ListId | Rest], Recorded, AccessByChannel, State) ->
case {maps:find(ListId, Recorded), channel_id(ListId)} of
{{ok, Inputs}, TwinChannelId} when is_integer(TwinChannelId) ->
case inputs(TwinChannelId, State, AccessByChannel) of
Inputs -> {ok, ListId};
_ -> find_current_twin(Inputs, Rest, Recorded, AccessByChannel, State)
end;
_ ->
find_current_twin(Inputs, Rest, Recorded, AccessByChannel, State)
end.
-spec record(list_id(), channel_id(), guild_state(), guild_state()) -> guild_state().
record(ListId, ChannelId, BuiltFrom, State) ->
put_recorded(maps:put(ListId, inputs(ChannelId, BuiltFrom), recorded(State)), State).
fluxer_gateway/src/guild/guild_member_list_write.erl
+69 -11
View File
@@ -6,6 +6,7 @@
-export([
broadcast_member_list_updates/3,
broadcast_member_list_updates/5,
queue_member_list_updates/5,
broadcast_all_member_list_updates/1,
broadcast_member_list_updates_for_channel/2,
broadcast_channel_engine_connection_change/2,
@@ -25,6 +26,7 @@
-type engine_ref() :: ets:table().
-type absence() :: {absent, engine_ref()} | present.
-type pending_mark() :: true | synced.
-type list_sync() :: immediate | deferred.
-define(MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT, 1000000000).
@@ -53,11 +55,34 @@ broadcast_member_list_updates(
) ->
{ok, UpdatedState};
broadcast_member_list_updates(UserId, OldState, UpdatedState, OldPresence, NewPresence) ->
member_list_updates(immediate, UserId, OldState, UpdatedState, OldPresence, NewPresence).
-spec queue_member_list_updates(
user_id() | undefined,
guild_state(),
guild_state(),
map() | undefined,
map() | undefined
) -> {ok, guild_state()}.
queue_member_list_updates(undefined, _OldState, UpdatedState, _OldPresence, _NewPresence) ->
{ok, UpdatedState};
queue_member_list_updates(UserId, OldState, UpdatedState, OldPresence, NewPresence) ->
member_list_updates(deferred, UserId, OldState, UpdatedState, OldPresence, NewPresence).
-spec member_list_updates(
list_sync(),
user_id(),
guild_state(),
guild_state(),
map() | undefined,
map() | undefined
) -> {ok, guild_state()}.
member_list_updates(ListSync, UserId, OldState, UpdatedState, OldPresence, NewPresence) ->
guild_member_list_write_context:with_guild_id(UpdatedState, fun(_GuildId) ->
OldMember = find_member_in_state_data(UserId, OldState),
NewMember = find_member_in_state_data(UserId, UpdatedState),
State1 = dispatch_presence_delta(
UserId, OldMember, NewMember, OldPresence, NewPresence, UpdatedState
UserId, OldMember, NewMember, OldPresence, NewPresence, ListSync, UpdatedState
),
{ok, State1}
end).
@@ -68,16 +93,19 @@ broadcast_member_list_updates(UserId, OldState, UpdatedState, OldPresence, NewPr
map() | undefined,
map() | undefined,
map() | undefined,
list_sync(),
guild_state()
) -> guild_state().
dispatch_presence_delta(UserId, OldMember, NewMember, OldPresence, NewPresence, State) ->
dispatch_presence_delta(
UserId, OldMember, NewMember, OldPresence, NewPresence, ListSync, State
) ->
case presence_delta_is_inert(OldPresence, NewPresence, OldMember, NewMember) of
true ->
invalidate_synced_lists(UserId, State);
false ->
SubsTab = maps:get(member_list_subscriptions, State),
dispatch_user_change_to_subscribed_lists(
UserId, OldMember, NewMember, SubsTab, State
UserId, OldMember, NewMember, SubsTab, ListSync, State
)
end.
@@ -216,16 +244,17 @@ fold_connection_change_lists(GuildId, UserId, Mark, State, SubsTab) ->
map() | undefined,
map() | undefined,
ets:table(),
list_sync(),
guild_state()
) -> guild_state().
dispatch_user_change_to_subscribed_lists(
UserId, OldMember, NewMember, SubsTab, State
UserId, OldMember, NewMember, SubsTab, ListSync, State
) ->
guild_member_list_read:with_member_item_memo(fun() ->
lists:foldl(
fun(ListId, AccState) ->
dispatch_user_change_to_subscribed_list(
UserId, OldMember, NewMember, ListId, AccState
UserId, OldMember, NewMember, ListId, ListSync, AccState
)
end,
State,
@@ -238,16 +267,23 @@ dispatch_user_change_to_subscribed_lists(
map() | undefined,
map() | undefined,
list_id(),
list_sync(),
guild_state()
) -> guild_state().
dispatch_user_change_to_subscribed_list(UserId, OldMember, NewMember, ListId, State) ->
dispatch_user_change_to_subscribed_list(UserId, OldMember, NewMember, ListId, ListSync, State) ->
Absence = user_change_absence(UserId, ListId, OldMember, NewMember, State),
State1 = apply_user_change_to_channel_store(UserId, ListId, OldMember, NewMember, State),
case sync_body_unchanged(UserId, ListId, Absence, State1) of
true -> record_member_list_sync_skipped_absent(State1);
false -> guild_member_list_sync_batch:queue_list_sync(ListId, State1)
false -> queue_user_change_sync(ListSync, ListId, State1)
end.
-spec queue_user_change_sync(list_sync(), list_id(), guild_state()) -> guild_state().
queue_user_change_sync(immediate, ListId, State) ->
guild_member_list_sync_batch:queue_list_sync(ListId, State);
queue_user_change_sync(deferred, ListId, State) ->
queue_connection_list_sync(ListId, true, State).
-spec user_change_absence(
user_id(), list_id(), map() | undefined, map() | undefined, guild_state()
) -> absence().
@@ -551,14 +587,18 @@ subscribed_list_sync_is_skipped_only_for_absent_member_test() ->
?assert(
maps:is_key(
pending_member_list_sync_batch,
dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
dispatch_user_change_to_subscribed_list(
7, Member, Member, <<"500">>, immediate, State
)
)
),
ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
?assertNot(
maps:is_key(
pending_member_list_sync_batch,
dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
dispatch_user_change_to_subscribed_list(
7, Member, Member, <<"500">>, immediate, State
)
)
)
end)
@@ -695,7 +735,9 @@ member_update_sync_stays_immediate_test() ->
?assertNot(
maps:is_key(
pending_member_list_sync_batch,
dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
dispatch_user_change_to_subscribed_list(
7, Member, Member, <<"500">>, immediate, State
)
)
)
after
@@ -703,6 +745,22 @@ member_update_sync_stays_immediate_test() ->
guild_member_list_engine:destroy(Ref)
end.
deferred_user_change_sync_is_debounced_test() ->
Ref = guild_member_list_engine:new(),
Old = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},
New = Old#{<<"nick">> => <<"seven">>},
try
ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
Next = dispatch_user_change_to_subscribed_list(
7, Old, New, <<"500">>, deferred, engine_state(Ref)
),
Batch = maps:get(pending_member_list_sync_batch, Next),
?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch)),
_ = erlang:cancel_timer(maps:get(timer_ref, Batch))
after
guild_member_list_engine:destroy(Ref)
end.
synced_connection_mark_never_downgrades_pending_sync_test() ->
State1 = queue_connection_list_sync(<<"500">>, synced, #{}),
TimerRef = maps:get(timer_ref, maps:get(pending_member_list_sync_batch, State1)),
@@ -756,7 +814,7 @@ inert_presence_delta_invalidates_synced_lists_test() ->
try
ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
Next = dispatch_presence_delta(
7, Member, Member, Old, Old#{<<"afk">> => true}, State
7, Member, Member, Old, Old#{<<"afk">> => true}, immediate, State
),
?assertEqual(
#{<<"500">> => true},
fluxer_gateway/src/guild/guild_presence.erl
+58 -16
View File
@@ -6,12 +6,14 @@
-export([handle_bus_presence/3, send_cached_presence_to_session/3]).
-export([cached_presences/1, send_presence_lookup_to_session/4]).
-export([sync_online_status/2]).
-export([apply_connect_presences/2]).
-export_type([guild_state/0, user_id/0]).
-type guild_state() :: map().
-type member() :: map().
-type user_id() :: integer().
-type list_sync() :: immediate | deferred.
%% members_sorted_ids trims with the member map it indexes: a snapshot that kept it would
%% answer sorted_member_ids/2 with ids for members the snapshot no longer carries.
@@ -40,15 +42,46 @@ handle_user_update(UserId, Payload, State) ->
),
{noreply, NewState}.
-spec apply_connect_presences([user_id()], guild_state()) -> guild_state().
apply_connect_presences([], State) ->
State;
apply_connect_presences(UserIds, State) ->
Found = cached_presences(UserIds),
lists:foldl(
fun(UserId, Acc) ->
apply_connect_presence(UserId, maps:get(UserId, Found, not_found), Acc)
end,
State,
UserIds
).
-spec apply_connect_presence(user_id(), {ok, map()} | not_found, guild_state()) ->
guild_state().
apply_connect_presence(UserId, {ok, Payload}, State) ->
{noreply, NewState} =
case maps:get(<<"user_update">>, Payload, false) of
true -> handle_user_update(UserId, Payload, State);
false -> handle_presence_update(UserId, Payload, deferred, State)
end,
NewState;
apply_connect_presence(_UserId, not_found, State) ->
State.
-spec handle_presence_update(user_id(), map(), guild_state()) -> {noreply, guild_state()}.
handle_presence_update(UserId, Payload, State) ->
handle_presence_update(UserId, Payload, immediate, State).
-spec handle_presence_update(user_id(), map(), list_sync(), guild_state()) ->
{noreply, guild_state()}.
handle_presence_update(UserId, Payload, ListSync, State) ->
case find_member_by_user_id(UserId, State) of
undefined -> {noreply, State};
Member -> process_presence(UserId, Payload, Member, State)
Member -> process_presence(UserId, Payload, Member, ListSync, State)
end.
-spec process_presence(user_id(), map(), member(), guild_state()) -> {noreply, guild_state()}.
process_presence(UserId, Payload, Member, State) ->
-spec process_presence(user_id(), map(), member(), list_sync(), guild_state()) ->
{noreply, guild_state()}.
process_presence(UserId, Payload, Member, ListSync, State) ->
PresenceMap = build_presence_map(Payload, Member),
NormalizedStatus = normalize_presence_status(
maps:get(<<"status">>, Payload, <<"offline">>)
@@ -58,17 +91,17 @@ process_presence(UserId, Payload, Member, State) ->
maps:get(member_presence, State),
UserId
),
process_presence_change(UserId, OldPresence, PresenceMap, Status, State).
process_presence_change(UserId, OldPresence, PresenceMap, Status, ListSync, State).
-spec process_presence_change(user_id(), map(), map(), atom(), guild_state()) ->
-spec process_presence_change(user_id(), map(), map(), atom(), list_sync(), guild_state()) ->
{noreply, guild_state()}.
process_presence_change(UserId, PresenceMap, PresenceMap, Status, State) ->
process_presence_change(UserId, PresenceMap, PresenceMap, Status, _ListSync, State) ->
{noreply, maybe_handle_unchanged_presence(Status, UserId, State)};
process_presence_change(UserId, OldPresence, PresenceMap, Status, State) ->
process_presence_change(UserId, OldPresence, PresenceMap, Status, ListSync, State) ->
StateWithPresence = store_member_presence(UserId, PresenceMap, State),
ok = guild_presence_sync:sync_online_status(UserId, StateWithPresence),
StateAfterBroadcast = spawn_presence_broadcast(
UserId, OldPresence, PresenceMap, State, StateWithPresence
UserId, OldPresence, PresenceMap, State, StateWithPresence, ListSync
),
StateAfterOffline = maybe_handle_offline(Status, UserId, StateAfterBroadcast),
{noreply, StateAfterOffline}.
@@ -104,20 +137,29 @@ sync_online_status(UserId, State) ->
map(),
map(),
guild_state(),
guild_state()
guild_state(),
list_sync()
) -> guild_state().
spawn_presence_broadcast(UserId, OldPresence, PresenceMap, OldState, NewState) ->
{ok, NewState1} = guild_member_list:broadcast_member_list_updates(
UserId,
OldState,
NewState,
OldPresence,
PresenceMap
spawn_presence_broadcast(UserId, OldPresence, PresenceMap, OldState, NewState, ListSync) ->
{ok, NewState1} = member_list_presence_update(
ListSync, UserId, OldState, NewState, OldPresence, PresenceMap
),
{Pid, NewState2} = guild_broadcaster:ensure(NewState1),
ok = cast_presence_update(Pid, UserId, PresenceMap, NewState2),
NewState2.
-spec member_list_presence_update(
list_sync(), user_id(), guild_state(), guild_state(), map(), map()
) -> {ok, guild_state()}.
member_list_presence_update(immediate, UserId, OldState, NewState, OldPresence, PresenceMap) ->
guild_member_list:broadcast_member_list_updates(
UserId, OldState, NewState, OldPresence, PresenceMap
);
member_list_presence_update(deferred, UserId, OldState, NewState, OldPresence, PresenceMap) ->
guild_member_list_write:queue_member_list_updates(
UserId, OldState, NewState, OldPresence, PresenceMap
).
-spec cast_presence_update(pid() | undefined, user_id(), map(), guild_state()) -> ok.
cast_presence_update(BroadcasterPid, UserId, PresenceMap, State) when is_pid(BroadcasterPid) ->
case safe_presence_update_recipients(UserId, presence_view(State)) of
fluxer_gateway/src/guild/guild_query_handler.erl
+110 -3
View File
@@ -11,6 +11,7 @@
-endif.
-define(INLINE_MEMBER_QUERY_MAX_IDS, 100).
-define(INFERRED_DEADLINE_MARGIN_MS, 1000).
-type guild_state() :: map().
-type user_id() :: integer().
@@ -36,16 +37,82 @@ handle_call(Msg, From, State) ->
end.
-spec is_expired(term(), gen_server:from()) -> boolean().
is_expired({_Tag, #{deadline_monotonic := Deadline}}, {Caller, _ReplyTag}) when
is_expired(Msg, From) ->
case deadline_expired(Msg, From) of
unknown -> inferred_deadline_expired(Msg);
Expired -> Expired
end.
-spec deadline_expired(term(), gen_server:from()) -> boolean() | unknown.
deadline_expired({_Tag, #{deadline_monotonic := Deadline}}, {Caller, _ReplyTag}) when
is_integer(Deadline)
->
case gateway_clock_offset:offset(node(Caller)) of
undefined -> false;
undefined -> unknown;
Offset -> erlang:monotonic_time(millisecond) > Deadline - Offset
end;
is_expired(_Msg, _From) ->
deadline_expired(_Msg, _From) ->
unknown.
-spec inferred_deadline_expired(term()) -> boolean().
inferred_deadline_expired(Msg) when is_tuple(Msg), tuple_size(Msg) > 0 ->
case caller_timeout_ms(element(1, Msg)) of
undefined -> false;
TimeoutMs -> waited_longer_than(TimeoutMs + ?INFERRED_DEADLINE_MARGIN_MS)
end;
inferred_deadline_expired(_Msg) ->
false.
-spec waited_longer_than(pos_integer()) -> boolean().
waited_longer_than(Ms) ->
case guild_mailbox_age:min_age_ms() of
undefined -> false;
Age when Age > Ms -> inferred_deadlines_enabled();
_ -> false
end.
-spec inferred_deadlines_enabled() -> boolean().
inferred_deadlines_enabled() ->
application:get_env(fluxer_gateway, guild_query_inferred_deadlines, true) =/= false.
-spec caller_timeout_ms(term()) -> pos_integer() | undefined.
caller_timeout_ms(get_large_guild_metadata) -> 200;
caller_timeout_ms(get_user_counts) -> 2000;
caller_timeout_ms(get_viewer_counts) -> 2000;
caller_timeout_ms(get_channel_member_counts) -> 2000;
caller_timeout_ms(check_permission) -> 5000;
caller_timeout_ms(get_guild_members_batch) -> 5000;
caller_timeout_ms(list_guild_members) -> 10000;
caller_timeout_ms(search_guild_members) -> 10000;
caller_timeout_ms(Tag) -> rpc_caller_timeout_ms(Tag).
-spec rpc_caller_timeout_ms(term()) -> pos_integer() | undefined.
rpc_caller_timeout_ms(get_user_permissions) -> 4000;
rpc_caller_timeout_ms(can_manage_roles) -> 4000;
rpc_caller_timeout_ms(can_manage_role) -> 4000;
rpc_caller_timeout_ms(get_assignable_roles) -> 4000;
rpc_caller_timeout_ms(get_user_max_role_position) -> 4000;
rpc_caller_timeout_ms(get_guild_data) -> 4000;
rpc_caller_timeout_ms(get_guild_auth_context) -> 4000;
rpc_caller_timeout_ms(get_guild_member) -> 4000;
rpc_caller_timeout_ms(has_member) -> 4000;
rpc_caller_timeout_ms(get_members_with_role) -> 4000;
rpc_caller_timeout_ms(check_target_member) -> 4000;
rpc_caller_timeout_ms(list_guild_members_cursor) -> 4000;
rpc_caller_timeout_ms(get_viewable_channels) -> 4000;
rpc_caller_timeout_ms(resolve_channel_mentions) -> 4000;
rpc_caller_timeout_ms(get_vanity_url_channel) -> 4000;
rpc_caller_timeout_ms(get_first_viewable_text_channel) -> 4000;
rpc_caller_timeout_ms(get_category_channel_count) -> 4000;
rpc_caller_timeout_ms(get_channel_count) -> 4000;
rpc_caller_timeout_ms(get_users_to_mention_by_roles) -> 4000;
rpc_caller_timeout_ms(get_users_to_mention_by_user_ids) -> 4000;
rpc_caller_timeout_ms(get_all_users_to_mention) -> 4000;
rpc_caller_timeout_ms(resolve_all_mentions) -> 4000;
rpc_caller_timeout_ms(resolve_mention_sources) -> 4000;
rpc_caller_timeout_ms(resolve_mention_sources_page) -> 4000;
rpc_caller_timeout_ms(_Tag) -> undefined.
-spec handle_query(term(), gen_server:from(), guild_state()) ->
{reply, term(), guild_state()}
| {noreply, guild_state()}.
@@ -482,6 +549,46 @@ monotonic_deadline_ignores_the_legacy_wall_clock_test() ->
),
?assertNot(is_expired({get_data, #{deadline => 0}}, From)).
deadline_less_queries_expire_once_every_known_caller_has_given_up_test() ->
Self = self(),
Ref = make_ref(),
spawn(fun() ->
From = {Self, make_ref()},
self() ! queued_request,
ok = guild_mailbox_age:note(),
Fresh = is_expired({get_large_guild_metadata}, From),
timer:sleep(1300),
Future = erlang:monotonic_time(millisecond) + 5000,
Results = #{
fresh => Fresh,
metadata => is_expired({get_large_guild_metadata}, From),
counts => is_expired({get_user_counts, 1}, From),
unknown_tag => is_expired({get_sessions}, From),
explicit_deadline =>
is_expired({get_large_guild_metadata, #{deadline_monotonic => Future}}, From)
},
ok = application:set_env(fluxer_gateway, guild_query_inferred_deadlines, false),
Disabled = is_expired({get_large_guild_metadata}, From),
ok = application:unset_env(fluxer_gateway, guild_query_inferred_deadlines),
Self ! {Ref, Results#{disabled => Disabled}}
end),
Results =
receive
{Ref, R} -> R
after 5000 -> error(timeout)
end,
?assertEqual(
#{
fresh => false,
metadata => true,
counts => false,
unknown_tag => false,
explicit_deadline => false,
disabled => false
},
Results
).
call_keeps_the_legacy_deadline_and_adds_a_monotonic_deadline_test() ->
Guild = spawn(fun() ->
receive
fluxer_gateway/src/guild/guild_read_model.erl
+1
View File
@@ -12,6 +12,7 @@
<<"roles">>,
<<"channels">>,
<<"channel_index">>,
channels_stale,
<<"emojis">>,
<<"stickers">>,
role_perms_cache,
fluxer_gateway/src/guild/guild_request_counts.erl
+65 -6
View File
@@ -15,6 +15,7 @@
-define(GUILD_CALL_TIMEOUT_MS, 2000).
-define(BATCH_OVERALL_TIMEOUT_MS, 3000).
-define(MAX_NONCE_BYTES, 64).
-define(LEGACY_NODE_KEY(Node), {?MODULE, legacy_node, Node}).
-type session_state() :: map().
@@ -105,14 +106,33 @@ worker(Parent, Tag, GuildId, GuildPid, UserId) ->
-spec fetch_counts(pid(), integer()) -> {ok, non_neg_integer(), non_neg_integer()} | error.
fetch_counts(GuildPid, UserId) ->
case is_legacy_node(node(GuildPid)) of
true -> fetch_legacy_counts(GuildPid, UserId);
false -> fetch_viewer_counts(GuildPid, UserId)
end.
-spec fetch_viewer_counts(pid(), integer()) ->
{ok, non_neg_integer(), non_neg_integer()} | error.
fetch_viewer_counts(GuildPid, UserId) ->
Request = {get_viewer_counts, #{user_id => UserId}},
try guild_query_handler:call(GuildPid, Request, ?GUILD_CALL_TIMEOUT_MS) of
ok -> fetch_legacy_counts(GuildPid, UserId);
Reply -> counts_result(Reply)
ok ->
ok = remember_legacy_node(node(GuildPid)),
fetch_legacy_counts(GuildPid, UserId);
Reply ->
counts_result(Reply)
catch
_:_ -> error
end.
-spec is_legacy_node(node()) -> boolean().
is_legacy_node(Node) ->
persistent_term:get(?LEGACY_NODE_KEY(Node), false) =:= true.
-spec remember_legacy_node(node()) -> ok.
remember_legacy_node(Node) ->
persistent_term:put(?LEGACY_NODE_KEY(Node), true).
-spec fetch_legacy_counts(pid(), integer()) ->
{ok, non_neg_integer(), non_neg_integer()} | error.
fetch_legacy_counts(GuildPid, UserId) ->
@@ -248,7 +268,12 @@ handle_request_echoes_nonce_test() ->
?assert(false)
end.
forget_legacy_nodes() ->
_ = persistent_term:erase(?LEGACY_NODE_KEY(node())),
ok.
handle_request_fetches_viewer_counts_with_deadline_test() ->
ok = forget_legacy_nodes(),
Self = self(),
Guild = spawn(fun() ->
receive
@@ -299,25 +324,26 @@ request_counts_payload(Guilds) ->
end.
handle_request_falls_back_to_user_counts_on_legacy_guild_test() ->
ok = forget_legacy_nodes(),
Guild = legacy_guild([#{member_count => 50, online_count => 10}]),
Payload = request_counts_payload(#{7 => {Guild, make_ref()}}),
?assertEqual([build_entry(7, 50, 10)], maps:get(<<"counts">>, Payload)).
handle_request_omits_guild_when_legacy_fallback_fails_test() ->
ok = forget_legacy_nodes(),
Guild = legacy_guild([ok]),
Payload = request_counts_payload(#{7 => {Guild, make_ref()}}),
?assertEqual([], maps:get(<<"counts">>, Payload)).
handle_request_mixes_legacy_and_current_guilds_test() ->
ok = forget_legacy_nodes(),
Legacy = legacy_guild([#{member_count => 50, online_count => 10}]),
Current = spawn(fun() ->
receive
{'$gen_call', From, {get_viewer_counts, #{user_id := 100}}} ->
gen_server:reply(From, #{member_count => 80, online_count => 20});
{'$gen_call', From, {get_user_counts, 100}} ->
gen_server:reply(From, #{member_count => 80, online_count => 20})
end,
receive
{'$gen_call', From2, _} -> gen_server:reply(From2, unexpected)
after 500 -> ok
end
end),
Payload = request_counts_payload(#{7 => {Legacy, make_ref()}, 9 => {Current, make_ref()}}),
@@ -327,6 +353,7 @@ handle_request_mixes_legacy_and_current_guilds_test() ->
).
fetch_counts_does_not_fall_back_on_current_guild_test() ->
ok = forget_legacy_nodes(),
Self = self(),
Guild = spawn(fun() ->
receive
@@ -346,7 +373,39 @@ fetch_counts_does_not_fall_back_on_current_guild_test() ->
after 400 -> ok
end.
fetch_counts_goes_straight_to_user_counts_after_a_legacy_answer_test() ->
ok = forget_legacy_nodes(),
Self = self(),
Guild = spawn(fun() -> recording_legacy_guild(Self, 3) end),
?assertEqual({ok, 50, 10}, fetch_counts(Guild, 100)),
?assertEqual({ok, 50, 10}, fetch_counts(Guild, 100)),
Calls = [
receive
{legacy_call, Tag} -> Tag
after 1000 -> none
end
|| _ <- [1, 2, 3]
],
ok = forget_legacy_nodes(),
?assertEqual([get_viewer_counts, get_user_counts, get_user_counts], Calls).
recording_legacy_guild(_Parent, 0) ->
ok;
recording_legacy_guild(Parent, N) ->
receive
{'$gen_call', From, {get_viewer_counts, _}} ->
Parent ! {legacy_call, get_viewer_counts},
gen_server:reply(From, ok);
{'$gen_call', From, {get_user_counts, 100}} ->
Parent ! {legacy_call, get_user_counts},
gen_server:reply(From, #{member_count => 50, online_count => 10})
after 1000 ->
ok
end,
recording_legacy_guild(Parent, N - 1).
fetch_counts_errors_on_malformed_reply_test() ->
ok = forget_legacy_nodes(),
Guild = spawn(fun() ->
receive
{'$gen_call', From, {get_viewer_counts, _}} -> gen_server:reply(From, #{})
fluxer_gateway/src/guild/guild_sessions.erl
+5 -71
View File
@@ -77,44 +77,13 @@ handle_session_down(Ref, State) ->
session_id(), session_data(), reference(), {session_id(), session_data()}, guild_state()
) -> {noreply, guild_state()} | {stop, normal, guild_state()}.
handle_pending_ref_down(SessionId, Session, Ref, Found, State) ->
Sessions = maps:get(sessions, State, #{}),
case pending_session_owns_connected_tracking(Session, Sessions, State) of
true -> guild_sessions_connect:handle_session_down(Ref, Found, State);
false -> handle_pending_session_down(SessionId, Ref, Sessions, State)
case guild_sessions_connect:counts_as_connected(Session) of
true ->
guild_sessions_connect:handle_session_down(Ref, Found, State);
false ->
handle_pending_session_down(SessionId, Ref, maps:get(sessions, State, #{}), State)
end.
-spec pending_session_owns_connected_tracking(session_data(), sessions_map(), guild_state()) ->
boolean().
pending_session_owns_connected_tracking(Session, Sessions, State) ->
UserId = maps:get(user_id, Session, undefined),
Counts = maps:get(user_session_counts, State, #{}),
TrackedCount = non_negative_count(maps:get(UserId, Counts, 0)),
TrackedCount > active_session_count(UserId, Sessions).
-spec non_negative_count(term()) -> non_neg_integer().
non_negative_count(Count) when is_integer(Count), Count >= 0 -> Count;
non_negative_count(_) -> 0.
-spec active_session_count(user_id() | undefined, sessions_map()) -> non_neg_integer().
active_session_count(UserId, Sessions) ->
maps:fold(
fun(_SessionId, Session, Count) ->
count_active_session(UserId, Session, Count)
end,
0,
Sessions
).
-spec count_active_session(user_id() | undefined, session_data(), non_neg_integer()) ->
non_neg_integer().
count_active_session(UserId, #{user_id := UserId} = Session, Count) ->
case maps:get(pending_connect, Session, false) of
true -> Count;
false -> Count + 1
end;
count_active_session(_UserId, _Session, Count) ->
Count.
-spec handle_pending_session_down(session_id(), reference(), sessions_map(), guild_state()) ->
{noreply, guild_state()}.
handle_pending_session_down(SessionId, Ref, Sessions, State) ->
@@ -757,41 +726,6 @@ filter_sessions_for_message_test() ->
filter_sessions_for_message(Sessions, 5, <<"1">>, undefined, State)
).
non_negative_count_test() ->
?assertEqual(3, non_negative_count(3)),
?assertEqual(0, non_negative_count(0)),
?assertEqual(0, non_negative_count(-1)),
?assertEqual(0, non_negative_count(undefined)).
active_session_count_test() ->
Sessions = #{
<<"a">> => #{user_id => 1, pending_connect => true},
<<"b">> => #{user_id => 1, pending_connect => false},
<<"c">> => #{user_id => 1},
<<"d">> => #{user_id => 2}
},
?assertEqual(2, active_session_count(1, Sessions)),
?assertEqual(1, active_session_count(2, Sessions)),
?assertEqual(0, active_session_count(3, Sessions)).
pending_session_owns_connected_tracking_untracked_test() ->
Session = #{user_id => 1, pending_connect => true},
Sessions = #{<<"a">> => Session},
?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, #{})).
pending_session_owns_connected_tracking_other_session_owns_test() ->
Session = #{user_id => 1, pending_connect => true},
Active = #{user_id => 1, pending_connect => false},
Sessions = #{<<"a">> => Session, <<"b">> => Active},
State = #{user_session_counts => #{1 => 1}},
?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, State)).
pending_session_owns_connected_tracking_true_test() ->
Session = #{user_id => 1, pending_connect => true},
Sessions = #{<<"a">> => Session},
State = #{user_session_counts => #{1 => 1}},
?assertEqual(true, pending_session_owns_connected_tracking(Session, Sessions, State)).
handle_pending_session_down_keeps_tracking_test() ->
Ref = make_ref(),
Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true},
fluxer_gateway/src/guild/guild_sessions_connect.erl
+29 -10
View File
@@ -15,6 +15,7 @@
remove_session_ref/2,
build_session_ref_index/1,
remove_session/2,
counts_as_connected/1,
invalidate_viewable_channels_cache/1
]).
@@ -299,14 +300,28 @@ remove_session(SessionId, State) ->
) -> guild_state().
do_remove_session(SessionId, Session, State) ->
maybe_demonitor_session(Session),
UserId = maps:get(user_id, Session, undefined),
StateAfterCleanup = cleanup_disconnecting_session(Session, State),
StateAfterCleanup = cleanup_removed_session(Session, State),
SessionsAfterCleanup = maps:get(sessions, StateAfterCleanup, #{}),
NewSessions = maps:remove(SessionId, SessionsAfterCleanup),
State2 = remove_session_ref(
remove_session_ref(
maps:get(mref, Session, undefined), StateAfterCleanup#{sessions => NewSessions}
),
track_connected_user(UserId, -1, State2).
).
-spec cleanup_removed_session(session_data(), guild_state()) -> guild_state().
cleanup_removed_session(Session, State) ->
case counts_as_connected(Session) of
true ->
UserId = maps:get(user_id, Session, undefined),
track_connected_user(UserId, -1, cleanup_disconnecting_session(Session, State));
false ->
cleanup_session_subscriptions(Session, State)
end.
-spec counts_as_connected(session_data()) -> boolean().
counts_as_connected(#{pending_connect := true} = Session) ->
maps:get(owns_connected_tracking, Session, false) =:= true;
counts_as_connected(_Session) ->
true.
-spec maybe_demonitor_session(session_data()) -> ok.
maybe_demonitor_session(Session) ->
@@ -393,15 +408,19 @@ cleanup_disconnecting_session(undefined, State) ->
State;
cleanup_disconnecting_session(Session, State) ->
UserId = maps:get(user_id, Session),
State1 = guild_sessions_presence:unsubscribe_from_user_presence(UserId, State),
cleanup_session_subscriptions(Session, State1).
-spec cleanup_session_subscriptions(session_data(), guild_state()) -> guild_state().
cleanup_session_subscriptions(Session, State) ->
SessionId = maps:get(session_id, Session),
GuildId = require_guild_id(maps:get(id, State)),
passive_sync_registry:delete(SessionId, GuildId),
State1 = guild_sessions_presence:unsubscribe_from_user_presence(UserId, State),
State2 = guild_member_list:unsubscribe_session(SessionId, State1),
MemberSubs = maps:get(member_subscriptions, State2, guild_subscriptions:init_state()),
State1 = guild_member_list:unsubscribe_session(SessionId, State),
MemberSubs = maps:get(member_subscriptions, State1, guild_subscriptions:init_state()),
NewMemberSubs = guild_subscriptions:unsubscribe_session(SessionId, MemberSubs),
State3 = State2#{member_subscriptions => NewMemberSubs},
guild_sessions_connect_cleanup:cleanup_connect_admission_for_session(SessionId, State3).
State2 = State1#{member_subscriptions => NewMemberSubs},
guild_sessions_connect_cleanup:cleanup_connect_admission_for_session(SessionId, State2).
-spec maybe_resection_disconnected_user(
user_id(), guild_state(), guild_state()
fluxer_gateway/src/guild/guild_sessions_presence.erl
+15 -8
View File
@@ -6,6 +6,7 @@
-export([
subscribe_connected_user_presence/2,
subscribe_to_user_presence/2,
subscribe_without_cached_presence/2,
unsubscribe_from_user_presence/2,
unsubscribe_many_from_user_presence/3,
handle_user_offline/2
@@ -21,17 +22,23 @@ subscribe_connected_user_presence(UserId, State) ->
-spec subscribe_to_user_presence(user_id(), guild_state()) -> guild_state().
subscribe_to_user_presence(UserId, State) ->
case subscribe_without_cached_presence(UserId, State) of
{fresh, StateWithSubs} -> maybe_send_cached_presence(UserId, StateWithSubs);
{existing, StateWithSubs} -> StateWithSubs
end.
-spec subscribe_without_cached_presence(user_id(), guild_state()) ->
{fresh | existing, guild_state()}.
subscribe_without_cached_presence(UserId, State) ->
PresenceSubs = maps:get(presence_subscriptions, State, #{}),
CurrentCount = maps:get(UserId, PresenceSubs, 0),
case CurrentCount of
case maps:get(UserId, PresenceSubs, 0) of
0 ->
presence_bus:subscribe(UserId),
NewSubs = PresenceSubs#{UserId => 1},
StateWithSubs = State#{presence_subscriptions => NewSubs},
maybe_send_cached_presence(UserId, StateWithSubs);
_ ->
NewSubs = PresenceSubs#{UserId => CurrentCount + 1},
State#{presence_subscriptions => NewSubs}
{fresh, State#{presence_subscriptions => PresenceSubs#{UserId => 1}}};
CurrentCount ->
{existing, State#{
presence_subscriptions => PresenceSubs#{UserId => CurrentCount + 1}
}}
end.
-spec unsubscribe_from_user_presence(user_id(), guild_state()) -> guild_state().
fluxer_gateway/src/guild/guild_subscription_handler.erl
+21 -7
View File
@@ -177,13 +177,27 @@ flush_lazy_subscribe_buffer(State) ->
State1 = maps:remove(lazy_subscribe_buffer, State),
State2 = maps:remove(lazy_subscribe_order, State1),
State3 = maps:remove(lazy_subscribe_timer, State2),
lists:foldl(
fun(BufferKey, AccState) ->
process_buffered_lazy_subscribe(BufferKey, Buffer, AccState)
end,
State3,
Order
).
flush_lazy_subscribe_keys(Order, Buffer, State3).
-spec flush_lazy_subscribe_keys([lazy_subscribe_key()], map(), guild_state()) -> guild_state().
flush_lazy_subscribe_keys([], _Buffer, State) ->
State;
flush_lazy_subscribe_keys([BufferKey | Rest], Buffer, State) ->
Engines = map_size(maps:get(?ENGINES_KEY, State, #{})),
State1 = process_buffered_lazy_subscribe(BufferKey, Buffer, State),
case Rest =/= [] andalso map_size(maps:get(?ENGINES_KEY, State1, #{})) > Engines of
true -> defer_lazy_subscribe_keys(Rest, Buffer, State1);
false -> flush_lazy_subscribe_keys(Rest, Buffer, State1)
end.
-spec defer_lazy_subscribe_keys([lazy_subscribe_key()], map(), guild_state()) -> guild_state().
defer_lazy_subscribe_keys(Keys, Buffer, State) ->
Ref = erlang:send_after(0, self(), flush_lazy_subscribe_buffer),
State#{
lazy_subscribe_buffer => maps:with(Keys, Buffer),
lazy_subscribe_order => Keys,
lazy_subscribe_timer => Ref
}.
-spec move_buffer_key_to_tail(lazy_subscribe_key(), [lazy_subscribe_key()]) ->
[lazy_subscribe_key()].
fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl
+29 -8
View File
@@ -3,7 +3,7 @@
-module(guild_subscription_mutual_channels).
-typing([eqwalizer]).
-export([filter_member_ids/3, filter_session_member_ids/2]).
-export([filter_member_ids/3, filter_session_member_ids/2, filter_session_member_ids/3]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@@ -12,7 +12,9 @@
-type guild_state() :: map().
-type user_id() :: integer().
-type memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => boolean()}}.
-export_type([guild_state/0, user_id/0]).
-type view_cache() :: #{term() => [integer()]}.
-type view_memo() :: #{exceptions => sets:set(user_id()), views => view_cache()}.
-export_type([guild_state/0, user_id/0, view_memo/0]).
-spec filter_member_ids(user_id(), [user_id()], guild_state()) -> [user_id()].
filter_member_ids(_SessionUserId, [], _State) ->
@@ -30,12 +32,23 @@ filter_member_ids(SessionUserId, MemberIds, State) ->
-spec filter_session_member_ids([{term(), user_id(), [term()]}], guild_state()) ->
#{term() => [user_id()]}.
filter_session_member_ids([], _State) ->
#{};
filter_session_member_ids(Requests, State) ->
Exceptions = exceptions(State),
{Results, _Memo} = filter_session_member_ids(Requests, #{}, State),
Results.
-spec filter_session_member_ids([{term(), user_id(), [term()]}], view_memo(), guild_state()) ->
{#{term() => [user_id()]}, view_memo()}.
filter_session_member_ids([], Memo, _State) ->
{#{}, Memo};
filter_session_member_ids(Requests, Memo, State) ->
Exceptions =
case Memo of
#{exceptions := Cached} -> Cached;
_ -> exceptions(State)
end,
Views = maps:get(views, Memo, #{}),
Sessions = maps:get(sessions, State, #{}),
{Results, _Cache} = lists:foldl(
{Results, Cache} = lists:foldl(
fun({SessionId, SessionUserId, MemberIds}, {Acc, Cache}) ->
SessionMap = request_session_map(SessionId, SessionUserId, Sessions, State),
{Kept, Cache1} = keep_session_members(
@@ -43,10 +56,18 @@ filter_session_member_ids(Requests, State) ->
),
{Acc#{SessionId => Kept}, Cache1}
end,
{#{}, #{}},
{#{}, Views},
Requests
),
Results.
{Results, #{exceptions => Exceptions, views => role_views(Cache)}}.
-spec role_views(view_cache()) -> view_cache().
role_views(Cache) ->
maps:filter(fun(Key, _Channels) -> is_role_key(Key) end, Cache).
-spec is_role_key(term()) -> boolean().
is_role_key({roles, _RawRoles}) -> true;
is_role_key(_Key) -> false.
-spec request_session_map(term(), user_id(), map(), guild_state()) -> map().
request_session_map(SessionId, SessionUserId, Sessions, State) ->
fluxer_gateway/src/presence/presence_cache_bulk.erl
+111 -44
View File
@@ -35,11 +35,51 @@ bulk_get_inner(UserIds) ->
-spec bulk_get_map_inner([integer()]) -> #{integer() => map()}.
bulk_get_map_inner(UserIds) ->
UniqueUserIds = normalize_user_ids(UserIds),
PrimaryPresenceMap = fetch_primary_presences(UniqueUserIds),
MissingUserIds = [U || U <- UniqueUserIds, not maps:is_key(U, PrimaryPresenceMap)],
FallbackPresenceMap = fetch_fallback_presences(MissingUserIds),
maps:merge(PrimaryPresenceMap, FallbackPresenceMap).
{Local, Remote} = lists:partition(
fun({OwnerNode, _OwnerUserIds}) -> OwnerNode =:= node() end,
group_user_ids_by_owner(UserIds)
),
LocalMap = lists:foldl(
fun({_OwnerNode, OwnerUserIds}, AccMap) ->
maps:merge(AccMap, local_bulk_presence_map(OwnerUserIds))
end,
#{},
Local
),
Servers = [
{{presence_cache, OwnerNode}, OwnerUserIds}
|| {OwnerNode, OwnerUserIds} <- Remote
],
maps:merge(LocalMap, fetch_remote_groups(Servers)).
-spec fetch_remote_groups([{gen_server:server_ref(), [integer()]}]) -> #{integer() => map()}.
fetch_remote_groups([]) ->
#{};
fetch_remote_groups(Groups) ->
Deadline = erlang:monotonic_time(millisecond) + ?REMOTE_CALL_TIMEOUT_MS,
Requests = [
{Server, Ids, gen_server:send_request(Server, {bulk_get_local_map, Ids})}
|| {Server, Ids} <- Groups
],
lists:foldl(
fun({Server, Ids, ReqId}, AccMap) ->
maps:merge(AccMap, remote_group_reply(Server, Ids, ReqId, Deadline))
end,
#{},
Requests
).
-spec remote_group_reply(
gen_server:server_ref(), [integer()], gen_server:request_id(), integer()
) ->
#{integer() => map()}.
remote_group_reply(Server, Ids, ReqId, Deadline) ->
case gen_server:receive_response(ReqId, {abs, Deadline}) of
{reply, Reply} when is_map(Reply) ->
sanitize_presence_map(Reply);
_ ->
map_from_presence_list(safe_server_call(Server, {bulk_get_local, Ids}, []))
end.
-spec get_from_cluster(integer()) -> {ok, map()} | not_found.
get_from_cluster(UserId) ->
@@ -100,7 +140,15 @@ safe_remote_call(TargetNode, Request, Fallback) ->
safe_remote_call(TargetNode, Request, Fallback, Timeout) when
is_integer(Timeout), Timeout > 0
->
try gen_server:call({presence_cache, TargetNode}, Request, Timeout) of
safe_server_call({presence_cache, TargetNode}, Request, Fallback, Timeout).
-spec safe_server_call(gen_server:server_ref(), term(), term()) -> term().
safe_server_call(Server, Request, Fallback) ->
safe_server_call(Server, Request, Fallback, ?REMOTE_CALL_TIMEOUT_MS).
-spec safe_server_call(gen_server:server_ref(), term(), term(), pos_integer()) -> term().
safe_server_call(Server, Request, Fallback, Timeout) ->
try gen_server:call(Server, Request, Timeout) of
Reply -> Reply
catch
error:_ -> Fallback;
@@ -199,44 +247,6 @@ presence_values(PresenceMap) ->
UserIds = lists:sort(maps:keys(PresenceMap)),
[maps:get(UserId, PresenceMap) || UserId <- UserIds].
-spec fetch_primary_presences([integer()]) -> #{integer() => map()}.
fetch_primary_presences(UniqueUserIds) ->
OwnerGroups = group_user_ids_by_owner(UniqueUserIds),
lists:foldl(
fun({OwnerNode, OwnerUserIds}, AccMap) ->
maps:merge(AccMap, fetch_from_node(OwnerNode, OwnerUserIds))
end,
#{},
OwnerGroups
).
-spec fetch_from_node(node(), [integer()]) -> #{integer() => map()}.
fetch_from_node(OwnerNode, UserIds) ->
case OwnerNode =:= node() of
true -> local_bulk_presence_map(UserIds);
false -> fetch_remote_bulk_presence_map(OwnerNode, UserIds)
end.
-spec fetch_fallback_presences([integer()]) -> #{integer() => map()}.
fetch_fallback_presences([]) ->
#{};
fetch_fallback_presences(MissingUserIds) ->
OwnerGroups = group_user_ids_by_owner(MissingUserIds),
lists:foldl(
fun({OwnerNode, OwnerUserIds}, AccMap) ->
maps:merge(AccMap, fetch_fallback_from_node(OwnerNode, OwnerUserIds))
end,
#{},
OwnerGroups
).
-spec fetch_fallback_from_node(node(), [integer()]) -> #{integer() => map()}.
fetch_fallback_from_node(OwnerNode, UserIds) ->
case OwnerNode =:= node() of
true -> local_bulk_presence_map(UserIds);
false -> fetch_remote_bulk_presence_map(OwnerNode, UserIds)
end.
-spec fetch_from_owner_nodes(integer(), [node()]) -> {ok, map()} | not_found.
fetch_from_owner_nodes(_UserId, []) ->
not_found;
@@ -327,6 +337,63 @@ group_user_ids_by_owner_matches_single_owner_resolution_test() ->
restore_persistent_term(Key, Previous)
end.
remote_groups_are_fetched_in_parallel_test() ->
Test = self(),
First = spawn(fun() ->
fake_presence_server(Test, #{1 => #{<<"status">> => <<"online">>}})
end),
Second = spawn(fun() ->
fake_presence_server(Test, #{2 => #{<<"status">> => <<"idle">>}})
end),
Legacy = spawn(fun() -> fake_presence_server(Test, legacy) end),
Fetcher = spawn(fun() ->
Test ! {fetched, fetch_remote_groups([{First, [1]}, {Second, [2]}, {Legacy, [3]}])}
end),
Received = [receive_request() || _ <- [First, Second, Legacy]],
Expected = [
{First, {bulk_get_local_map, [1]}},
{Second, {bulk_get_local_map, [2]}},
{Legacy, {bulk_get_local_map, [3]}}
],
?assertEqual(lists:sort(Expected), lists:sort(Received)),
[Server ! go || Server <- [First, Second, Legacy]],
?assertMatch({Legacy, {bulk_get_local, [3]}}, receive_request()),
Legacy ! go,
Fetched =
receive
{fetched, Map} -> Map
after 5000 -> timeout
end,
?assertEqual(
#{
1 => #{<<"status">> => <<"online">>},
2 => #{<<"status">> => <<"idle">>},
3 => #{<<"user">> => #{<<"id">> => 3}}
},
Fetched
),
[exit(Pid, kill) || Pid <- [First, Second, Legacy, Fetcher]].
fake_presence_server(Test, Reply) ->
receive
{'$gen_call', From, Request} ->
Test ! {request, self(), Request},
receive
go -> gen_server:reply(From, fake_reply(Reply, Request))
end,
fake_presence_server(Test, Reply)
end.
fake_reply(legacy, {bulk_get_local_map, _Ids}) -> unknown_request;
fake_reply(legacy, {bulk_get_local, Ids}) -> [#{<<"user">> => #{<<"id">> => Id}} || Id <- Ids];
fake_reply(Map, _Request) -> Map.
receive_request() ->
receive
{request, Server, Request} -> {Server, Request}
after 900 -> no_request
end.
restore_persistent_term(Key, undefined) ->
persistent_term:erase(Key);
restore_persistent_term(Key, Value) ->
fluxer_gateway/src/telemetry/gateway_stall_monitor.erl
+281
View File
@@ -0,0 +1,281 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_stall_monitor).
-typing([eqwalizer]).
-behaviour(gen_server).
-export([start_link/0, snapshot/0, reset/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-define(TICK_MS, 100).
-define(LATE_TICK_MS, 250).
-define(SEVERE_MS, 1000).
-define(RECENT_MAX, 32).
-define(LOG_INTERVAL_MS, 10_000).
-define(DEFAULT_LONG_SCHEDULE_MS, 200).
-define(DEFAULT_LONG_GC_MS, 200).
-define(SNAPSHOT_TIMEOUT_MS, 5000).
-type kind() :: long_schedule | long_gc | busy_dist_port | timer_late.
-type stat() :: #{
count := non_neg_integer(), severe := non_neg_integer(), max_ms := non_neg_integer()
}.
-type event() :: #{
at_ms := integer(),
kind := kind(),
ms := non_neg_integer(),
who := pid() | port() | undefined,
info := term()
}.
-type monitor_status() :: installed | not_owner | disabled.
-type state() :: #{
since_ms := integer(),
status := monitor_status(),
stats := #{kind() => stat()},
recent := [event()],
tick_due := integer(),
last_log_ms := integer(),
suppressed := non_neg_integer()
}.
-spec start_link() -> {ok, pid()} | {error, term()}.
start_link() ->
case gen_server:start_link({local, ?MODULE}, ?MODULE, [], []) of
{ok, Pid} -> {ok, Pid};
ignore -> {error, ignore};
{error, E} -> {error, E}
end.
-spec snapshot() -> map().
snapshot() ->
Raw = gen_server:call(?MODULE, snapshot, ?SNAPSHOT_TIMEOUT_MS),
Recent = maps:get(recent, Raw, []),
Raw#{recent => [describe(E) || E <- Recent]}.
-spec reset() -> ok.
reset() ->
gen_server:call(?MODULE, reset, ?SNAPSHOT_TIMEOUT_MS).
-spec init([]) -> {ok, state()}.
init([]) ->
_ = process_flag(priority, high),
Now = now_ms(),
{ok, #{
since_ms => erlang:system_time(millisecond),
status => install(),
stats => #{},
recent => [],
tick_due => schedule_tick(Now),
last_log_ms => Now - ?LOG_INTERVAL_MS,
suppressed => 0
}}.
-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.
handle_call(snapshot, _From, State) ->
{reply, view(State), State};
handle_call(reset, _From, State) ->
{reply, ok, State#{
since_ms := erlang:system_time(millisecond), stats := #{}, recent := [], suppressed := 0
}};
handle_call(_Request, _From, State) ->
{reply, ok, State}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info({monitor, Who, Kind, Info}, State) when
Kind =:= long_schedule; Kind =:= long_gc; Kind =:= busy_dist_port
->
{noreply, record(Kind, event_ms(Kind, Info), Who, compact_info(Kind, Info), State)};
handle_info(tick, #{tick_due := Due} = State) ->
Now = now_ms(),
State1 = State#{tick_due := schedule_tick(Now)},
{noreply, maybe_record_late(Now - Due, State1)};
handle_info(_Info, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, #{status := installed}) ->
_ = release(),
ok;
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {ok, state()}.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-spec install() -> monitor_status().
install() ->
case enabled() of
false ->
disabled;
true ->
case erlang:system_monitor() of
undefined ->
_ = erlang:system_monitor(self(), [
{long_schedule,
threshold(
stall_monitor_long_schedule_ms, ?DEFAULT_LONG_SCHEDULE_MS
)},
{long_gc, threshold(stall_monitor_long_gc_ms, ?DEFAULT_LONG_GC_MS)},
busy_dist_port
]),
installed;
{Pid, _} when Pid =:= self() ->
installed;
_ ->
not_owner
end
end.
-spec release() -> ok.
release() ->
case erlang:system_monitor() of
{Pid, _} when Pid =:= self() ->
_ = erlang:system_monitor(undefined),
ok;
_ ->
ok
end.
-spec enabled() -> boolean().
enabled() ->
application:get_env(fluxer_gateway, stall_monitor_enabled, true) =/= false.
-spec threshold(atom(), pos_integer()) -> pos_integer().
threshold(Key, Default) ->
case application:get_env(fluxer_gateway, Key) of
{ok, N} when is_integer(N), N > 0 -> N;
_ -> Default
end.
-spec maybe_record_late(integer(), state()) -> state().
maybe_record_late(Late, State) when Late >= ?LATE_TICK_MS ->
record(timer_late, Late, undefined, #{late_ms => Late}, State);
maybe_record_late(_Late, State) ->
State.
-spec record(kind(), non_neg_integer(), pid() | port() | undefined, term(), state()) -> state().
record(Kind, Ms, Who, Info, #{stats := Stats, recent := Recent} = State) ->
Event = #{
at_ms => erlang:system_time(millisecond),
kind => Kind,
ms => Ms,
who => Who,
info => Info
},
State1 = State#{
stats := Stats#{Kind => bump(maps:get(Kind, Stats, empty_stat()), Ms)},
recent := lists:sublist([Event | Recent], ?RECENT_MAX)
},
maybe_log(Event, State1).
-spec empty_stat() -> stat().
empty_stat() ->
#{count => 0, severe => 0, max_ms => 0}.
-spec bump(stat(), non_neg_integer()) -> stat().
bump(#{count := C, severe := S, max_ms := M}, Ms) ->
#{
count => C + 1,
severe => S + severe_inc(Ms),
max_ms => max(M, Ms)
}.
-spec severe_inc(non_neg_integer()) -> 0 | 1.
severe_inc(Ms) when Ms >= ?SEVERE_MS -> 1;
severe_inc(_Ms) -> 0.
-spec maybe_log(event(), state()) -> state().
maybe_log(Event, #{last_log_ms := Last, suppressed := Suppressed} = State) ->
Now = now_ms(),
case Now - Last >= ?LOG_INTERVAL_MS of
true ->
logger:warning("Scheduler stall detected", Event#{suppressed => Suppressed}),
State#{last_log_ms := Now, suppressed := 0};
false ->
State#{suppressed := Suppressed + 1}
end.
-spec event_ms(kind(), term()) -> non_neg_integer().
event_ms(_Kind, Info) when is_list(Info) ->
case proplists:get_value(timeout, Info) of
Ms when is_integer(Ms), Ms >= 0 -> Ms;
_ -> 0
end;
event_ms(_Kind, _Info) ->
0.
-spec compact_info(kind(), term()) -> term().
compact_info(long_schedule, Info) when is_list(Info) ->
maps:from_list([{K, V} || {K, V} <- Info, K =:= in orelse K =:= out]);
compact_info(long_gc, Info) when is_list(Info) ->
maps:from_list([
{K, V}
|| {K, V} <- Info, K =:= heap_size orelse K =:= old_heap_size orelse K =:= mbuf_size
]);
compact_info(_Kind, Info) when is_pid(Info); is_port(Info) ->
#{peer => Info};
compact_info(_Kind, _Info) ->
#{}.
-spec view(state()) -> map().
view(#{since_ms := Since, status := Status, stats := Stats, recent := Recent, suppressed := S}) ->
#{
since_ms => Since,
status => Status,
stats => Stats,
recent => Recent,
suppressed_logs => S
}.
-spec describe(event()) -> map().
describe(#{who := Who} = Event) when is_pid(Who) ->
Event#{who_info => pid_info(Who)};
describe(#{who := Who} = Event) when is_port(Who) ->
Event#{who_info => port_name(Who)};
describe(Event) ->
Event.
-spec pid_info(pid()) -> term().
pid_info(Pid) when node(Pid) =/= node() ->
remote;
pid_info(Pid) ->
Base =
case erlang:process_info(Pid, [registered_name, {dictionary, '$initial_call'}]) of
undefined -> #{alive => false};
Items -> maps:from_list([{item_key(K), V} || {K, V} <- Items])
end,
Base#{guild_id => guild_id_of(Pid)}.
-spec item_key(term()) -> term().
item_key({dictionary, '$initial_call'}) -> initial_call;
item_key(K) -> K.
-spec guild_id_of(pid()) -> integer() | undefined.
guild_id_of(Pid) ->
try ets:select(guild_pid_cache, [{{'$1', '$2'}, [{'=:=', '$2', Pid}], ['$1']}], 1) of
{[GuildId | _], _} when is_integer(GuildId) -> GuildId;
_ -> undefined
catch
error:badarg -> undefined
end.
-spec port_name(port()) -> term().
port_name(Port) ->
case erlang:port_info(Port, name) of
{name, Name} -> Name;
undefined -> closed
end.
-spec schedule_tick(integer()) -> integer().
schedule_tick(Now) ->
_ = erlang:send_after(?TICK_MS, self(), tick),
Now + ?TICK_MS.
-spec now_ms() -> integer().
now_ms() ->
erlang:monotonic_time(millisecond).
fluxer_gateway/test/fluxer_gateway_config_tests.erl
+34
View File
@@ -187,6 +187,40 @@ rpc_concurrency_key_defaults_test() ->
?assertEqual(512, maps:get(gateway_nats_rpc_max_handlers, Config)),
?assertEqual(512, maps:get(gateway_http_rpc_max_concurrency, Config)).
pinned_node_defaults_keep_release_behaviour_test() ->
Config = fluxer_gateway_config:load(),
?assertEqual(true, maps:get(nats_rpc_enabled, Config)),
?assertEqual([], maps:get(pinned_guild_ids, Config)),
?assertEqual(undefined, maps:get(guild_pin_keeper_beam, Config)).
pinned_node_env_test() ->
with_envs(
[
{"FLUXER_GATEWAY_NATS_RPC_ENABLED", "false"},
{"FLUXER_GATEWAY_PINNED_GUILD_IDS", "1100000000000000001, 42"},
{"FLUXER_GATEWAY_GUILD_PIN_KEEPER_BEAM", "/etc/fluxer/gw/gateway_node_router.beam"},
{"FLUXER_GATEWAY_GUILD_PIN_KEEPER_BEAM_MD5", "D5E42B1D6D85C4CDEE93AA0CCA18A420"}
],
fun() ->
Config = fluxer_gateway_config:load(),
?assertEqual(false, maps:get(nats_rpc_enabled, Config)),
?assertEqual([42, 1100000000000000001], maps:get(pinned_guild_ids, Config)),
?assertEqual(
"/etc/fluxer/gw/gateway_node_router.beam",
maps:get(guild_pin_keeper_beam, Config)
),
?assertEqual(
<<"D5E42B1D6D85C4CDEE93AA0CCA18A420">>,
maps:get(guild_pin_keeper_beam_md5, Config)
)
end
).
pinned_guild_ids_reject_non_snowflakes_test() ->
with_env("FLUXER_GATEWAY_PINNED_GUILD_IDS", "1100000000000000001,ab", fun() ->
?assertError({invalid_pinned_guild_id, "ab"}, fluxer_gateway_config:load())
end).
optional_string_test() ->
?assertEqual(undefined, fluxer_gateway_config:optional_string(undefined)),
?assertEqual("hello", fluxer_gateway_config:optional_string(<<"hello">>)),
fluxer_gateway/test/gateway_stall_monitor_tests.erl
+93
View File
@@ -0,0 +1,93 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_stall_monitor_tests).
-include_lib("eunit/include/eunit.hrl").
stall_monitor_test_() ->
{foreach, fun setup/0, fun cleanup/1, [
fun installs_and_releases_system_monitor/1,
fun counts_forwarded_events_and_names_the_process/1,
fun leaves_a_foreign_system_monitor_alone/1,
fun records_late_timer_ticks/1
]}.
setup() ->
_ = erlang:system_monitor(undefined),
application:unset_env(fluxer_gateway, stall_monitor_enabled),
ok.
cleanup(_) ->
case whereis(gateway_stall_monitor) of
undefined -> ok;
Pid -> gen_server:stop(Pid)
end,
_ = erlang:system_monitor(undefined),
ok.
installs_and_releases_system_monitor(_) ->
fun() ->
{ok, Pid} = gateway_stall_monitor:start_link(),
unlink(Pid),
{Pid, Opts} = erlang:system_monitor(),
?assertEqual(200, proplists:get_value(long_schedule, Opts)),
?assertEqual(200, proplists:get_value(long_gc, Opts)),
?assertMatch(#{status := installed}, gateway_stall_monitor:snapshot()),
ok = gen_server:stop(Pid),
?assertEqual(undefined, erlang:system_monitor())
end.
counts_forwarded_events_and_names_the_process(_) ->
fun() ->
{ok, Pid} = gateway_stall_monitor:start_link(),
unlink(Pid),
Busy = spawn(fun() ->
receive
stop -> ok
end
end),
Info = [{timeout, 1500}, {in, {guild, handle_info, 2}}, {out, {guild, handle_info, 2}}],
Pid ! {monitor, Busy, long_schedule, Info},
Pid ! {monitor, Busy, long_gc, [{timeout, 300}, {heap_size, 10}, {old_heap_size, 0}]},
#{stats := Stats, recent := Recent} = gateway_stall_monitor:snapshot(),
?assertEqual(
#{count => 1, severe => 1, max_ms => 1500}, maps:get(long_schedule, Stats)
),
?assertEqual(#{count => 1, severe => 0, max_ms => 300}, maps:get(long_gc, Stats)),
[GcEvent, SchedEvent] = Recent,
?assertMatch(#{kind := long_gc, ms := 300, info := #{heap_size := 10}}, GcEvent),
?assertMatch(
#{kind := long_schedule, who := Busy, info := #{in := {guild, handle_info, 2}}},
SchedEvent
),
?assertMatch(#{who_info := #{guild_id := undefined}}, SchedEvent),
Busy ! stop
end.
leaves_a_foreign_system_monitor_alone(_) ->
fun() ->
Foreign = spawn(fun() ->
receive
stop -> ok
end
end),
_ = erlang:system_monitor(Foreign, [{long_gc, 500}]),
{ok, Pid} = gateway_stall_monitor:start_link(),
unlink(Pid),
?assertMatch(#{status := not_owner}, gateway_stall_monitor:snapshot()),
ok = gen_server:stop(Pid),
?assertMatch({Foreign, _}, erlang:system_monitor()),
Foreign ! stop
end.
records_late_timer_ticks(_) ->
fun() ->
{ok, Pid} = gateway_stall_monitor:start_link(),
unlink(Pid),
_ = sys:replace_state(Pid, fun(S) -> S#{tick_due := maps:get(tick_due, S) - 1300} end),
Pid ! tick,
#{stats := Stats} = gateway_stall_monitor:snapshot(),
#{count := Count, max_ms := Max} = maps:get(timer_late, Stats),
?assert(Count >= 1),
?assert(Max >= 1200)
end.
fluxer_gateway/test/guild_channel_move_equivalence_tests.erl
+70
View File
@@ -479,6 +479,76 @@ stale_detection_tracks_every_permission_input_test_() ->
end)
end}.
twin_channel_engines_match_a_fresh_build_test_() ->
{timeout, 120, fun() ->
with_harness(fun() ->
{State0, Env} = materialize(twin_spec()),
Lists = twin_lists(),
State = lists:foldl(fun guild_member_list_channel_engine:ensure/2, State0, Lists),
Refs = [guild_member_list_channel_engine:ref(L, State) || L <- Lists],
?assertEqual(length(Lists), length(lists:usort(Refs))),
Final = lists:foldl(fun assert_engine_matches_fresh_build/2, State, Lists),
teardown(Final, Env)
end)
end}.
stale_twin_engine_is_not_cloned_test_() ->
{timeout, 120, fun() ->
with_harness(fun() ->
{State0, Env} = materialize(twin_spec()),
[A, B | _] = twin_lists(),
State1 = guild_member_list_channel_engine:ensure(A, State0),
Hidden = with_channels(
fun(C) ->
case integer_to_binary(channel_int_id(C)) of
A -> C#{<<"permission_overwrites">> => [hidden_overwrite()]};
_ -> C
end
end,
State1
),
[
ok = guild_member_list_channel_engine:update_user(user_id(I), A, Hidden)
|| I <- lists:seq(1, 60)
],
State2 = guild_member_list_channel_engine:ensure(B, Hidden),
?assert(guild_member_list_engine_inputs:is_stale(A, State2)),
?assertNotEqual(engine_content(A, State2), engine_content(B, State2)),
teardown(assert_engine_matches_fresh_build(B, State2), Env)
end)
end}.
twin_spec() ->
Restricted = [overwrite(role_id(1), 0, 0, view())],
single_session_spec(
renumber([
channel(?CHAN_BASE + 700, 0, null, []),
channel(?CHAN_BASE + 701, 0, null, []),
channel(?CHAN_BASE + 702, 0, null, Restricted),
channel(?CHAN_BASE + 703, 0, null, Restricted)
])
).
twin_lists() ->
[integer_to_binary(?CHAN_BASE + I) || I <- [700, 701, 702, 703]].
assert_engine_matches_fresh_build(ListId, State) ->
Built = engine_content(ListId, State),
Rebuilt = guild_member_list_channel_engine:rebuild(ListId, State),
?assertEqual(engine_content(ListId, Rebuilt), Built),
Rebuilt.
engine_content(ListId, State) ->
Ref = guild_member_list_channel_engine:ref(ListId, State),
Items = guild_member_list_engine:get_all_item_keys(Ref),
Members = [UserId || {member, UserId} <- Items],
{
guild_member_list_engine:get_counts(Ref),
guild_member_list_engine:get_groups(Ref),
Items,
[{U, guild_member_list_engine:is_member_online(Ref, U)} || U <- Members]
}.
input_changes() ->
Flip = flip_channel(),
[
fluxer_gateway/test/guild_connect_async_tests.erl
+126
View File
@@ -123,6 +123,132 @@ enqueue_session_connect_async_allows_immediate_start_when_wait_queue_disabled_te
WorkerRefs = maps:get(session_connect_worker_refs, State1, #{}),
cleanup_worker_refs(WorkerRefs).
finalize_batch_upserts_every_result_and_frees_one_worker_test() ->
UserId = 10,
Sessions = [<<"s1">>, <<"s2">>],
State0 = (finalize_state(<<"s1">>, UserId, 1, true, #{}, sets:new(), #{UserId => 1}))#{
sessions => maps:from_list([{S, pending_entry(S, UserId)} || S <- Sessions]),
session_connect_pending => maps:from_list([{S, 1} || S <- Sessions]),
session_connect_inflight => 2
},
State1 = guild_connect_async:finalize_session_connect_batch(
[{S, 1, {ok, #{}}, finalize_computed(S, UserId)} || S <- Sessions], State0
),
?assertEqual(1, maps:get(session_connect_inflight, State1)),
?assertEqual(#{}, maps:get(session_connect_pending, State1)),
[
?assertEqual(false, maps:get(pending_connect, maps:get(S, maps:get(sessions, State1))))
|| S <- Sessions
],
?assertEqual(#{UserId => 2}, maps:get(user_session_counts, State1)),
?assertEqual(3, maps:get(UserId, maps:get(presence_subscriptions, State1))),
?assertEqual(2, length(flush_connect_results())).
finalize_batch_skips_stale_attempts_test() ->
UserId = 11,
State0 = finalize_state(<<"s1">>, UserId, 2, true, #{}, sets:new(), #{UserId => 1}),
State1 = guild_connect_async:finalize_session_connect_batch(
[{<<"s1">>, 1, {ok, #{}}, finalize_computed(<<"s1">>, UserId)}], State0
),
?assertEqual(#{<<"s1">> => 2}, maps:get(session_connect_pending, State1)),
?assertEqual(
true, maps:get(pending_connect, maps:get(<<"s1">>, maps:get(sessions, State1)))
),
?assertEqual([], flush_connect_results()).
full_guild_reconnect_fits_the_connect_queue_test() ->
GuildId = 42,
State0 = (saturated_connect_state(GuildId, <<"s-0">>, 0, 1))#{
session_connect_queue => queue:new(),
session_connect_pending => #{},
sessions => #{}
},
State1 = lists:foldl(
fun(N, Acc) ->
SessionId = integer_to_binary(N),
guild_connect_async:enqueue_session_connect_async(
GuildId,
0,
connect_request(SessionId, N),
#{},
maps:remove(session_connect_max_queue, Acc)
)
end,
State0,
lists:seq(1, 3000)
),
?assertEqual(3000, queue:len(maps:get(session_connect_queue, State1))),
?assertEqual(3000, map_size(maps:get(session_connect_pending, State1))),
receive
{guild_connect_result, GuildId, _, {error, overloaded}} ->
?assert(false, connect_dropped)
after 0 ->
ok
end.
queued_connects_spread_over_batched_workers_test() ->
ok = flush_batches(),
GuildId = 42,
Requests = [connect_request(integer_to_binary(N), N) || N <- lists:seq(1, 20)],
State0 = #{
id => GuildId,
sessions => #{},
session_connect_queue => queue:from_list([
#{guild_id => GuildId, attempt => 0, request => R, reply_via_pid => undefined}
|| R <- Requests
]),
session_connect_pending => #{},
session_connect_inflight => 0,
data => #{},
member_count => 0,
voice_states => #{},
member_list_engine => undefined,
virtual_channel_access => #{}
},
State1 = guild_connect_async:maybe_start_session_connect_workers(State0),
WorkerRefs = maps:get(session_connect_worker_refs, State1),
?assertEqual(8, maps:get(session_connect_inflight, State1)),
?assertEqual(8, map_size(WorkerRefs)),
?assertEqual(0, queue:len(maps:get(session_connect_queue, State1))),
Batches = [receive_batch() || _ <- lists:seq(1, 8)],
?assertEqual(
lists:sort([maps:get(session_id, R) || R <- Requests]),
lists:sort([SessionId || Batch <- Batches, {SessionId, 0, _, _} <- Batch])
),
cleanup_worker_refs(WorkerRefs).
pending_entry(SessionId, UserId) ->
#{
session_id => SessionId,
user_id => UserId,
pid => self(),
mref => make_ref(),
pending_connect => true,
active_guilds => sets:new()
}.
receive_batch() ->
receive
{'$gen_cast', {session_connect_worker_batch_done, Results}} -> Results
after 5000 ->
?assert(false, connect_batch_not_received)
end.
flush_batches() ->
receive
{'$gen_cast', {session_connect_worker_batch_done, _Results}} -> flush_batches()
after 100 ->
ok
end.
flush_connect_results() ->
receive
{guild_connect_result, _GuildId, _Attempt, _Reply} = Msg ->
[Msg | flush_connect_results()]
after 0 ->
[]
end.
finalize_state(SessionId, UserId, Attempt, PendingConnect, Counts, Connected, PresenceSubs) ->
Existing = #{
session_id => SessionId,
fluxer_gateway/test/guild_data_tests.erl
+29
View File
@@ -81,6 +81,35 @@ read_model_observes_role_and_collection_changes_test() ->
cleanup_read_model(State)
end.
read_model_observes_last_message_and_pin_advances_test() ->
State = read_model_state(),
try
ok = guild_read_model:put_state(State),
Data = maps:get(data, State),
Advanced = guild_state_channels:handle_message_create(
#{<<"channel_id">> => <<"500">>, <<"id">> => <<"900">>}, Data
),
Pinned = guild_state_channels:handle_channel_pins_update(
#{
<<"channel_id">> => <<"500">>,
<<"last_pin_timestamp">> => <<"2026-10-02T00:00:00Z">>
},
Advanced
),
Updated = State#{data => Pinned},
ok = guild_read_model:update(State, Updated),
{reply, Expected, _} = guild_data:get_guild_data(#{user_id => 200}, Updated),
?assertEqual(
{ok, Expected}, guild_read_model:query(100, {get_guild_data, #{user_id => 200}})
),
#{guild_data := #{<<"channels">> := Channels}} = Expected,
[Channel] = [C || C <- Channels, maps:get(<<"id">>, C) =:= 500],
?assertEqual(900, maps:get(<<"last_message_id">>, Channel)),
?assertEqual(<<"2026-10-02T00:00:00Z">>, maps:get(<<"last_pin_timestamp">>, Channel))
after
cleanup_read_model(State)
end.
read_model_survives_blocked_owner_and_rejects_dead_tables_test() ->
State = read_model_state(),
Self = self(),
fluxer_gateway/test/guild_dm_partners_tests.erl
+16
View File
@@ -83,6 +83,17 @@ guild_wide_visibility_change_reevaluates_test() ->
?assertEqual({dm_partner_mutual, ?GUILD_ID, [30]}, receive_mutual())
end).
partner_role_change_after_registration_reevaluates_test() ->
with_flag(true, fun() ->
Before = register_partners(state(), [30]),
?assertEqual(none, receive_mutual()),
After = with_member(member(30, [?VIEWER_ROLE]), Before),
_ = guild_dm_partners:maybe_reevaluate(
guild_member_update, #{<<"user">> => #{<<"id">> => <<"30">>}}, Before, After
),
?assertEqual({dm_partner_mutual, ?GUILD_ID, [30]}, receive_mutual())
end).
disconnected_sessions_are_dropped_on_reevaluation_test() ->
with_flag(true, fun() ->
Before = register_partners(state(), [20]),
@@ -136,6 +147,11 @@ members() ->
without_member(UserId, #{data := Data} = State) ->
State#{data => Data#{<<"members">> => maps:remove(UserId, maps:get(<<"members">>, Data))}}.
with_member(Member, #{data := Data} = State) ->
#{<<"user">> := #{<<"id">> := RawId}} = Member,
Members = maps:get(<<"members">>, Data),
State#{data => Data#{<<"members">> => Members#{binary_to_integer(RawId) => Member}}}.
with_channels(Channels, #{data := Data} = State) ->
State#{
data => Data#{
fluxer_gateway/test/guild_handoff_freeze_tests.erl
+468
View File
@@ -0,0 +1,468 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(guild_handoff_freeze_tests).
-behaviour(gen_server).
-include_lib("eunit/include/eunit.hrl").
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
-define(GUILD_ID, 4242).
-define(SHARD_TABLE, guild_manager_shard_table).
-define(SINK, guild_handoff_freeze_tests_sink).
transfer_test_() ->
{foreach, fun setup/0, fun cleanup/1, [
instantiate(fun frozen_backlog_reaches_target_and_source_applies_nothing/1),
instantiate(fun final_window_call_fails_unapplied_and_cast_is_kept/1),
instantiate(fun route_failure_restores_routes_resumes_source_and_stops_target/1),
instantiate(fun controller_death_before_start_resumes_source/1),
instantiate(fun controller_death_after_start_kills_target_and_resumes_source/1),
instantiate(fun target_death_before_commit_keeps_source/1),
instantiate(fun sessions_stay_behind_when_not_transferred/1),
instantiate(fun cast_during_source_terminate_reaches_target/1),
instantiate(fun monitor_flush_in_source_terminate_loses_no_cast/1),
instantiate(fun connect_worker_results_stay_with_the_source/1),
instantiate(fun abort_after_forwarding_reports_the_forwarded_count/1),
instantiate(fun late_source_stop_keeps_the_committed_target/1),
instantiate(fun controller_death_after_commit_keeps_the_target/1),
instantiate(fun lost_guard_aborts_before_other_nodes_route_to_the_target/1)
]}.
instantiate(Test) ->
fun(Ctx) -> {timeout, 30, fun() -> Test(Ctx) end} end.
frozen_backlog_reaches_target_and_source_applies_nothing(#{src := Src, src_shard := SrcShard}) ->
Test = self(),
BeforeStart = fun() ->
gen_server:cast(Src, {append, b}),
spawn(fun() -> Test ! {call_reply, gen_server:call(Src, {append_call, c}, 30000)} end),
wait_mailbox(Src, 2)
end,
Result = guild_handoff_freeze:transfer(
?GUILD_ID, Src, SrcShard, node(), #{before_start => BeforeStart}
),
?assertMatch({ok, #{new_pid := _}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual({call_reply, {ok, NewPid}}, receive_tagged(call_reply)),
?assertEqual([{Src, {shutdown, handoff}, [a]}], ets:lookup(?SINK, Src)),
?assertEqual([a, b, c], gen_server:call(NewPid, get_log)),
?assertEqual(false, is_process_alive(Src)).
final_window_call_fails_unapplied_and_cast_is_kept(#{src := Src, src_shard := SrcShard}) ->
Test = self(),
BeforeStart = fun() ->
gen_server:cast(Src, {append, b}),
spawn(fun() -> Test ! {call_reply, call_or_exit(Src, {append_call, c})} end),
wait_mailbox(Src, 2)
end,
Opts = #{before_start => BeforeStart, forward_rounds => 0},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({ok, #{final := #{cast := 1, dropped_call := 1}}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertMatch({call_reply, {exit, {{shutdown, handoff}, _}}}, receive_tagged(call_reply)),
?assertEqual([a, b], gen_server:call(NewPid, get_log)).
route_failure_restores_routes_resumes_source_and_stops_target(#{
src := Src, src_shard := SrcShard, dst_shard := DstShard
}) ->
Test = self(),
Opts = #{
before_start => fun() -> gen_server:cast(Src, {append, b}) end,
after_start => fun(_NewPid) -> {error, route_rejected} end,
on_abort => fun() ->
Test ! routes_restored,
ok
end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({error, #{phase := after_start, reason := route_rejected}}, Result),
?assertEqual(routes_restored, receive_tagged(routes_restored)),
?assertEqual({error, not_found}, gen_server:call(DstShard, {lookup, ?GUILD_ID})),
?assertEqual([a, b], gen_server:call(Src, get_log)),
?assertEqual(false, guild_handoff_freeze:is_frozen(Src)).
controller_death_before_start_resumes_source(#{src := Src, src_shard := SrcShard}) ->
Opts = #{
before_start => fun() ->
gen_server:cast(Src, {append, b}),
exit(self(), kill)
end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({error, #{phase := crashed, reason := killed}}, Result),
?assertEqual([a, b], gen_server:call(Src, get_log, 5000)).
controller_death_after_start_kills_target_and_resumes_source(#{
src := Src, src_shard := SrcShard, dst_shard := DstShard
}) ->
Test = self(),
Opts = #{
after_start => fun(NewPid) ->
Test ! {target, NewPid},
exit(self(), kill)
end,
on_abort => fun() ->
Test ! routes_restored,
ok
end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({error, #{phase := crashed, abort := #{routes := ok, target := ok}}}, Result),
?assertEqual(routes_restored, receive_tagged(routes_restored)),
{target, NewPid} = receive_tagged(target),
?assertEqual(ok, wait_dead(NewPid)),
?assertEqual({error, not_found}, gen_server:call(DstShard, {lookup, ?GUILD_ID})),
?assertEqual([a], gen_server:call(Src, get_log, 5000)).
target_death_before_commit_keeps_source(#{src := Src, src_shard := SrcShard}) ->
Opts = #{
after_start => fun(NewPid) ->
exit(NewPid, kill),
wait_dead(NewPid)
end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({error, #{phase := target_died, reason := killed}}, Result),
?assertEqual([a], gen_server:call(Src, get_log, 5000)),
?assertEqual(false, guild_handoff_freeze:is_frozen(Src)).
sessions_stay_behind_when_not_transferred(#{src := Src, src_shard := SrcShard}) ->
ok = gen_server:call(Src, {put_session, <<"s1">>, #{pid => self(), user_id => 7}}),
Opts = #{transfer_sessions => false},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({ok, #{new_pid := _}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual(#{}, gen_server:call(NewPid, get_sessions)),
?assertEqual([a], gen_server:call(NewPid, get_log)).
cast_during_source_terminate_reaches_target(#{src := Src, src_shard := SrcShard}) ->
ok = gen_server:call(Src, cast_late_on_terminate),
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), #{}),
?assertMatch({ok, #{final := #{cast := 1}}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual([a, late], gen_server:call(NewPid, get_log)).
monitor_flush_in_source_terminate_loses_no_cast(#{src := Src, src_shard := SrcShard}) ->
ok = gen_server:call(Src, flush_down_on_terminate),
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), #{}),
?assertMatch({ok, #{final := #{cast := 1}}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual([a, late], gen_server:call(NewPid, get_log)).
connect_worker_results_stay_with_the_source(#{src := Src, src_shard := SrcShard}) ->
BeforeStart = fun() ->
gen_server:cast(Src, {session_connect_worker_done, <<"s1">>, 1, {ok, #{}}, #{}}),
gen_server:cast(
Src, {session_connect_worker_batch_done, [{<<"s1">>, 1, {ok, #{}}, #{}}]}
),
gen_server:cast(Src, {append, b}),
wait_mailbox(Src, 3)
end,
Result = guild_handoff_freeze:transfer(
?GUILD_ID, Src, SrcShard, node(), #{before_start => BeforeStart}
),
?assertMatch({ok, #{new_pid := _}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual([a, b], gen_server:call(NewPid, get_log)).
abort_after_forwarding_reports_the_forwarded_count(#{src := Src, src_shard := SrcShard}) ->
BeforeStart = fun() ->
gen_server:cast(Src, crash),
wait_mailbox(Src, 1)
end,
Result = guild_handoff_freeze:transfer(
?GUILD_ID, Src, SrcShard, node(), #{before_start => BeforeStart}
),
?assertMatch(
{error, #{phase := target_died, abort := #{forwarded := 1, thaw := ok}}}, Result
).
late_source_stop_keeps_the_committed_target(#{
src := Src, src_shard := SrcShard, dst_shard := DstShard
}) ->
Opts = #{
stop_timeout => 100,
after_start => fun(_NewPid) -> hold(SrcShard, 600) end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch({ok, #{new_pid := _}}, Result),
{ok, #{new_pid := NewPid}} = Result,
?assertEqual(ok, wait_dead(Src)),
?assert(is_process_alive(NewPid)),
?assertEqual({ok, NewPid}, gen_server:call(DstShard, {lookup, ?GUILD_ID})),
?assertEqual([a], gen_server:call(NewPid, get_log)).
controller_death_after_commit_keeps_the_target(#{
src := Src, src_shard := SrcShard, dst_shard := DstShard
}) ->
Test = self(),
Opts = #{
after_start => fun(NewPid) ->
Test ! {started, self(), NewPid},
hold(SrcShard, 1000)
end
},
spawn(fun() ->
Test ! {result, guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts)}
end),
{started, Controller, NewPid} = receive_tagged(started),
ok = wait_mailbox(SrcShard, 1),
exit(Controller, kill),
{result, Result} = receive_tagged(result),
?assertMatch({error, #{phase := crashed, committed := true}}, Result),
?assertEqual(ok, wait_dead(Src)),
?assert(is_process_alive(NewPid)),
?assertEqual({ok, NewPid}, gen_server:call(DstShard, {lookup, ?GUILD_ID})).
lost_guard_aborts_before_other_nodes_route_to_the_target(#{
src := Src, src_shard := SrcShard, dst_shard := DstShard
}) ->
Test = self(),
Opts = #{
before_start => fun() ->
{monitored_by, Watchers} = process_info(self(), monitored_by),
[exit(W, kill) || W <- Watchers, W =/= Test],
ok
end,
after_start => fun(_NewPid) ->
Test ! routed,
ok
end
},
Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts),
?assertMatch(
{error, #{
phase := after_start, reason := {guard_lost, killed}, abort := #{exposed := false}
}},
Result
),
?assertEqual({timeout, routed}, receive_tagged(routed, 200)),
?assertEqual({error, not_found}, gen_server:call(DstShard, {lookup, ?GUILD_ID})),
?assertEqual(false, guild_handoff_freeze:is_frozen(Src)),
?assertEqual([a], gen_server:call(Src, get_log, 5000)).
hold(Pid, Ms) ->
Test = self(),
spawn(fun() ->
true = erlang:suspend_process(Pid),
Test ! {held, Pid},
timer:sleep(Ms),
true = erlang:resume_process(Pid)
end),
receive
{held, Pid} -> ok
end.
export_drops_only_keys_the_importer_rebuilds_test() ->
Raw = #{
<<"guild">> => #{<<"id">> => <<"4242">>},
<<"roles">> => [#{<<"id">> => <<"4242">>, <<"permissions">> => <<"1024">>}],
<<"channels">> => [#{<<"id">> => <<"77">>, <<"type">> => 0}],
<<"members">> => [
#{<<"user">> => #{<<"id">> => <<"5">>}, <<"roles">> => []},
#{<<"user">> => #{<<"id">> => <<"6">>}, <<"roles">> => [<<"4242">>]}
]
},
Full = guild_data_index:normalize_map(Raw),
State = #{id => ?GUILD_ID, data => Full#{members_ets => make_ref()}, sessions => #{}},
Slim = maps:get(data, guild_handoff:export_handoff_state(State)),
?assertEqual([], [K || K <- guild_handoff:derived_data_keys(), maps:is_key(K, Slim)]),
Rebuild = fun(D) ->
maps:remove(member_list_revision, guild_data_index:normalize_map(D))
end,
?assertEqual(Rebuild(Full), Rebuild(Slim)).
export_keeps_channel_fields_that_only_the_index_holds_test() ->
Raw = #{
<<"guild">> => #{<<"id">> => <<"4242">>},
<<"channels">> => [
#{<<"id">> => <<"77">>, <<"type">> => 0, <<"last_message_id">> => <<"10">>}
]
},
Full = guild_state_channels:handle_message_create(
#{<<"channel_id">> => <<"77">>, <<"id">> => <<"900">>},
guild_data_index:normalize_map(Raw)
),
State = #{id => ?GUILD_ID, data => Full, sessions => #{}},
Slim = maps:get(data, guild_handoff:export_handoff_state(State)),
?assertMatch({rebuilt, _}, bounded_rebuild(Slim)),
{rebuilt, Rebuilt} = bounded_rebuild(Slim),
[Channel] = guild_data_index:channel_list(Rebuilt),
?assertEqual(900, maps:get(<<"last_message_id">>, Channel)).
bounded_rebuild(Data) ->
{Pid, Ref} = spawn_monitor(fun() ->
process_flag(max_heap_size, #{size => 4000000, kill => true, error_logger => false}),
exit({rebuilt, guild_data_index:normalize_map(Data)})
end),
receive
{'DOWN', Ref, process, Pid, Reason} -> Reason
after 10000 ->
exit(Pid, kill),
timeout
end.
setup() ->
ets:new(?SHARD_TABLE, [named_table, public, set]),
ets:new(?SINK, [named_table, public, set]),
{ok, SrcShard} = gen_server:start(?MODULE, shard, []),
{ok, DstShard} = gen_server:start(?MODULE, shard, []),
Index = guild_manager_shards:select_shard(?GUILD_ID, 1),
ets:insert(?SHARD_TABLE, [{shard_count, 1}, {{shard_pid, Index}, DstShard}]),
{ok, Src} = gen_server:call(
SrcShard, {start_transferred, ?GUILD_ID, guild_export([a])}
),
#{src => Src, src_shard => SrcShard, dst_shard => DstShard}.
cleanup(#{src_shard := SrcShard, dst_shard := DstShard}) ->
Guilds = lists:append([gen_server:call(S, all_guilds) || S <- [SrcShard, DstShard]]),
[exit(G, kill) || G <- Guilds],
[gen_server:stop(S) || S <- [SrcShard, DstShard]],
ets:delete(?SHARD_TABLE),
ets:delete(?SINK),
flush().
guild_export(Log) ->
#{
id => ?GUILD_ID,
data => #{<<"log">> => Log},
sessions => #{},
voice_states => #{}
}.
init(shard) ->
{ok, #{role => shard, guilds => #{}}};
init({guild, Export}) ->
process_flag(trap_exit, true),
{ok, Export#{role => guild}}.
handle_call({start_transferred, GuildId, Export}, _From, #{role := shard} = State) ->
{ok, Pid} = gen_server:start(?MODULE, {guild, Export}, []),
{reply, {ok, Pid}, put_guild(GuildId, Pid, State)};
handle_call({stop_guild, GuildId}, From, #{role := shard} = State) ->
handle_call({stop_guild, GuildId, normal}, From, State);
handle_call({stop_guild, GuildId, Reason}, _From, #{role := shard, guilds := Guilds} = State) ->
case maps:get(GuildId, Guilds, undefined) of
Pid when is_pid(Pid) -> stop_quietly(Pid, Reason);
undefined -> ok
end,
{reply, ok, State#{guilds => maps:remove(GuildId, Guilds)}};
handle_call({lookup, GuildId}, _From, #{role := shard, guilds := Guilds} = State) ->
case maps:get(GuildId, Guilds, undefined) of
Pid when is_pid(Pid) ->
case is_process_alive(Pid) of
true -> {reply, {ok, Pid}, State};
false -> {reply, {error, not_found}, State}
end;
undefined ->
{reply, {error, not_found}, State}
end;
handle_call(all_guilds, _From, #{role := shard, guilds := Guilds} = State) ->
{reply, maps:values(Guilds), State};
handle_call({get_guild_id}, _From, #{role := guild, id := Id} = State) ->
{reply, Id, State};
handle_call({put_session, Id, Session}, _From, #{role := guild, sessions := Sessions} = State) ->
{reply, ok, State#{sessions => Sessions#{Id => Session}}};
handle_call(flush_down_on_terminate, _From, #{role := guild} = State) ->
Helper = spawn(fun() ->
receive
stop -> ok
end
end),
{reply, ok, State#{flush_down => {Helper, erlang:monitor(process, Helper)}}};
handle_call(cast_late_on_terminate, _From, #{role := guild} = State) ->
{reply, ok, State#{cast_late => true}};
handle_call(get_sessions, _From, #{role := guild, sessions := Sessions} = State) ->
{reply, Sessions, State};
handle_call(get_log, _From, #{role := guild} = State) ->
{reply, log(State), State};
handle_call({append_call, Item}, _From, #{role := guild} = State) ->
{reply, {ok, self()}, append(Item, State)}.
handle_cast({append, Item}, #{role := guild} = State) ->
{noreply, append(Item, State)};
handle_cast(crash, #{role := guild} = State) ->
{stop, crashed, State}.
handle_info({'EXIT', _Pid, Reason}, #{role := guild} = State) ->
{stop, Reason, State};
handle_info(_Msg, State) ->
{noreply, State}.
terminate(Reason, #{role := guild} = State) ->
ets:insert(?SINK, {self(), Reason, log(State)}),
case maps:get(cast_late, State, false) of
true ->
gen_server:cast(self(), {append, late}),
timer:sleep(200);
false ->
ok
end,
case maps:get(flush_down, State, undefined) of
{Helper, Ref} ->
exit(Helper, kill),
timer:sleep(100),
erlang:demonitor(Ref, [flush]),
gen_server:cast(self(), {append, late}),
timer:sleep(200);
undefined ->
ok
end;
terminate(_Reason, _State) ->
ok.
put_guild(GuildId, Pid, #{guilds := Guilds} = State) ->
State#{guilds => Guilds#{GuildId => Pid}}.
log(#{data := Data}) ->
maps:get(<<"log">>, Data).
append(Item, #{data := Data} = State) ->
State#{data => Data#{<<"log">> => log(State) ++ [Item]}}.
call_or_exit(Pid, Request) ->
try
gen_server:call(Pid, Request, 30000)
catch
exit:Reason -> {exit, Reason}
end.
stop_quietly(Pid, Reason) ->
try
gen_server:stop(Pid, Reason, 5000)
catch
exit:_ -> ok
end.
wait_mailbox(Pid, N) ->
case process_info(Pid, message_queue_len) of
{message_queue_len, Len} when Len >= N ->
ok;
_ ->
timer:sleep(5),
wait_mailbox(Pid, N)
end.
wait_dead(Pid) ->
Ref = erlang:monitor(process, Pid),
receive
{'DOWN', Ref, process, Pid, _} -> ok
after 5000 -> still_alive
end.
receive_tagged(Tag) ->
receive_tagged(Tag, 10000).
receive_tagged(Tag, Timeout) ->
receive
Msg when element(1, Msg) =:= Tag -> Msg;
Tag -> Tag
after Timeout -> {timeout, Tag}
end.
flush() ->
receive
_ -> flush()
after 0 -> ok
end.
fluxer_gateway/test/guild_handoff_tests.erl
+9 -2
View File
@@ -36,7 +36,13 @@ restore_transferred_session_state_rebuilds_connected_counts_test() ->
<<"s1">> => #{pid => self(), user_id => UserId},
<<"s2">> => #{pid => self(), user_id => UserId, pending_connect => false},
<<"pending">> => #{pid => self(), user_id => 11, pending_connect => true},
<<"bad">> => #{user_id => 12}
<<"bad">> => #{user_id => 12},
<<"reconnecting">> => #{
pid => self(),
user_id => 13,
pending_connect => true,
owns_connected_tracking => true
}
},
State0 = #{
sessions => Sessions,
@@ -45,10 +51,11 @@ restore_transferred_session_state_rebuilds_connected_counts_test() ->
presence_subscriptions => #{UserId => 1}
},
State1 = guild_handoff:restore_transferred_session_state(State0),
?assertEqual(#{UserId => 2}, maps:get(user_session_counts, State1)),
?assertEqual(#{UserId => 2, 13 => 1}, maps:get(user_session_counts, State1)),
Connected = maps:get(connected_user_ids, State1),
?assertEqual(true, sets:is_element(UserId, Connected)),
?assertEqual(false, sets:is_element(11, Connected)),
?assertEqual(true, sets:is_element(13, Connected)),
?assertEqual(3, maps:get(UserId, maps:get(presence_subscriptions, State1))).
export_handoff_state_contains_all_required_keys_test() ->
fluxer_gateway/test/guild_manager_tests.erl
+29
View File
@@ -69,6 +69,35 @@ handoff_guild_ids_counts_attempts_and_successes_test() ->
?assertEqual(#{attempted => 2, handed_off => 1}, Result),
?assertEqual(3, maps:get(shard_count, FinalState)).
handoff_to_topology_keeps_guild_the_router_owns_here_test() ->
GuildId = 77,
ShardPid = spawn(fun() -> local_ids_stub_loop([GuildId]) end),
State = #{shards => #{0 => #{pid => ShardPid, ref => make_ref()}}, shard_count => 1},
persistent_term:put({gateway_cluster_membership, members}, [node()]),
persistent_term:put({gateway_cluster_membership, members_by_role}, #{guilds => [node()]}),
try
{Result, _State} = guild_manager_handoff:perform_handoff_to_topology(
['[email protected]'], State
),
?assertEqual(#{attempted => 0, handed_off => 0}, Result)
after
ShardPid ! stop,
persistent_term:erase({gateway_cluster_membership, members}),
persistent_term:erase({gateway_cluster_membership, members_by_role})
end.
local_ids_stub_loop(GuildIds) ->
receive
stop ->
ok;
{'$gen_call', From, get_local_guild_ids} ->
gen_server:reply(From, {ok, GuildIds}),
local_ids_stub_loop(GuildIds);
{'$gen_call', From, _Request} ->
gen_server:reply(From, {error, not_found}),
local_ids_stub_loop(GuildIds)
end.
find_shard_by_ref_found_test() ->
Ref = make_ref(),
Shards = #{0 => #{pid => self(), ref => Ref}},
fluxer_gateway/test/guild_member_list_chaos_tests.erl
+9 -2
View File
@@ -153,8 +153,7 @@ subscribe_and_drain(GuildPid, Requests) ->
end,
Requests
),
GuildPid ! flush_lazy_subscribe_buffer,
SubscribedState = get_guild_state(GuildPid),
SubscribedState = drain_lazy_subscribe_buffer(GuildPid),
assert_requests_subscribed(Requests, SubscribedState),
await_initial_syncs(Requests, SubscribedState),
_ = collect_captures(100),
@@ -259,6 +258,14 @@ presence_update(GuildPid, UserId, Status) ->
_ = gen_server:call(GuildPid, {get_counts}, 60000),
ok.
drain_lazy_subscribe_buffer(GuildPid) ->
GuildPid ! flush_lazy_subscribe_buffer,
State = get_guild_state(GuildPid),
case maps:is_key(lazy_subscribe_buffer, State) of
true -> drain_lazy_subscribe_buffer(GuildPid);
false -> State
end.
get_guild_state(GuildPid) ->
gen_server:call(GuildPid, {get_sessions}, 60000).
fluxer_gateway/test/guild_session_tracking_tests.erl
+396
View File
@@ -0,0 +1,396 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(guild_session_tracking_tests).
-typing([eqwalizer]).
-include_lib("eunit/include/eunit.hrl").
-define(GUILD_ID, 910100).
-define(CHANNEL_ID, 910500).
-define(USER_A, 810001).
-define(USER_B, 810002).
-define(USER_C, 810003).
handoff_reconnect_counts_each_session_once_test_() ->
{timeout, 60, fun handoff_reconnect_counts_each_session_once/0}.
handoff_reconnect_overflow_keeps_tracking_test_() ->
{timeout, 60, fun handoff_reconnect_overflow_keeps_tracking/0}.
reconnect_of_connected_session_counts_once_test_() ->
{timeout, 60, fun reconnect_of_connected_session_counts_once/0}.
not_member_reconnect_releases_tracking_test_() ->
{timeout, 60, fun not_member_reconnect_releases_tracking/0}.
reconnecting_session_down_releases_tracking_test_() ->
{timeout, 60, fun reconnecting_session_down_releases_tracking/0}.
fresh_pending_down_keeps_reconnecting_owner_tracked_test_() ->
{timeout, 60, fun fresh_pending_down_keeps_reconnecting_owner_tracked/0}.
unavailable_removal_of_fresh_pending_keeps_owner_tracked_test_() ->
{timeout, 60, fun unavailable_removal_of_fresh_pending_keeps_owner_tracked/0}.
fresh_pending_sibling_keeps_mutual_members_test_() ->
{timeout, 60, fun fresh_pending_sibling_keeps_mutual_members/0}.
handoff_reconnect_counts_each_session_once() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"a2">>, ?USER_A}, {<<"b1">>, ?USER_B}],
fun(Sessions) ->
with_guild(base_state(Sessions), fun(Source) ->
await_tracking(#{?USER_A => 2, ?USER_B => 1}, Source),
{ok, Exported} = gen_server:call(Source, export_handoff_state, 10000),
with_guild(transferred_state(Exported), fun(Target) ->
await_tracking(#{?USER_A => 2, ?USER_B => 1}, Target),
reconnect_all(Target, Sessions, 0),
await_tracking(#{?USER_A => 2, ?USER_B => 1}, Target),
kill_session(<<"b1">>, Sessions),
await_tracking(#{?USER_A => 2}, Target),
kill_session(<<"a1">>, Sessions),
await_tracking(#{?USER_A => 1}, Target),
kill_session(<<"a2">>, Sessions),
await_tracking(#{}, Target)
end)
end)
end
).
handoff_reconnect_overflow_keeps_tracking() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"b1">>, ?USER_B}, {<<"c1">>, ?USER_C}],
fun(Sessions) ->
with_guild(base_state(Sessions), fun(Source) ->
{ok, Exported} = gen_server:call(Source, export_handoff_state, 10000),
with_guild(transferred_state(Exported), fun(Target) ->
Expected = #{?USER_A => 1, ?USER_B => 1, ?USER_C => 1},
await_tracking(Expected, Target),
hold_connect_workers(Target, 1),
[
send_connect(Target, Sid, Sessions, 0)
|| Sid <- [<<"a1">>, <<"b1">>, <<"c1">>]
],
?assertEqual({error, overloaded}, await_result(<<"a1">>, 0)),
?assertEqual({error, overloaded}, await_result(<<"b1">>, 0)),
State = get_state(Target),
Dropped = maps:get(<<"a1">>, maps:get(sessions, State)),
?assertEqual(true, maps:get(pending_connect, Dropped)),
?assertNot(maps:is_key(<<"a1">>, maps:get(session_connect_pending, State))),
assert_tracking(Expected, State),
release_connect_workers(Target),
?assertMatch({ok, _, _}, await_result(<<"c1">>, 0)),
send_connect(Target, <<"a1">>, Sessions, 1),
send_connect(Target, <<"b1">>, Sessions, 1),
?assertMatch({ok, _, _}, await_result(<<"a1">>, 1)),
?assertMatch({ok, _, _}, await_result(<<"b1">>, 1)),
await_tracking(Expected, Target),
kill_session(<<"a1">>, Sessions),
await_tracking(#{?USER_B => 1, ?USER_C => 1}, Target)
end)
end)
end
).
reconnect_of_connected_session_counts_once() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"b1">>, ?USER_B}],
fun(Sessions) ->
with_guild(base_state(Sessions), fun(Guild) ->
reconnect_all(Guild, Sessions, 1),
reconnect_all(Guild, Sessions, 2),
State = get_state(Guild),
assert_tracking(#{?USER_A => 1, ?USER_B => 1}, State),
[
?assertEqual(false, maps:get(pending_connect, Entry))
|| Entry <- maps:values(maps:get(sessions, State))
],
kill_session(<<"b1">>, Sessions),
await_tracking(#{?USER_A => 1}, Guild)
end)
end
).
not_member_reconnect_releases_tracking() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"b1">>, ?USER_B}],
fun(Sessions) ->
with_guild(base_state(Sessions), fun(Guild) ->
sys:replace_state(Guild, fun(S) ->
S#{data => guild_data_index:remove_member(?USER_B, maps:get(data, S))}
end),
send_connect(Guild, <<"b1">>, Sessions, 1),
?assertEqual({error, not_member}, await_result(<<"b1">>, 1)),
State = get_state(Guild),
?assertNot(maps:is_key(<<"b1">>, maps:get(sessions, State))),
assert_tracking(#{?USER_A => 1}, State)
end)
end
).
reconnecting_session_down_releases_tracking() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"b1">>, ?USER_B}],
fun(Sessions) ->
with_guild(base_state(Sessions), fun(Guild) ->
hold_connect_workers(Guild, 16),
send_connect(Guild, <<"b1">>, Sessions, 1),
Pending = maps:get(<<"b1">>, maps:get(sessions, get_state(Guild))),
?assertEqual(true, maps:get(pending_connect, Pending)),
await_tracking(#{?USER_A => 1, ?USER_B => 1}, Guild),
kill_session(<<"b1">>, Sessions),
await_tracking(#{?USER_A => 1}, Guild)
end)
end
).
fresh_pending_down_keeps_reconnecting_owner_tracked() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"a2">>, ?USER_A}, {<<"c1">>, ?USER_C}],
fun(Sessions) ->
with_guild(base_state(without(<<"a1">>, Sessions)), fun(Guild) ->
hold_connect_workers(Guild, 16),
send_connect(Guild, <<"a2">>, Sessions, 1),
send_connect(Guild, <<"a1">>, Sessions, 1),
kill_session(<<"a1">>, Sessions),
await_tracking(#{?USER_A => 1, ?USER_C => 1}, Guild),
release_connect_workers(Guild),
?assertMatch({ok, _, _}, await_result(<<"a2">>, 1)),
await_tracking(#{?USER_A => 1, ?USER_C => 1}, Guild),
kill_session(<<"a2">>, Sessions),
await_tracking(#{?USER_C => 1}, Guild)
end)
end
).
unavailable_removal_of_fresh_pending_keeps_owner_tracked() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"a2">>, ?USER_A}],
fun(Sessions) ->
with_guild(base_state(without(<<"a1">>, Sessions)), fun(Guild) ->
hold_connect_workers(Guild, 16),
send_connect(Guild, <<"a1">>, Sessions, 1),
sys:replace_state(Guild, fun(S) ->
guild_sessions:remove_session(<<"a1">>, S)
end),
State = get_state(Guild),
?assertNot(maps:is_key(<<"a1">>, maps:get(sessions, State))),
assert_tracking(#{?USER_A => 1}, State)
end)
end
).
fresh_pending_sibling_keeps_mutual_members() ->
with_sessions(
[{<<"a1">>, ?USER_A}, {<<"a2">>, ?USER_A}, {<<"c1">>, ?USER_C}],
fun(Sessions) ->
with_guild(base_state(without(<<"a1">>, Sessions)), fun(Guild) ->
send_connect(Guild, <<"a2">>, Sessions, 1),
?assertMatch({ok, _, _}, await_result(<<"a2">>, 1)),
hold_connect_workers(Guild, 16),
send_connect(Guild, <<"a1">>, Sessions, 1),
State = get_state(Guild),
SessionMap = maps:get(<<"a2">>, maps:get(sessions, State)),
?assertEqual(
#{?CHANNEL_ID => true}, maps:get(viewable_channels, SessionMap)
),
?assertEqual(
true,
maps:get(pending_connect, maps:get(<<"a1">>, maps:get(sessions, State)))
),
?assertEqual(
#{?CHANNEL_ID => true},
guild_visibility_channels:get_cached_viewable_channel_map(?USER_A, State)
),
?assertEqual(
[?USER_C],
guild_subscription_mutual_channels:filter_member_ids(
?USER_A, [?USER_C], State
)
)
end)
end
).
await_tracking(Expected, Guild) ->
await_tracking(Expected, Guild, 100).
await_tracking(Expected, Guild, 0) ->
assert_tracking(Expected, get_state(Guild));
await_tracking(Expected, Guild, Tries) ->
case tracking(get_state(Guild)) =:= expected_tracking(Expected) of
true ->
ok;
false ->
timer:sleep(20),
await_tracking(Expected, Guild, Tries - 1)
end.
assert_tracking(Expected, State) ->
?assertEqual(expected_tracking(Expected), tracking(State)).
expected_tracking(Expected) ->
{Expected, lists:sort(maps:keys(Expected)), Expected}.
tracking(State) ->
{
maps:get(user_session_counts, State),
lists:sort(sets:to_list(maps:get(connected_user_ids, State))),
maps:get(presence_subscriptions, State)
}.
with_sessions(Specs, Fun) ->
Parent = self(),
Sessions = [
{Sid, UserId, spawn(fun() -> session_loop(Parent, Sid) end)}
|| {Sid, UserId} <- Specs
],
try
Fun(Sessions)
after
[exit(Pid, kill) || {_, _, Pid} <- Sessions],
flush()
end.
session_loop(Parent, Sid) ->
receive
{guild_connect_result, _GuildId, Attempt, Reply} ->
Parent ! {connect_result, Sid, Attempt, Reply},
session_loop(Parent, Sid);
_ ->
session_loop(Parent, Sid)
end.
without(Sid, Sessions) ->
lists:keydelete(Sid, 1, Sessions).
with_guild(State, Fun) ->
{ok, Pid} = gen_server:start(guild, State, []),
try
Fun(Pid)
after
catch gen_server:call(Pid, {terminate}, 5000)
end.
get_state(Guild) ->
gen_server:call(Guild, {get_sessions}, 10000).
hold_connect_workers(Guild, MaxQueue) ->
sys:replace_state(Guild, fun(S) ->
S#{session_connect_inflight => 8, session_connect_max_queue => MaxQueue}
end),
ok.
release_connect_workers(Guild) ->
sys:replace_state(Guild, fun(S) ->
guild_connect_async:maybe_start_session_connect_workers(
S#{session_connect_inflight => 0, session_connect_max_queue => 16}
)
end),
ok.
reconnect_all(Guild, Sessions, Attempt) ->
[send_connect(Guild, Sid, Sessions, Attempt) || {Sid, _, _} <- Sessions],
[?assertMatch({ok, _, _}, await_result(Sid, Attempt)) || {Sid, _, _} <- Sessions],
ok.
send_connect(Guild, Sid, Sessions, Attempt) ->
{Sid, UserId, Pid} = lists:keyfind(Sid, 1, Sessions),
Request = #{
session_id => Sid,
user_id => UserId,
session_pid => Pid,
bot => false,
is_staff => false,
initial_guild_id => ?GUILD_ID,
active_guilds => sets:from_list([?GUILD_ID])
},
gen_server:cast(
Guild,
{session_connect_async, #{
guild_id => ?GUILD_ID, attempt => Attempt, request => Request
}}
).
await_result(Sid, Attempt) ->
receive
{connect_result, Sid, Attempt, Reply} -> Reply
after 10000 ->
error({no_connect_result, Sid, Attempt})
end.
kill_session(Sid, Sessions) ->
{Sid, _UserId, Pid} = lists:keyfind(Sid, 1, Sessions),
Ref = monitor(process, Pid),
exit(Pid, kill),
receive
{'DOWN', Ref, process, Pid, _} -> ok
end.
flush() ->
receive
{connect_result, _, _, _} -> flush()
after 0 ->
ok
end.
transferred_state(Exported) ->
guild_manager_shard_lifecycle:normalize_transferred_guild_state(?GUILD_ID, Exported).
base_state(Sessions) ->
Members = [member(UserId) || UserId <- [?USER_A, ?USER_B, ?USER_C]],
#{
id => ?GUILD_ID,
member_count => length(Members),
sessions => maps:from_list([
{Sid, #{
session_id => Sid,
user_id => UserId,
pid => Pid,
active_guilds => sets:from_list([?GUILD_ID])
}}
|| {Sid, UserId, Pid} <- Sessions
]),
data => #{
<<"guild">> => #{
<<"id">> => ?GUILD_ID,
<<"owner_id">> => ?USER_A,
<<"features">> => [],
<<"member_count">> => length(Members)
},
<<"roles">> => [
#{
<<"id">> => ?GUILD_ID,
<<"name">> => <<"everyone">>,
<<"permissions">> =>
constants:view_channel_permission() bor
constants:view_channel_members_permission(),
<<"hoist">> => false,
<<"position">> => 0
}
],
<<"channels">> => [
#{
<<"id">> => ?CHANNEL_ID,
<<"name">> => <<"general">>,
<<"type">> => 0,
<<"permission_overwrites">> => []
}
],
<<"members">> => Members
}
}.
member(UserId) ->
Name = integer_to_binary(UserId),
#{
<<"user">> => #{
<<"id">> => UserId,
<<"username">> => <<"u", Name/binary>>,
<<"global_name">> => <<"U", Name/binary>>,
<<"bot">> => false
},
<<"nick">> => null,
<<"roles">> => [],
<<"joined_at">> => <<"2026-01-01T00:00:00.000000+00:00">>
}.