From 5f4295e3994bd0a7c51ec02c4327e36d8526b534 Mon Sep 17 00:00:00 2001 From: Hampus Date: Wed, 7 Oct 2026 15:24:40 +0200 Subject: [PATCH] feat(threads): gateway, messages and push thread support (#3250) --- .../channel_threads_config.erl | 590 +++++ .../channel_threads/channel_threads_flip.erl | 368 +++ .../src/gateway/fluxer_gateway_app.erl | 1 + .../src/gateway/fluxer_gateway_sup.erl | 16 +- .../src/gateway/gateway_handler.erl | 2 + .../src/gateway/gateway_handler_dispatch.erl | 63 +- .../src/gateway/gateway_handler_identify.erl | 3 + .../gateway/gateway_handler_rate_limit.erl | 33 + .../src/gateway/gateway_nats_rpc.erl | 1 + .../src/gateway/gateway_rpc_guild.erl | 1 + .../gateway/gateway_rpc_guild_channels.erl | 4 + .../gateway/gateway_rpc_guild_lifecycle.erl | 20 + .../src/gateway/gateway_rpc_presence.erl | 7 + .../src/gateway/metrics_handler.erl | 129 ++ fluxer_gateway/src/guild/guild.erl | 61 +- .../src/guild/guild_availability.erl | 16 +- .../src/guild/guild_connect_async.erl | 74 +- fluxer_gateway/src/guild/guild_data.erl | 46 +- .../src/guild/guild_data_channels_order.erl | 3 +- fluxer_gateway/src/guild/guild_data_index.erl | 3 +- .../src/guild/guild_data_index_channels.erl | 63 +- .../src/guild/guild_data_normalize.erl | 60 +- fluxer_gateway/src/guild/guild_data_wire.erl | 18 + fluxer_gateway/src/guild/guild_dispatch.erl | 97 +- .../src/guild/guild_dispatch_push.erl | 687 +++++- .../src/guild/guild_dispatch_send.erl | 63 +- fluxer_gateway/src/guild/guild_handoff.erl | 8 +- fluxer_gateway/src/guild/guild_health.erl | 8 + fluxer_gateway/src/guild/guild_init.erl | 24 +- .../src/guild/guild_manager_shard_fetch.erl | 51 +- .../guild/guild_manager_shard_lifecycle.erl | 5 +- .../src/guild/guild_member_list_connected.erl | 5 + .../src/guild/guild_members_mutation.erl | 60 +- .../src/guild/guild_mutual_online.erl | 53 +- .../src/guild/guild_passive_sync.erl | 21 +- .../src/guild/guild_permission_cache.erl | 5 +- .../src/guild/guild_permissions.erl | 42 +- .../guild/guild_permissions_overwrites.erl | 9 +- fluxer_gateway/src/guild/guild_presence.erl | 3 +- .../src/guild/guild_public_online.erl | 7 +- .../src/guild/guild_query_handler.erl | 18 +- fluxer_gateway/src/guild/guild_read_model.erl | 6 +- .../src/guild/guild_request_forum_unreads.erl | 408 ++++ fluxer_gateway/src/guild/guild_sessions.erl | 51 +- .../src/guild/guild_sessions_connect.erl | 15 +- .../src/guild/guild_sessions_passive.erl | 8 +- .../src/guild/guild_state_channels.erl | 2 + .../src/guild/guild_state_threads.erl | 526 +++++ .../src/guild/guild_subscription_handler.erl | 28 +- .../guild_subscription_mutual_channels.erl | 53 +- .../src/guild/guild_thread_dispatch.erl | 602 +++++ .../src/guild/guild_thread_flip.erl | 833 +++++++ .../src/guild/guild_thread_gate.erl | 754 +++++++ .../src/guild/guild_thread_load.erl | 999 +++++++++ .../src/guild/guild_thread_permissions.erl | 209 ++ .../src/guild/guild_thread_push.erl | 308 +++ .../src/guild/guild_thread_store.erl | 436 ++++ .../src/guild/guild_thread_subscriptions.erl | 732 ++++++ .../src/guild/guild_thread_view.erl | 187 ++ .../src/guild/guild_unified_subscriptions.erl | 62 + .../src/guild/guild_visibility_overwrites.erl | 58 +- .../src/guild/guild_visibility_roles.erl | 53 +- fluxer_gateway/src/presence/presence.erl | 11 + fluxer_gateway/src/push/push.erl | 2 +- .../src/push/push_eligibility_checks.erl | 347 +++ .../src/push/push_job_publisher.erl | 137 +- fluxer_gateway/src/push/push_notification.erl | 55 +- fluxer_gateway/src/session/session.erl | 46 + .../src/session/session_connection_guild.erl | 3 +- .../session_connection_guild_resolve.erl | 38 +- .../src/session/session_dispatch.erl | 185 +- fluxer_gateway/src/session/session_guilds.erl | 1 + fluxer_gateway/src/session/session_init.erl | 1 + .../src/session/session_lifecycle.erl | 4 +- .../session/session_manager_shard_drain.erl | 169 +- .../src/session/session_monitor.erl | 1 + .../src/session/session_state_transfer.erl | 4 +- fluxer_gateway/src/utils/constants.erl | 30 +- fluxer_gateway/src/utils/event_atoms.erl | 24 + .../src/utils/experiment_bucket.erl | 24 + .../test/channel_threads_config_tests.erl | 331 +++ .../test/channel_threads_flip_tests.erl | 296 +++ .../test/experiment_bucket_tests.erl | 41 + fluxer_gateway/test/gateway_handler_tests.erl | 119 + .../test/gateway_rpc_guild_channels_tests.erl | 28 + .../gateway_rpc_guild_lifecycle_tests.erl | 115 + .../test/guild_availability_tests.erl | 33 + .../test/guild_connect_async_tests.erl | 32 + fluxer_gateway/test/guild_data_tests.erl | 25 + .../test/guild_handoff_freeze_tests.erl | 57 +- fluxer_gateway/test/guild_handoff_tests.erl | 56 + .../guild_manager_shard_lifecycle_tests.erl | 8 + .../test/guild_session_tracking_tests.erl | 2 +- .../test/guild_thread_push_tests.erl | 194 ++ fluxer_gateway/test/guild_threads_tests.erl | 1986 +++++++++++++++++ .../test/session_lifecycle_tests.erl | 10 +- .../session_manager_shard_drain_tests.erl | 56 + .../test/thread_permission_cases.json | 1457 ++++++++++++ fluxer_messages/src/router_impl.rs | 61 +- fluxer_messages/src/shard_impl.rs | 675 +++++- fluxer_messages/src/types.rs | 10 + fluxer_push/src/delivery.rs | 39 +- fluxer_push/src/job.rs | 83 + fluxer_push/src/payload.rs | 117 +- fluxer_push/src/providers/mod.rs | 1 + fluxer_push/src/providers/own_relay.rs | 1 + fluxer_push/src/subscription.rs | 2 + .../experiment/ExperimentBucketVectors.json | 38 + 108 files changed, 15857 insertions(+), 176 deletions(-) create mode 100644 fluxer_gateway/src/channel_threads/channel_threads_config.erl create mode 100644 fluxer_gateway/src/channel_threads/channel_threads_flip.erl create mode 100644 fluxer_gateway/src/guild/guild_request_forum_unreads.erl create mode 100644 fluxer_gateway/src/guild/guild_state_threads.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_dispatch.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_flip.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_gate.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_load.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_permissions.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_push.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_store.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_subscriptions.erl create mode 100644 fluxer_gateway/src/guild/guild_thread_view.erl create mode 100644 fluxer_gateway/src/utils/experiment_bucket.erl create mode 100644 fluxer_gateway/test/channel_threads_config_tests.erl create mode 100644 fluxer_gateway/test/channel_threads_flip_tests.erl create mode 100644 fluxer_gateway/test/experiment_bucket_tests.erl create mode 100644 fluxer_gateway/test/gateway_rpc_guild_lifecycle_tests.erl create mode 100644 fluxer_gateway/test/guild_thread_push_tests.erl create mode 100644 fluxer_gateway/test/guild_threads_tests.erl create mode 100644 fluxer_gateway/test/thread_permission_cases.json create mode 100644 packages/schema/src/domains/experiment/ExperimentBucketVectors.json diff --git a/fluxer_gateway/src/channel_threads/channel_threads_config.erl b/fluxer_gateway/src/channel_threads/channel_threads_config.erl new file mode 100644 index 000000000..1643755f5 --- /dev/null +++ b/fluxer_gateway/src/channel_threads/channel_threads_config.erl @@ -0,0 +1,590 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(channel_threads_config). +-typing([eqwalizer]). +-behaviour(gen_server). + +-export([ + start_link/0, + config/0, + version/0, + enabled/0, + loaded/0, + ever_enabled/0, + guild_active/1, + guild_active/2, + user_active/1, + user_active/2, + user_excluded/1, + user_excluded/2, + guild_fields_changed/2, + user_fields_changed/2, + update_counts/0, + note_identify/2, + identify_counts/0, + identify_os_label/1 +]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-ifdef(TEST). +-export([validate_config/1, store_validated_config/3, default_config/0, apply_nats_payload/1]). +-endif. + +-define(PERSISTENT_TERM_KEY, channel_threads_config). +-define(PULLED_KEY, {channel_threads_config, pulled}). +-define(UPDATE_COUNTS_KEY, {channel_threads_config, update_counts}). +-define(IDENTIFY_COUNTS_KEY, {channel_threads_config, identify_counts}). +-define(NATS_SUBJECT, <<"config.channel.threads">>). +-define(FETCH_DELAY_MS, 2000). +-define(RECONCILE_INTERVAL_MS, 30000). +-define(NATS_SUBSCRIBE_RETRY_MS, 2000). +-define(MAX_TARGETED_IDS, 1000). +-define(MAX_SALT_BYTES, 64). +-define(MAX_ID_DIGITS, 20). +-define(DEFAULT_GUILD_SALT, <<"channel-threads-guild-v1">>). +-define(DEFAULT_USER_SALT, <<"channel-threads-user-v1">>). + +-type id_set() :: #{binary() => true}. +-type config() :: #{ + enabled := boolean(), + config_version := non_neg_integer(), + ever_enabled := boolean(), + guild_basis_points := non_neg_integer(), + guild_salt := binary(), + enabled_guilds := id_set(), + disabled_guilds := id_set(), + user_basis_points := non_neg_integer(), + user_salt := binary(), + included_users := id_set(), + excluded_users := id_set() +}. +-type state() :: #{ + nats_subscription := term(), + nats_monitor := reference() | undefined, + fetch_timer := reference() | undefined +}. +-type store_result() :: updated | unchanged | stale | rejected. +-type os_label() :: binary(). + +-export_type([config/0]). + +-spec start_link() -> gen_server:start_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec config() -> config(). +config() -> + case persistent_term:get(?PERSISTENT_TERM_KEY, undefined) of + #{config_version := _} = Config -> Config; + _ -> default_config() + end. + +-spec version() -> non_neg_integer(). +version() -> + maps:get(config_version, config()). + +-spec enabled() -> boolean(). +enabled() -> + maps:get(enabled, config()). + +-spec loaded() -> boolean(). +loaded() -> + persistent_term:get(?PULLED_KEY, false) orelse + case config() of + #{enabled := false, config_version := 0} -> false; + _ -> true + end. + +-spec ever_enabled() -> boolean(). +ever_enabled() -> + maps:get(ever_enabled, config()). + +-spec guild_active(integer()) -> boolean(). +guild_active(GuildId) -> + guild_active(config(), GuildId). + +-spec guild_active(config(), integer()) -> boolean(). +guild_active(#{enabled := true} = Config, GuildId) when is_integer(GuildId) -> + Id = integer_to_binary(GuildId), + #{ + enabled_guilds := Included, + disabled_guilds := Excluded, + guild_basis_points := BasisPoints, + guild_salt := Salt + } = Config, + active(Id, Included, Excluded, BasisPoints, Salt); +guild_active(_Config, _GuildId) -> + false. + +-spec user_active(integer()) -> boolean(). +user_active(UserId) -> + user_active(config(), UserId). + +-spec user_active(config(), integer()) -> boolean(). +user_active(#{enabled := true} = Config, UserId) when is_integer(UserId) -> + Id = integer_to_binary(UserId), + #{ + included_users := Included, + excluded_users := Excluded, + user_basis_points := BasisPoints, + user_salt := Salt + } = Config, + active(Id, Included, Excluded, BasisPoints, Salt); +user_active(_Config, _UserId) -> + false. + +-spec user_excluded(integer()) -> boolean(). +user_excluded(UserId) -> + user_excluded(config(), UserId). + +-spec user_excluded(config(), integer()) -> boolean(). +user_excluded(#{excluded_users := Excluded}, UserId) when is_integer(UserId) -> + maps:is_key(integer_to_binary(UserId), Excluded); +user_excluded(_Config, _UserId) -> + false. + +-spec active(binary(), id_set(), id_set(), non_neg_integer(), binary()) -> boolean(). +active(Id, Included, Excluded, BasisPoints, Salt) -> + case maps:is_key(Id, Excluded) of + true -> + false; + false -> + maps:is_key(Id, Included) orelse + (BasisPoints > 0 andalso experiment_bucket:bucket(Id, Salt) < BasisPoints) + end. + +-spec guild_fields_changed(config(), config()) -> boolean(). +guild_fields_changed(Previous, Current) -> + fields_changed( + [enabled, guild_basis_points, guild_salt, enabled_guilds, disabled_guilds], + Previous, + Current + ). + +-spec user_fields_changed(config(), config()) -> boolean(). +user_fields_changed(Previous, Current) -> + fields_changed( + [enabled, user_basis_points, user_salt, included_users, excluded_users], + Previous, + Current + ). + +-spec fields_changed([atom()], config(), config()) -> boolean(). +fields_changed(Keys, Previous, Current) -> + maps:with(Keys, Previous) =/= maps:with(Keys, Current). + +-spec update_counts() -> #{store_result() => non_neg_integer()}. +update_counts() -> + Counters = persistent_term:get(?UPDATE_COUNTS_KEY, undefined), + maps:from_list([{Result, read_count(Counters, Result)} || Result <- store_results()]). + +-spec note_identify(boolean(), map()) -> ok. +note_identify(Capable, Properties) -> + case persistent_term:get(?IDENTIFY_COUNTS_KEY, undefined) of + undefined -> + ok; + Counters -> + Os = identify_os_label(maps:get(<<"os">>, Properties, undefined)), + counters:add(Counters, identify_index(Capable, Os), 1) + end. + +-spec identify_counts() -> #{{boolean(), os_label()} => non_neg_integer()}. +identify_counts() -> + case persistent_term:get(?IDENTIFY_COUNTS_KEY, undefined) of + undefined -> + #{}; + Counters -> + maps:from_list([ + {{Capable, Os}, counters:get(Counters, identify_index(Capable, Os))} + || Capable <- [true, false], Os <- os_labels() + ]) + end. + +-spec identify_os_label(term()) -> os_label(). +identify_os_label(Os) when is_binary(Os) -> + classify_os(string:lowercase(Os)); +identify_os_label(_Os) -> + <<"other">>. + +-spec classify_os(unicode:chardata()) -> os_label(). +classify_os(Os) -> + first_os_match(Os, [ + {<<"android">>, [<<"android">>]}, + {<<"ios">>, [<<"ios">>, <<"iphone">>, <<"ipad">>]}, + {<<"macos">>, [<<"mac">>, <<"darwin">>, <<"os x">>]}, + {<<"windows">>, [<<"win">>]}, + {<<"linux">>, [<<"linux">>, <<"bsd">>]} + ]). + +-spec first_os_match(unicode:chardata(), [{os_label(), [binary()]}]) -> os_label(). +first_os_match(_Os, []) -> + <<"other">>; +first_os_match(Os, [{Label, Needles} | Rest]) -> + case lists:any(fun(Needle) -> string:find(Os, Needle) =/= nomatch end, Needles) of + true -> Label; + false -> first_os_match(Os, Rest) + end. + +-spec os_labels() -> [os_label()]. +os_labels() -> + [<<"android">>, <<"ios">>, <<"windows">>, <<"macos">>, <<"linux">>, <<"other">>]. + +-spec identify_index(boolean(), os_label()) -> pos_integer(). +identify_index(Capable, Os) -> + CapableOffset = + case Capable of + true -> 0; + false -> length(os_labels()) + end, + CapableOffset + os_index(Os, os_labels(), 1). + +-spec os_index(os_label(), [os_label()], pos_integer()) -> pos_integer(). +os_index(Os, [Os | _Rest], Index) -> + Index; +os_index(Os, [_Other | Rest], Index) -> + os_index(Os, Rest, Index + 1); +os_index(_Os, [], Index) -> + Index - 1. + +-spec init([]) -> {ok, state()}. +init([]) -> + erlang:process_flag(fullsweep_after, 50), + persistent_term:put(?PERSISTENT_TERM_KEY, config()), + ensure_identify_counters(), + self() ! subscribe_nats, + {ok, #{ + nats_subscription => undefined, + nats_monitor => undefined, + fetch_timer => erlang:send_after(?FETCH_DELAY_MS, self(), fetch_config) + }}. + +-spec ensure_identify_counters() -> ok. +ensure_identify_counters() -> + case persistent_term:get(?IDENTIFY_COUNTS_KEY, undefined) of + undefined -> + Size = 2 * length(os_labels()), + persistent_term:put(?IDENTIFY_COUNTS_KEY, counters:new(Size, [write_concurrency])); + _Counters -> + ok + end. + +-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}. +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +-spec handle_cast(term(), state()) -> {noreply, state()}. +handle_cast(_Msg, State) -> + {noreply, State}. + +-spec handle_info(term(), state()) -> {noreply, state()}. +handle_info(subscribe_nats, State) -> + {noreply, subscribe_to_nats(State)}; +handle_info(fetch_config, State) -> + count(fetch_config_from_api()), + {noreply, State#{ + fetch_timer => erlang:send_after(?RECONCILE_INTERVAL_MS, self(), fetch_config) + }}; +handle_info({nats_resubscribed, ?NATS_SUBJECT}, State) -> + count(fetch_config_from_api()), + {noreply, State}; +handle_info({nats_msg, ?NATS_SUBJECT, Payload, _ReplyTo}, State) when is_binary(Payload) -> + count(apply_nats_payload(Payload)), + {noreply, State}; +handle_info({'DOWN', MonRef, process, _Pid, _Reason}, #{nats_monitor := MonRef} = State) -> + erlang:send_after(?NATS_SUBSCRIBE_RETRY_MS, self(), subscribe_nats), + {noreply, State#{nats_subscription => undefined, nats_monitor => undefined}}; +handle_info(_Info, State) -> + {noreply, State}. + +-spec terminate(term(), state()) -> ok. +terminate(_Reason, _State) -> + ok. + +-spec code_change(term(), state(), term()) -> {ok, state()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +-spec default_config() -> config(). +default_config() -> + #{ + enabled => false, + config_version => 0, + ever_enabled => false, + guild_basis_points => 0, + guild_salt => ?DEFAULT_GUILD_SALT, + enabled_guilds => #{}, + disabled_guilds => #{}, + user_basis_points => 0, + user_salt => ?DEFAULT_USER_SALT, + included_users => #{}, + excluded_users => #{} + }. + +-spec fetch_config_from_api() -> store_result(). +fetch_config_from_api() -> + RpcRequest = #{<<"type">> => <<"get_channel_threads_config">>}, + case api_rpc_client:call(RpcRequest) of + {ok, #{<<"config">> := Config}} when is_map(Config) -> + store_valid_config(Config, api); + {ok, _Other} -> + logger:warning("Channel threads config: unexpected API response format"), + rejected; + {error, Reason} -> + logger:warning("Channel threads config failed to fetch from API", #{ + reason => Reason + }), + rejected + end. + +-spec apply_nats_payload(binary()) -> store_result(). +apply_nats_payload(Payload) -> + try json:decode(Payload) of + #{<<"type">> := <<"channel_threads_config">>, <<"config">> := Config} when + is_map(Config) + -> + store_valid_config(Config, nats); + #{<<"config">> := Config} when is_map(Config) -> + store_valid_config(Config, nats); + _Other -> + logger:warning("Channel threads config: unexpected NATS payload format"), + rejected + catch + Class:Reason -> + logger:warning("Channel threads config failed to decode NATS payload", #{ + class => Class, reason => Reason + }), + rejected + end. + +-spec store_valid_config(map(), api | nats) -> store_result(). +store_valid_config(Wire, Source) -> + case validate_config(Wire) of + {ok, Validated} -> + Result = store_validated_config(config(), Validated, Source), + persistent_term:put(?PULLED_KEY, true), + Result; + {error, {invalid_field, WireKey, Length}} -> + logger:warning("Channel threads config rejected invalid config", #{ + source => Source, field => WireKey, length => Length + }), + rejected + end. + +-spec store_validated_config(config(), config(), api | nats) -> store_result(). +store_validated_config(Previous, Previous, _Source) -> + unchanged; +store_validated_config(#{config_version := Held}, #{config_version := Offered}, Source) when + Offered < Held +-> + logger:warning("Channel threads config ignored a lower config_version", #{ + source => Source, held => Held, offered => Offered + }), + stale; +store_validated_config(Previous, Offered, Source) -> + Current = Offered#{ + ever_enabled := maps:get(ever_enabled, Previous) orelse maps:get(ever_enabled, Offered) + }, + store_changed_config(Previous, Current, Source). + +-spec store_changed_config(config(), config(), api | nats) -> store_result(). +store_changed_config(Previous, Previous, _Source) -> + unchanged; +store_changed_config(Previous, Current, Source) -> + persistent_term:put(?PERSISTENT_TERM_KEY, Current), + logger:notice("Channel threads config updated", #{ + source => Source, + config_version => maps:get(config_version, Current), + enabled => maps:get(enabled, Current), + guild_basis_points => maps:get(guild_basis_points, Current), + user_basis_points => maps:get(user_basis_points, Current), + enabled_guilds => map_size(maps:get(enabled_guilds, Current)), + disabled_guilds => map_size(maps:get(disabled_guilds, Current)), + included_users => map_size(maps:get(included_users, Current)), + excluded_users => map_size(maps:get(excluded_users, Current)) + }), + ok = channel_threads_flip:config_changed(Previous, Current), + updated. + +-spec validate_config(map()) -> + {ok, config()} | {error, {invalid_field, binary(), non_neg_integer() | undefined}}. +validate_config(Wire) -> + lists:foldl( + fun(Field, Acc) -> validate_field(Field, Wire, Acc) end, + {ok, default_config()}, + config_fields() + ). + +-spec config_fields() -> [{atom(), binary(), fun((term()) -> {ok, term()} | error)}]. +config_fields() -> + [ + {enabled, <<"enabled">>, fun validate_boolean/1}, + {config_version, <<"config_version">>, fun validate_config_version/1}, + {ever_enabled, <<"ever_enabled">>, fun validate_boolean/1}, + {guild_basis_points, <<"guild_basis_points">>, fun validate_basis_points/1}, + {guild_salt, <<"guild_salt">>, fun validate_salt/1}, + {enabled_guilds, <<"enabled_guild_ids">>, fun validate_ids/1}, + {disabled_guilds, <<"disabled_guild_ids">>, fun validate_ids/1}, + {user_basis_points, <<"user_basis_points">>, fun validate_basis_points/1}, + {user_salt, <<"user_salt">>, fun validate_salt/1}, + {included_users, <<"included_user_ids">>, fun validate_ids/1}, + {excluded_users, <<"excluded_user_ids">>, fun validate_ids/1} + ]. + +-spec validate_field( + {atom(), binary(), fun((term()) -> {ok, term()} | error)}, + map(), + {ok, config()} | {error, term()} +) -> {ok, config()} | {error, term()}. +validate_field(_Field, _Wire, {error, _} = Error) -> + Error; +validate_field({Key, WireKey, Validate}, Wire, {ok, Acc}) -> + case maps:find(WireKey, Wire) of + error -> {ok, Acc}; + {ok, Value} -> store_validated_field(Key, WireKey, Value, Validate(Value), Acc) + end. + +-spec store_validated_field(atom(), binary(), term(), {ok, term()} | error, config()) -> + {ok, config()} | {error, term()}. +store_validated_field(Key, _WireKey, _Value, {ok, Normalised}, Acc) -> + {ok, Acc#{Key => Normalised}}; +store_validated_field(_Key, WireKey, Value, error, _Acc) when is_list(Value) -> + {error, {invalid_field, WireKey, length(Value)}}; +store_validated_field(_Key, WireKey, _Value, error, _Acc) -> + {error, {invalid_field, WireKey, undefined}}. + +-spec validate_boolean(term()) -> {ok, boolean()} | error. +validate_boolean(Value) when is_boolean(Value) -> + {ok, Value}; +validate_boolean(_Value) -> + error. + +-spec validate_config_version(term()) -> {ok, non_neg_integer()} | error. +validate_config_version(Value) when is_integer(Value), Value >= 0 -> + {ok, Value}; +validate_config_version(_Value) -> + error. + +-spec validate_basis_points(term()) -> {ok, non_neg_integer()} | error. +validate_basis_points(Value) when is_integer(Value), Value >= 0 -> + case Value =< experiment_bucket:resolution() of + true -> {ok, Value}; + false -> error + end; +validate_basis_points(_Value) -> + error. + +-spec validate_salt(term()) -> {ok, binary()} | error. +validate_salt(Value) when is_binary(Value) -> + validate_salt_bytes(Value, byte_size(Value)); +validate_salt(_Value) -> + error. + +-spec validate_salt_bytes(binary(), non_neg_integer()) -> {ok, binary()} | error. +validate_salt_bytes(Value, Size) when Size >= 1, Size =< ?MAX_SALT_BYTES -> + validate_printable_ascii(Value, Value); +validate_salt_bytes(_Value, _Size) -> + error. + +-spec validate_printable_ascii(binary(), binary()) -> {ok, binary()} | error. +validate_printable_ascii(<<>>, Value) -> + {ok, Value}; +validate_printable_ascii(<>, Value) when Byte >= 16#20, Byte =< 16#7e -> + validate_printable_ascii(Rest, Value); +validate_printable_ascii(_Remaining, _Value) -> + error. + +-spec validate_ids(term()) -> {ok, id_set()} | error. +validate_ids(Value) when is_list(Value), length(Value) =< ?MAX_TARGETED_IDS -> + collect_ids(Value, #{}); +validate_ids(_Value) -> + error. + +-spec collect_ids([term()], id_set()) -> {ok, id_set()} | error. +collect_ids([], Acc) -> + {ok, Acc}; +collect_ids([Id | Rest], Acc) when is_binary(Id) -> + case is_id(Id, byte_size(Id)) of + true -> collect_ids(Rest, Acc#{Id => true}); + false -> error + end; +collect_ids(_Ids, _Acc) -> + error. + +-spec is_id(binary(), non_neg_integer()) -> boolean(). +is_id(Id, Size) when Size >= 1, Size =< ?MAX_ID_DIGITS -> + is_all_digits(Id); +is_id(_Id, _Size) -> + false. + +-spec is_all_digits(binary()) -> boolean(). +is_all_digits(<<>>) -> + true; +is_all_digits(<>) when Byte >= $0, Byte =< $9 -> + is_all_digits(Rest); +is_all_digits(_Remaining) -> + false. + +-spec subscribe_to_nats(state()) -> state(). +subscribe_to_nats(#{nats_subscription := Sid} = State) when Sid =/= undefined -> + State; +subscribe_to_nats(State) -> + case subscribe_to_config_subject() of + {ok, Sid} -> + MonRef = monitor_nats_rpc(), + logger:info("Channel threads config subscribed to NATS", #{subject => ?NATS_SUBJECT}), + count(fetch_config_from_api()), + State#{nats_subscription => Sid, nats_monitor => MonRef}; + {error, Reason} -> + logger:debug("Channel threads config waiting for NATS subscription", #{ + subject => ?NATS_SUBJECT, reason => Reason + }), + erlang:send_after(?NATS_SUBSCRIBE_RETRY_MS, self(), subscribe_nats), + State + end. + +-spec subscribe_to_config_subject() -> {ok, term()} | {error, term()}. +subscribe_to_config_subject() -> + try gateway_nats_rpc:subscribe(?NATS_SUBJECT, <<>>) of + {ok, Sid} -> {ok, Sid}; + {error, Reason} -> {error, Reason} + catch + Class:Reason -> {error, {Class, Reason}} + end. + +-spec monitor_nats_rpc() -> reference() | undefined. +monitor_nats_rpc() -> + case whereis(gateway_nats_rpc) of + Pid when is_pid(Pid) -> erlang:monitor(process, Pid); + _ -> undefined + end. + +-spec count(store_result()) -> ok. +count(Result) -> + counters:add(update_counters(), result_index(Result), 1). + +-spec update_counters() -> counters:counters_ref(). +update_counters() -> + case persistent_term:get(?UPDATE_COUNTS_KEY, undefined) of + undefined -> + Counters = counters:new(length(store_results()), [atomics]), + persistent_term:put(?UPDATE_COUNTS_KEY, Counters), + Counters; + Counters -> + Counters + end. + +-spec read_count(counters:counters_ref() | undefined, store_result()) -> non_neg_integer(). +read_count(undefined, _Result) -> + 0; +read_count(Counters, Result) -> + counters:get(Counters, result_index(Result)). + +-spec store_results() -> [store_result()]. +store_results() -> + [updated, unchanged, stale, rejected]. + +-spec result_index(store_result()) -> pos_integer(). +result_index(updated) -> 1; +result_index(unchanged) -> 2; +result_index(stale) -> 3; +result_index(rejected) -> 4. diff --git a/fluxer_gateway/src/channel_threads/channel_threads_flip.erl b/fluxer_gateway/src/channel_threads/channel_threads_flip.erl new file mode 100644 index 000000000..4c0817fa3 --- /dev/null +++ b/fluxer_gateway/src/channel_threads/channel_threads_flip.erl @@ -0,0 +1,368 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(channel_threads_flip). +-typing([eqwalizer]). +-behaviour(gen_server). + +-export([ + start_link/0, + config_changed/2, + self_heal/1, + flip_counts/0, + affected_targets/3 +]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-ifdef(TEST). +-export([schedule/4, take_due/3, heal/4]). +-endif. + +-define(REGISTRY_TABLE, process_registry_table). +-define(FLIP_COUNTS_KEY, {channel_threads_flip, flip_counts}). +-define(JITTER_WINDOW_MS, 60000). +-define(GUILD_BATCH_SIZE, 10). +-define(VIEWERS_BATCH_SIZE, 10). +-define(USER_BATCH_SIZE, 1000). +-define(BATCH_DELAY_MS, 100). +-define(HEAL_COOLDOWN_MS, 30000). + +-type kind() :: guild | viewers | user. +-type target() :: {kind(), integer(), pid()}. +-type pending_key() :: {kind(), integer()}. +-type due() :: gb_sets:set({integer(), pending_key(), pid()}). +-type pending() :: #{ + due := #{kind() => due()}, + keys := #{pending_key() => {integer(), pid()}} +}. +-type state() :: #{ + pending := pending(), + timer := reference() | undefined, + healed := #{integer() => integer()} +}. +-type counted() :: guild | viewers | user | heal. + +-spec start_link() -> gen_server:start_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec config_changed(channel_threads_config:config(), channel_threads_config:config()) -> ok. +config_changed(Previous, Current) -> + gen_server:cast(?MODULE, {config_changed, Previous, Current}). + +-spec self_heal(integer()) -> ok. +self_heal(GuildId) when is_integer(GuildId) -> + gen_server:cast(?MODULE, {self_heal, GuildId, self()}). + +-spec flip_counts() -> #{counted() => non_neg_integer()}. +flip_counts() -> + case persistent_term:get(?FLIP_COUNTS_KEY, undefined) of + undefined -> + #{}; + Counters -> + maps:from_list([ + {Kind, counters:get(Counters, count_index(Kind))} + || Kind <- counted() + ]) + end. + +-spec affected_targets( + channel_threads_config:config(), channel_threads_config:config(), [target()] +) -> [target()]. +affected_targets(Previous, Current, Candidates) -> + Changed = changed_fun(Previous, Current), + lists:foldr( + fun({Kind, Id, Pid}, Acc) -> local_target(Kind, Id, Pid, Changed, Acc) end, + [], + Candidates + ). + +-spec changed_fun(channel_threads_config:config(), channel_threads_config:config()) -> + fun((kind(), integer()) -> boolean()). +changed_fun(Previous, Current) -> + GuildsChanged = channel_threads_config:guild_fields_changed(Previous, Current), + UsersChanged = channel_threads_config:user_fields_changed(Previous, Current), + Unloaded = unloaded(Previous), + fun + (user, _Id) when Unloaded -> + false; + (Kind, Id) -> + target_changed(Kind, Id, GuildsChanged, UsersChanged, Previous, Current) + end. + +-spec unloaded(channel_threads_config:config()) -> boolean(). +unloaded(#{enabled := false, config_version := 0}) -> true; +unloaded(_Config) -> false. + +-spec target_changed( + kind(), + integer(), + boolean(), + boolean(), + channel_threads_config:config(), + channel_threads_config:config() +) -> boolean(). +target_changed(guild, GuildId, true, _UsersChanged, Previous, Current) -> + channel_threads_config:guild_active(Previous, GuildId) =/= + channel_threads_config:guild_active(Current, GuildId); +target_changed(viewers, GuildId, _GuildsChanged, true, _Previous, Current) -> + channel_threads_config:guild_active(Current, GuildId); +target_changed(user, UserId, _GuildsChanged, true, Previous, Current) -> + channel_threads_config:user_active(Previous, UserId) =/= + channel_threads_config:user_active(Current, UserId) orelse + (maps:get(enabled, Previous) orelse maps:get(enabled, Current)) andalso + channel_threads_config:user_excluded(Previous, UserId) =/= + channel_threads_config:user_excluded(Current, UserId); +target_changed(_Kind, _Id, _GuildsChanged, _UsersChanged, _Previous, _Current) -> + false. + +-spec init([]) -> {ok, state()}. +init([]) -> + erlang:process_flag(fullsweep_after, 50), + ensure_counters(), + {ok, #{pending => empty_pending(), timer => undefined, healed => #{}}}. + +-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}. +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +-spec handle_cast(term(), state()) -> {noreply, state()}. +handle_cast({config_changed, Previous, Current}, State) -> + case + (maps:get(enabled, Previous) orelse maps:get(enabled, Current)) andalso + (channel_threads_config:guild_fields_changed(Previous, Current) orelse + channel_threads_config:user_fields_changed(Previous, Current)) + of + true -> {noreply, schedule_changed(Previous, Current, State)}; + false -> {noreply, State} + end; +handle_cast({self_heal, GuildId, Pid}, State) when is_integer(GuildId), is_pid(Pid) -> + {Healed, Pending} = heal(GuildId, Pid, now_ms(), State), + {noreply, State#{healed := Healed, pending := Pending}}; +handle_cast(_Msg, State) -> + {noreply, State}. + +-spec handle_info(term(), state()) -> {noreply, state()}. +handle_info(flip_tick, State) -> + {Due, Pending} = take_due( + maps:get(pending, State), + now_ms(), + #{guild => ?GUILD_BATCH_SIZE, viewers => ?VIEWERS_BATCH_SIZE, user => ?USER_BATCH_SIZE} + ), + lists:foreach(fun send_flip/1, Due), + {noreply, arm_timer(State#{pending := Pending, timer := undefined})}; +handle_info(_Info, State) -> + {noreply, State}. + +-spec terminate(term(), state()) -> ok. +terminate(_Reason, _State) -> + ok. + +-spec code_change(term(), state(), term()) -> {ok, state()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +-spec schedule_changed( + channel_threads_config:config(), channel_threads_config:config(), state() +) -> state(). +schedule_changed(Previous, Current, State) -> + Targets = local_targets(changed_fun(Previous, Current)), + Pending = schedule(Targets, now_ms(), maps:get(pending, State), fun jitter/0), + arm_timer(State#{pending := Pending}). + +-spec empty_pending() -> pending(). +empty_pending() -> + #{ + due => #{guild => gb_sets:new(), viewers => gb_sets:new(), user => gb_sets:new()}, + keys => #{} + }. + +-spec schedule([target()], integer(), pending(), fun(() -> non_neg_integer())) -> pending(). +schedule(Targets, Now, Pending, Jitter) -> + lists:foldl( + fun(Target, Acc) -> schedule_target(Target, Now, Acc, Jitter) end, Pending, Targets + ). + +-spec schedule_target(target(), integer(), pending(), fun(() -> non_neg_integer())) -> + pending(). +schedule_target({Kind, Id, Pid}, Now, #{due := Due, keys := Keys} = Pending, Jitter) -> + Key = {Kind, Id}, + KindDue = maps:get(Kind, Due), + case maps:find(Key, Keys) of + {ok, {_At, Pid}} -> + Pending; + {ok, {At, StalePid}} -> + #{ + due => Due#{ + Kind => gb_sets:add( + {At, Key, Pid}, gb_sets:del_element({At, Key, StalePid}, KindDue) + ) + }, + keys => Keys#{Key => {At, Pid}} + }; + error -> + At = Now + Jitter(), + count(Kind), + #{ + due => Due#{Kind => gb_sets:add({At, Key, Pid}, KindDue)}, + keys => Keys#{Key => {At, Pid}} + } + end. + +-spec take_due(pending(), integer(), #{kind() => non_neg_integer()}) -> + {[target()], pending()}. +take_due(Pending, Now, Limits) -> + maps:fold( + fun(Kind, Limit, {Acc, P}) -> + {Taken, Rest} = take_due_kind(Kind, P, Now, Limit, []), + {Acc ++ Taken, Rest} + end, + {[], Pending}, + Limits + ). + +-spec take_due_kind(kind(), pending(), integer(), non_neg_integer(), [target()]) -> + {[target()], pending()}. +take_due_kind(_Kind, Pending, _Now, 0, Acc) -> + {lists:reverse(Acc), Pending}; +take_due_kind(Kind, #{due := Due, keys := Keys} = Pending, Now, Limit, Acc) -> + KindDue = maps:get(Kind, Due), + case gb_sets:is_empty(KindDue) of + true -> + {lists:reverse(Acc), Pending}; + false -> + case gb_sets:take_smallest(KindDue) of + {{At, {_Kind, Id} = Key, Pid}, Rest} when At =< Now -> + take_due_kind( + Kind, + #{due => Due#{Kind => Rest}, keys => maps:remove(Key, Keys)}, + Now, + Limit - 1, + [{Kind, Id, Pid} | Acc] + ); + _Later -> + {lists:reverse(Acc), Pending} + end + end. + +-spec heal(integer(), pid(), integer(), state()) -> {#{integer() => integer()}, pending()}. +heal(GuildId, Pid, Now, #{healed := Healed0, pending := Pending}) -> + Healed = maps:filter(fun(_Id, At) -> Now - At < ?HEAL_COOLDOWN_MS end, Healed0), + case maps:is_key(GuildId, Healed) of + true -> + {Healed, Pending}; + false -> + count(heal), + send_flip({guild, GuildId, Pid}), + {Healed#{GuildId => Now}, drop_pending({guild, GuildId}, Pending)} + end. + +-spec drop_pending(pending_key(), pending()) -> pending(). +drop_pending({Kind, _Id} = Key, #{due := Due, keys := Keys} = Pending) -> + case maps:take(Key, Keys) of + {{At, Pid}, RestKeys} -> + KindDue = gb_sets:del_element({At, Key, Pid}, maps:get(Kind, Due)), + #{due => Due#{Kind => KindDue}, keys => RestKeys}; + error -> + Pending + end. + +-spec arm_timer(state()) -> state(). +arm_timer(#{timer := Timer} = State) when is_reference(Timer) -> + State; +arm_timer(#{pending := #{due := Due}} = State) -> + case [At || KindDue <- maps:values(Due), {At, _Key, _Pid} <- smallest(KindDue)] of + [] -> + State; + Ats -> + Delay = max(?BATCH_DELAY_MS, lists:min(Ats) - now_ms()), + State#{timer := erlang:send_after(Delay, self(), flip_tick)} + end. + +-spec smallest(due()) -> [{integer(), pending_key(), pid()}]. +smallest(KindDue) -> + case gb_sets:is_empty(KindDue) of + true -> []; + false -> [gb_sets:smallest(KindDue)] + end. + +-spec send_flip(target()) -> ok. +send_flip({Kind, _GuildId, Pid}) when Kind =:= guild; Kind =:= viewers -> + Pid ! {thread_gate_flip, channel_threads_config:version()}, + ok; +send_flip({user, _UserId, Pid}) -> + Pid ! {thread_user_flip, channel_threads_config:version()}, + ok. + +-spec local_targets(fun((kind(), integer()) -> boolean())) -> [target()]. +local_targets(Changed) -> + try + ets:foldl( + fun(Entry, Acc) -> collect_local_target(Entry, Changed, Acc) end, + [], + ?REGISTRY_TABLE + ) + catch + error:badarg -> [] + end. + +-spec collect_local_target(term(), fun((kind(), integer()) -> boolean()), [target()]) -> + [target()]. +collect_local_target({{guild, GuildId}, Pid}, Changed, Acc) when + is_integer(GuildId), is_pid(Pid) +-> + local_target(guild, GuildId, Pid, Changed, Acc); +collect_local_target({{presence, UserId}, Pid}, Changed, Acc) when + is_integer(UserId), is_pid(Pid) +-> + local_target(user, UserId, Pid, Changed, Acc); +collect_local_target(_Entry, _Changed, Acc) -> + Acc. + +-spec local_target(kind(), integer(), pid(), fun((kind(), integer()) -> boolean()), [target()]) -> + [target()]. +local_target(guild, Id, Pid, Changed, Acc) when node(Pid) =:= node() -> + case Changed(guild, Id) of + true -> [{guild, Id, Pid} | Acc]; + false -> local_target(viewers, Id, Pid, Changed, Acc) + end; +local_target(Kind, Id, Pid, Changed, Acc) when node(Pid) =:= node() -> + case Changed(Kind, Id) of + true -> [{Kind, Id, Pid} | Acc]; + false -> Acc + end; +local_target(_Kind, _Id, _Pid, _Changed, Acc) -> + Acc. + +-spec jitter() -> non_neg_integer(). +jitter() -> + rand:uniform(?JITTER_WINDOW_MS) - 1. + +-spec now_ms() -> integer(). +now_ms() -> + erlang:monotonic_time(millisecond). + +-spec ensure_counters() -> ok. +ensure_counters() -> + case persistent_term:get(?FLIP_COUNTS_KEY, undefined) of + undefined -> + persistent_term:put(?FLIP_COUNTS_KEY, counters:new(length(counted()), [atomics])); + _Counters -> + ok + end. + +-spec count(counted()) -> ok. +count(Kind) -> + case persistent_term:get(?FLIP_COUNTS_KEY, undefined) of + undefined -> ok; + Counters -> counters:add(Counters, count_index(Kind), 1) + end. + +-spec counted() -> [counted()]. +counted() -> + [guild, viewers, user, heal]. + +-spec count_index(counted()) -> pos_integer(). +count_index(guild) -> 1; +count_index(user) -> 2; +count_index(heal) -> 3; +count_index(viewers) -> 4. diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_app.erl b/fluxer_gateway/src/gateway/fluxer_gateway_app.erl index e368beafb..2f7c61f59 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_app.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_app.erl @@ -20,6 +20,7 @@ init_subsystems() -> gateway_compress:init(), gateway_cluster_metrics:init(), process_registry:init(), + guild_thread_store:init(), passive_sync_registry:init(), ok. diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl index 2ead13f9b..657f89df0 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl @@ -38,7 +38,9 @@ common_children() -> child_spec(gateway_nats_pool, gateway_nats_pool), child_spec(gateway_event_pause, gateway_event_pause), child_spec(gateway_concurrency, gateway_concurrency), - child_spec(gateway_rollout_config, gateway_rollout_config) + child_spec(gateway_rollout_config, gateway_rollout_config), + child_spec(channel_threads_flip, channel_threads_flip), + child_spec(channel_threads_config, channel_threads_config) ] ++ cluster_children() ++ [ child_spec(gateway_dispatch_relay, gateway_dispatch_relay), @@ -264,6 +266,18 @@ index_of(Id, [_Other | Rest], Index) -> index_of(_Id, [], _Index) -> 1000000. +channel_threads_config_runs_on_every_role_after_the_flip_scheduler_test() -> + lists:foreach( + fun(Role) -> + Ids = init_child_ids(#{cluster_enabled => false, gateway_role => Role}), + ?assertEqual(1, count_id(channel_threads_config, Ids)), + ?assertEqual(1, count_id(channel_threads_flip, Ids)), + ?assert(owner_precedes(gateway_nats_rpc, channel_threads_config, Ids)), + ?assert(owner_precedes(channel_threads_flip, channel_threads_config, Ids)) + end, + [all, websocket, sessions, presence, guilds, calls, push] + ). + count_id(Id, Ids) -> length([Item || Item <- Ids, Item =:= Id]). diff --git a/fluxer_gateway/src/gateway/gateway_handler.erl b/fluxer_gateway/src/gateway/gateway_handler.erl index 07268e8a2..b01bec98b 100644 --- a/fluxer_gateway/src/gateway/gateway_handler.erl +++ b/fluxer_gateway/src/gateway/gateway_handler.erl @@ -141,6 +141,8 @@ websocket_info({'DOWN', Ref, process, Pid, Reason}, State) when gateway_handler_dispatch:handle_request_worker_down(Ref, Pid, Reason, State); websocket_info({gateway_request_worker_timeout, Ref, Type}, State) when is_reference(Ref) -> gateway_handler_dispatch:handle_request_worker_timeout(Ref, Type, State); +websocket_info(forum_unreads_unknown_opcode, State) -> + gateway_handler_encode:close_with_reason(unknown_opcode, <<"Unknown opcode">>, State); websocket_info({process_voice_queue}, State) -> QueueState = State#{voice_queue_timer => undefined}, NewState = gateway_handler_voice:process_queued_voice_updates(QueueState), diff --git a/fluxer_gateway/src/gateway/gateway_handler_dispatch.erl b/fluxer_gateway/src/gateway/gateway_handler_dispatch.erl index b11884f8f..4fe66ba97 100644 --- a/fluxer_gateway/src/gateway/gateway_handler_dispatch.erl +++ b/fluxer_gateway/src/gateway/gateway_handler_dispatch.erl @@ -16,6 +16,7 @@ cleanup_request_workers/1, handle_request_guild_counts/3, handle_request_channel_member_counts/3, + handle_request_forum_unreads/3, handle_lazy_request/3, validate_presence_data/1, adjust_status/1 @@ -30,7 +31,11 @@ -define(MAX_REQUEST_WORKERS, 4). -type request_worker_type() :: - request_guild_members | request_guild_counts | request_channel_member_counts | lazy_request. + request_guild_members + | request_guild_counts + | request_channel_member_counts + | request_forum_unreads + | lazy_request. -spec handle_opcode(atom(), map(), state()) -> ws_result(). handle_opcode(heartbeat, #{<<"d">> := Seq}, State) -> @@ -85,6 +90,10 @@ handle_authenticated_opcode( is_pid(Pid) -> handle_request_channel_member_counts(Data, Pid, State); +handle_authenticated_opcode(request_forum_unreads, Data, #{session_pid := Pid} = State) when + is_pid(Pid) +-> + handle_request_forum_unreads(Data, Pid, State); handle_authenticated_opcode(_, _, State) -> gateway_handler_encode:close_with_reason(unknown_opcode, <<"Unknown opcode">>, State). @@ -445,6 +454,58 @@ do_request_channel_member_counts_inner(Data, SocketPid, SessionPid) -> ok end. +-spec handle_request_forum_unreads(term(), pid(), state()) -> ws_result(). +handle_request_forum_unreads(Data, Pid, State) when is_map(Data) -> + case channel_threads_config:enabled() orelse not channel_threads_config:loaded() of + true -> + SocketPid = self(), + Fun = fun() -> do_request_forum_unreads(Data, SocketPid, Pid) end, + case start_request_worker(request_forum_unreads, Fun, State) of + {ok, _WorkerPid, WorkerState} -> + {ok, WorkerState}; + dropped -> + _ = spawn(fun() -> do_request_forum_unreads(#{}, SocketPid, Pid) end), + {ok, State} + end; + false -> + gateway_handler_encode:close_with_reason( + unknown_opcode, <<"Unknown opcode">>, State + ) + end; +handle_request_forum_unreads(_Data, _Pid, State) -> + gateway_handler_encode:close_with_reason(unknown_opcode, <<"Unknown opcode">>, State). + +-spec do_request_forum_unreads(map(), pid(), pid()) -> ok. +do_request_forum_unreads(Data, SocketPid, SessionPid) -> + case forum_unreads_session_state(SessionPid) of + {ok, SessionState} -> + try + guild_request_forum_unreads:handle_request( + Data, SocketPid, SessionState#{session_pid => SessionPid} + ) + of + ok -> ok + catch + error:_Reason -> ok; + exit:_Reason -> ok; + throw:_Reason -> ok + end; + error -> + SocketPid ! forum_unreads_unknown_opcode, + ok + end. + +-spec forum_unreads_session_state(pid()) -> {ok, map()} | error. +forum_unreads_session_state(SessionPid) -> + try gen_server:call(SessionPid, {get_state}, 5000) of + SessionState when is_map(SessionState) -> {ok, SessionState}; + _ -> error + catch + error:_Reason -> error; + exit:_Reason -> error; + throw:_Reason -> error + end. + -spec handle_lazy_request(map(), pid(), state()) -> ws_result(). handle_lazy_request(Data, Pid, State) -> SocketPid = self(), diff --git a/fluxer_gateway/src/gateway/gateway_handler_identify.erl b/fluxer_gateway/src/gateway/gateway_handler_identify.erl index 741096663..5e1a6973c 100644 --- a/fluxer_gateway/src/gateway/gateway_handler_identify.erl +++ b/fluxer_gateway/src/gateway/gateway_handler_identify.erl @@ -103,6 +103,9 @@ do_identify( GwTimings, State ) -> + ok = channel_threads_config:note_identify( + session_manager_shard_drain:thread_channels_capable(#{flags => Flags}), Properties + ), SessionId = utils:generate_session_id(), IdentifyData = build_identify_data( Token, diff --git a/fluxer_gateway/src/gateway/gateway_handler_rate_limit.erl b/fluxer_gateway/src/gateway/gateway_handler_rate_limit.erl index 6912a92f3..89ce28a74 100644 --- a/fluxer_gateway/src/gateway/gateway_handler_rate_limit.erl +++ b/fluxer_gateway/src/gateway/gateway_handler_rate_limit.erl @@ -16,6 +16,8 @@ -define(GATEWAY_RATE_LIMIT_MAX_EVENTS, 600). -define(PRESENCE_RATE_LIMIT_WINDOW_MS, 20000). -define(PRESENCE_RATE_LIMIT_MAX_EVENTS, 5). +-define(FORUM_UNREADS_RATE_LIMIT_WINDOW_MS, 5000). +-define(FORUM_UNREADS_RATE_LIMIT_MAX_EVENTS, 5). -define(SHARED_IP_RATE_TABLE, gateway_shared_ip_rate). -define(SHARED_USER_RATE_TABLE, gateway_shared_user_rate). @@ -99,6 +101,14 @@ check_opcode_rate_limit(State, presence_update, Now) -> ?PRESENCE_RATE_LIMIT_WINDOW_MS, ?PRESENCE_RATE_LIMIT_MAX_EVENTS ); +check_opcode_rate_limit(State, request_forum_unreads, Now) -> + check_named_opcode_rate_limit( + State, + request_forum_unreads, + Now, + ?FORUM_UNREADS_RATE_LIMIT_WINDOW_MS, + ?FORUM_UNREADS_RATE_LIMIT_MAX_EVENTS + ); check_opcode_rate_limit(State, _Op, _Now) -> {ok, State}. @@ -300,6 +310,29 @@ with_rate_limits_enabled(Fun) -> restore_env("FLUXER_DISABLE_RATE_LIMITS", OldValue) end. +forum_unreads_allows_five_requests_per_five_seconds_test() -> + with_rate_limits_enabled(fun() -> + Now = erlang:system_time(millisecond), + Recent = #{request_forum_unreads => lists:duplicate(4, Now)}, + State = eqwalizer:dynamic_cast(#{ + rate_limit_state => #{events => [], op_events => Recent} + }), + {ok, Next} = check_rate_limit_limited(State, request_forum_unreads), + ?assertMatch( + {opcode_rate_limited, _}, check_rate_limit_limited(Next, request_forum_unreads) + ), + Old = #{request_forum_unreads => lists:duplicate(5, Now - 5000)}, + Expired = eqwalizer:dynamic_cast(#{ + rate_limit_state => #{events => [], op_events => Old} + }), + ?assertMatch({ok, _}, check_rate_limit_limited(Expired, request_forum_unreads)), + Full = #{request_guild_counts => lists:duplicate(5, Now)}, + Other = eqwalizer:dynamic_cast(#{ + rate_limit_state => #{events => [], op_events => Full} + }), + ?assertMatch({ok, _}, check_rate_limit_limited(Other, request_guild_counts)) + end). + shared_ip_rate_blocks_over_limit_test() -> with_rate_limits_enabled(fun() -> IP = <<"198.51.100.10">>, diff --git a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl b/fluxer_gateway/src/gateway/gateway_nats_rpc.erl index 3e192df8a..2af8b1b70 100644 --- a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl +++ b/fluxer_gateway/src/gateway/gateway_nats_rpc.erl @@ -388,6 +388,7 @@ handle_msg( -spec subject_owner(binary()) -> atom() | undefined. subject_owner(<<"config.gateway.rollout">>) -> gateway_rollout_config; +subject_owner(<<"config.channel.threads">>) -> channel_threads_config; subject_owner(_Subject) -> undefined. -spec notify_resubscribed(binary()) -> ok. diff --git a/fluxer_gateway/src/gateway/gateway_rpc_guild.erl b/fluxer_gateway/src/gateway/gateway_rpc_guild.erl index 57ef1fe6e..b68e9ee60 100644 --- a/fluxer_gateway/src/gateway/gateway_rpc_guild.erl +++ b/fluxer_gateway/src/gateway/gateway_rpc_guild.erl @@ -36,6 +36,7 @@ route(M, P) -> -define(HANDLER_MAP, #{ <<"guild.dispatch">> => fun gateway_rpc_guild_lifecycle:handle/2, + <<"guild.dispatch_many">> => fun gateway_rpc_guild_lifecycle:handle/2, <<"guild.get_data">> => fun gateway_rpc_guild_lifecycle:handle/2, <<"guild.get_auth_context">> => fun gateway_rpc_guild_lifecycle:handle/2, <<"guild.start">> => fun gateway_rpc_guild_lifecycle:handle/2, diff --git a/fluxer_gateway/src/gateway/gateway_rpc_guild_channels.erl b/fluxer_gateway/src/gateway/gateway_rpc_guild_channels.erl index 9a858604f..596338b96 100644 --- a/fluxer_gateway/src/gateway/gateway_rpc_guild_channels.erl +++ b/fluxer_gateway/src/gateway/gateway_rpc_guild_channels.erl @@ -209,6 +209,10 @@ channel_mention_from_snapshot(CId, ChIndex, GuildId, BasePerm) -> end. -spec viewable_channel_mention(map(), integer(), integer()) -> {true, map()} | false. +viewable_channel_mention(#{<<"type">> := Type}, _GuildId, _BasePerm) when + Type =:= 15; Type =:= 16 +-> + false; viewable_channel_mention(Channel, GuildId, BasePerm) -> case everyone_can_view(Channel, GuildId, BasePerm) of true -> build_channel_mention(Channel); diff --git a/fluxer_gateway/src/gateway/gateway_rpc_guild_lifecycle.erl b/fluxer_gateway/src/gateway/gateway_rpc_guild_lifecycle.erl index fd7cd4c77..79aa5cf6a 100644 --- a/fluxer_gateway/src/gateway/gateway_rpc_guild_lifecycle.erl +++ b/fluxer_gateway/src/gateway/gateway_rpc_guild_lifecycle.erl @@ -14,6 +14,7 @@ -spec handle(binary(), map()) -> term(). handle(<<"guild.dispatch">>, P) -> handle_dispatch(P); +handle(<<"guild.dispatch_many">>, P) -> handle_dispatch_many(P); handle(<<"guild.get_data">>, P) -> handle_get_data(P); handle(<<"guild.get_auth_context">>, P) -> handle_get_auth_context(P); handle(<<"guild.start">>, P) -> handle_start(P); @@ -36,6 +37,25 @@ handle_dispatch(#{<<"guild_id">> := GuildIdBin, <<"event">> := Event, <<"data">> true end). +-spec handle_dispatch_many(map()) -> term(). +handle_dispatch_many(#{<<"guild_id">> := GuildIdBin, <<"events">> := Events}) when + is_list(Events) +-> + GuildId = validation:snowflake_or_throw(<<"guild_id">>, GuildIdBin), + Requests = [dispatch_request(Event) || Event <- Events], + gateway_rpc_guild_infra:with_guild_unchecked(GuildId, fun(Pid) -> + gen_server:cast(Pid, {dispatch_many, Requests}), + true + end); +handle_dispatch_many(_Params) -> + gateway_rpc_error:raise(validation_invalid_params). + +-spec dispatch_request(term()) -> map(). +dispatch_request(#{<<"event">> := Event, <<"data">> := Data}) when is_binary(Event) -> + #{event => constants:dispatch_event_atom(Event), data => Data}; +dispatch_request(_Event) -> + gateway_rpc_error:raise(validation_invalid_params). + -spec handle_get_data(map()) -> term(). handle_get_data(#{<<"guild_id">> := GuildIdBin, <<"user_id">> := UserIdBin}) -> GuildId = validation:snowflake_or_throw(<<"guild_id">>, GuildIdBin), diff --git a/fluxer_gateway/src/gateway/gateway_rpc_presence.erl b/fluxer_gateway/src/gateway/gateway_rpc_presence.erl index 966435cae..17837a4d7 100644 --- a/fluxer_gateway/src/gateway/gateway_rpc_presence.erl +++ b/fluxer_gateway/src/gateway/gateway_rpc_presence.erl @@ -367,6 +367,13 @@ resolve_owner_node_returns_unavailable_when_invalid_owner_test() -> resolve_owner_node(123, fun(_UserId) -> {bad_owner} end) ). +dispatch_event_atom_accepts_thread_events_test() -> + ?assertEqual( + thread_member_update, dispatch_event_atom_or_error(<<"THREAD_MEMBER_UPDATE">>) + ), + ?assertEqual(thread_list_sync, dispatch_event_atom_or_error(<<"THREAD_LIST_SYNC">>)), + ?assertEqual(forum_unreads, dispatch_event_atom_or_error(<<"FORUM_UNREADS">>)). + presence_manager_server_ref_local_test() -> ?assertEqual(presence_manager, presence_manager_server_ref(node())). diff --git a/fluxer_gateway/src/gateway/metrics_handler.erl b/fluxer_gateway/src/gateway/metrics_handler.erl index d954889b5..0e54149c1 100644 --- a/fluxer_gateway/src/gateway/metrics_handler.erl +++ b/fluxer_gateway/src/gateway/metrics_handler.erl @@ -44,6 +44,7 @@ render_metrics() -> render_gateway_gauges(), render_cluster_counters(), render_process_counts(), + render_channel_threads_stats(), render_push_outbox_stats(safe_apply_map(fun push_outbox:stats/0)), render_vm_metrics() ]. @@ -273,6 +274,134 @@ count_registry_prefix(Prefix) -> error:badarg -> 0 end. +-spec render_channel_threads_stats() -> iolist(). +render_channel_threads_stats() -> + UpdateCounts = safe_apply_map(fun channel_threads_config:update_counts/0), + FlipCounts = safe_apply_map(fun channel_threads_flip:flip_counts/0), + [ + format_metric( + <<"fluxer_gateway_channel_threads_config_version">>, + <<"gauge">>, + <<"Channel threads config version in effect">>, + integer_to_binary(safe_apply_int(fun channel_threads_config:version/0)) + ), + format_labeled_series( + <<"fluxer_gateway_channel_threads_config_updates_total">>, + <<"counter">>, + <<"Channel threads config reads by outcome">>, + [ + { + <<"result=\"", (atom_to_binary(Result))/binary, "\"">>, + gate_counter(Result, UpdateCounts) + } + || Result <- [updated, unchanged, stale, rejected] + ] + ), + format_labeled_series( + <<"fluxer_gateway_channel_threads_flips_total">>, + <<"counter">>, + <<"Channel threads gate flips scheduled by kind">>, + [ + { + <<"kind=\"", (atom_to_binary(Kind))/binary, "\"">>, + gate_counter(Kind, FlipCounts) + } + || Kind <- [guild, viewers, user, heal] + ] + ), + render_identify_counts(safe_apply_map(fun channel_threads_config:identify_counts/0)), + render_thread_stats() + ]. + +-spec render_thread_stats() -> iolist(). +render_thread_stats() -> + DispatchCounts = safe_apply_map(fun guild_thread_dispatch:dispatch_counts/0), + SyncCounts = safe_apply_map(fun guild_thread_subscriptions:list_sync_counts/0), + FetchCounts = safe_apply_map(fun guild_thread_load:fetch_counts/0), + [ + format_metric( + <<"fluxer_gateway_thread_store_threads">>, + <<"gauge">>, + <<"Active threads held in guild thread stores on this node">>, + integer_to_binary(safe_apply_int(fun guild_thread_store:total_threads/0)) + ), + format_labeled_series( + <<"fluxer_gateway_thread_dispatch_total">>, + <<"counter">>, + <<"Thread-gated events delivered to sessions by event and variant">>, + [ + { + <<"event=\"", (atom_to_binary(Event))/binary, "\",variant=\"", + (atom_to_binary(Variant))/binary, "\"">>, + integer_to_binary(Count) + } + || {{Event, Variant}, Count} <- lists:sort(maps:to_list(DispatchCounts)), + is_atom(Event), + is_atom(Variant), + is_integer(Count) + ] + ), + format_labeled_series( + <<"fluxer_gateway_thread_list_sync_total">>, + <<"counter">>, + <<"THREAD_LIST_SYNC sends by reason">>, + [ + { + <<"reason=\"", (atom_to_binary(Reason))/binary, "\"">>, + gate_counter(Reason, SyncCounts) + } + || Reason <- [full, access] + ] + ), + format_labeled_series( + <<"fluxer_gateway_thread_collection_fetch_total">>, + <<"counter">>, + <<"Thread collection fetches by result">>, + [ + { + <<"result=\"", (atom_to_binary(Result))/binary, "\"">>, + gate_counter(Result, FetchCounts) + } + || Result <- [ok, error, refetch] + ] + ), + format_metric( + <<"fluxer_gateway_thread_permission_unknown_parent_total">>, + <<"counter">>, + <<"Thread permission checks whose parent channel was unknown">>, + integer_to_binary( + safe_apply_int(fun guild_thread_permissions:unknown_parent_count/0) + ) + ), + format_metric( + <<"fluxer_gateway_channel_threads_flip_resend_total">>, + <<"counter">>, + <<"GUILD_CREATE resends after a channel threads gate flip">>, + integer_to_binary(safe_apply_int(fun guild_thread_flip:resend_count/0)) + ) + ]. + +-spec render_identify_counts(map()) -> iolist(). +render_identify_counts(Counts) when map_size(Counts) =:= 0 -> + []; +render_identify_counts(Counts) -> + format_labeled_series( + <<"fluxer_gateway_identify_total">>, + <<"counter">>, + <<"Identify attempts by channel threads capability and OS">>, + [ + {identify_labels(Key), integer_to_binary(Count)} + || {Key, Count} <- lists:sort(maps:to_list(Counts)), is_integer(Count) + ] + ). + +-spec identify_labels(term()) -> binary(). +identify_labels({Capable, Os}) when is_boolean(Capable), is_binary(Os) -> + <<"channel_threads_capable=\"", (atom_to_binary(Capable))/binary, "\",os=\"", Os/binary, + "\"">>; +identify_labels(_Key) -> + <<"channel_threads_capable=\"false\",os=\"other\"">>. + -spec render_push_outbox_stats(map()) -> iolist(). render_push_outbox_stats(Stats) when map_size(Stats) =:= 0 -> []; diff --git a/fluxer_gateway/src/guild/guild.erl b/fluxer_gateway/src/guild/guild.erl index 387b02d4c..dedd2d892 100644 --- a/fluxer_gateway/src/guild/guild.erl +++ b/fluxer_gateway/src/guild/guild.erl @@ -124,6 +124,7 @@ query_call_handler(get_counts) -> query; query_call_handler(get_user_counts) -> query; query_call_handler(get_viewer_counts) -> query; query_call_handler(get_channel_member_counts) -> query; +query_call_handler(get_forum_unread_scope) -> query; query_call_handler(get_large_guild_metadata) -> query; query_call_handler(get_users_to_mention_by_roles) -> query; query_call_handler(get_users_to_mention_by_user_ids) -> query; @@ -184,6 +185,15 @@ subscription_call_handler(_) -> undefined. -spec handle_cast_internal(term(), guild_state()) -> cast_reply(). handle_cast_internal({dispatch, Request}, State) -> handle_dispatch_cast(Request, State); +handle_cast_internal({dispatch_many, Requests}, State) when is_list(Requests) -> + {noreply, + lists:foldl( + fun(#{event := Event, data := EventData}, Acc) -> + dispatch_event(Event, EventData, Acc) + end, + State, + Requests + )}; handle_cast_internal( {session_connect_async, #{guild_id := GuildId, attempt := Attempt, request := Request} = Msg}, @@ -214,6 +224,12 @@ handle_cast_internal({send_guild_sync, SessionId}, State) -> handle_send_guild_sync_cast(SessionId, State); handle_cast_internal({send_members_chunk, SessionId, ChunkData}, State) -> handle_send_members_chunk_cast(SessionId, ChunkData, State); +handle_cast_internal({thread_user_flip, SessionId}, State) when is_binary(SessionId) -> + {noreply, guild_thread_flip:user_flip(SessionId, State)}; +handle_cast_internal({thread_subscriptions, SessionId, Subscriptions}, State) when + is_binary(SessionId), is_map(Subscriptions) +-> + {noreply, guild_thread_subscriptions:update(SessionId, Subscriptions, State)}; handle_cast_internal({patch_everyone_perms, Bit}, State) when is_integer(Bit), Bit > 0 -> {noreply, guild_maintenance:apply_everyone_perm_bit(Bit, State)}; handle_cast_internal(Msg, State) when is_tuple(Msg) -> @@ -279,6 +295,29 @@ handle_info_internal({check_auto_stop_empty, Token}, State) -> handle_auto_stop_info(Token, State); handle_info_internal(check_auto_stop_empty, State) -> {noreply, State}; +handle_info_internal({thread_list_sync_tick, GuildIdToken}, State) -> + {noreply, guild_thread_subscriptions:handle_tick(GuildIdToken, State)}; +handle_info_internal(thread_list_sync_drain, State) -> + {noreply, guild_thread_subscriptions:handle_drain(State)}; +handle_info_internal(thread_member_list_flush, State) -> + {noreply, guild_thread_subscriptions:handle_list_flush(State)}; +handle_info_internal(thread_load_retry, State) -> + {noreply, guild_thread_load:handle_retry(State)}; +handle_info_internal({thread_load_result, Ref, Result}, State) when is_reference(Ref) -> + {noreply, guild_thread_load:handle_load_result(Ref, Result, State)}; +handle_info_internal({thread_gate_flip, Version}, State) when is_integer(Version) -> + {noreply, guild_thread_flip:start(Version, State)}; +handle_info_internal({thread_flip_result, Ref, Result}, State) when is_reference(Ref) -> + {noreply, guild_thread_flip:handle_result(Ref, Result, State)}; +handle_info_internal({thread_flip_resend, SessionId}, State) when is_binary(SessionId) -> + {noreply, guild_thread_flip:resend(SessionId, State)}; +handle_info_internal({thread_store_retire, Tab}, State) -> + ok = + case maps:get(thread_store, maps:get(data, State, #{}), undefined) of + Tab -> ok; + _ -> guild_thread_store:destroy(Tab) + end, + {noreply, State}; handle_info_internal({timeout, TimerRef, member_list_sync_item_cache_rotate}, State) when is_reference(TimerRef) -> @@ -534,6 +573,7 @@ safe_cleanup(Fun, Label) -> cleanup_per_guild_ets(State) -> Data = maps:get(data, State, #{}), safe_delete_ets(maps:get(members_ets, Data, undefined)), + safe_delete_ets(maps:get(thread_store, Data, undefined)), safe_delete_ets(maps:get(member_presence, State, undefined)), safe_delete_ets(maps:get(viewable_channels_cache, State, undefined)), ok. @@ -759,7 +799,9 @@ voice_guild_data_keys() -> <<"channels">>, <<"channel_index">>, overwrite_perms_cache, - members_ets + members_ets, + thread_gate, + thread_store ]. -spec voice_members_table_unavailable(guild_state()) -> map(). @@ -799,6 +841,23 @@ handle_non_voice_exit_broadcaster_keeps_guild_alive_test() -> State = #{id => 42, broadcaster_pid => BPid}, ?assertEqual({noreply, #{id => 42}}, handle_non_voice_exit(BPid, killed, State)). +unknown_binary_event_dispatch_cast_keeps_guild_state_test() -> + State = #{ + id => 1, + data => #{<<"guild">> => #{}}, + sessions => #{}, + dm_partners => #{ + <<"s1">> => #{user_id => 2, pid => self(), partners => #{}, eligible => #{}} + } + }, + Event = <<"FUTURE_EVENT_THE_GATEWAY_HAS_NEVER_SEEN">>, + ?assertEqual( + {noreply, State}, handle_cast({dispatch, #{event => Event, data => #{}}}, State) + ), + ?assertEqual( + {noreply, State}, handle_cast({dispatch, #{event => Event, data => <<"{}">>}}, State) + ). + handle_non_voice_exit_other_linked_stops_guild_test() -> BPid = list_to_pid("<0.250.0>"), OtherPid = list_to_pid("<0.251.0>"), diff --git a/fluxer_gateway/src/guild/guild_availability.erl b/fluxer_gateway/src/guild/guild_availability.erl index 91f6be75e..fe368985c 100644 --- a/fluxer_gateway/src/guild/guild_availability.erl +++ b/fluxer_gateway/src/guild/guild_availability.erl @@ -14,7 +14,9 @@ is_guild_unavailable_for_user_from_cache/2, update_unavailability_cache_for_state/1, schedule_availability_recheck/1, - handle_availability_recheck/1 + handle_availability_recheck/1, + send_guild_create_to_session/4, + send_guild_create_to_session/5 ]). -type guild_state() :: map(). @@ -118,14 +120,22 @@ send_guild_create_to_sessions(State, GuildId) -> -spec send_guild_create_to_session(map(), guild_id(), guild_state(), list()) -> ok. send_guild_create_to_session(SessionData, GuildId, State, BulkPresences) -> + send_guild_create_to_session(SessionData, GuildId, State, BulkPresences, #{}). + +-spec send_guild_create_to_session(map(), guild_id(), guild_state(), list(), map()) -> ok. +send_guild_create_to_session(SessionData, GuildId, State, BulkPresences, Extra) -> case maps:get(pending_connect, SessionData, false) of true -> ok; false -> UserId = maps:get(user_id, SessionData), Pid = maps:get(pid, SessionData), - GuildState = guild_data:get_guild_state(UserId, State), - gateway_dispatch_relay:dispatch(Pid, guild_create, GuildState, GuildId), + GuildState = guild_data:get_guild_state( + UserId, State, guild_thread_gate:state_opts(SessionData) + ), + gateway_dispatch_relay:dispatch( + Pid, guild_create, maps:merge(GuildState, Extra), GuildId + ), presence_utils:send_presence_bulk(Pid, GuildId, UserId, BulkPresences) end. diff --git a/fluxer_gateway/src/guild/guild_connect_async.erl b/fluxer_gateway/src/guild/guild_connect_async.erl index fdd471b46..9d5c25311 100644 --- a/fluxer_gateway/src/guild/guild_connect_async.erl +++ b/fluxer_gateway/src/guild/guild_connect_async.erl @@ -394,7 +394,9 @@ upsert_pending_session(S, U, P, Request, State) -> active_guilds => maps:get(active_guilds, Request, sets:new()), bot => maps:get(bot, Request, false), is_staff => maps:get(is_staff, Request, false), - pending_connect => true + pending_connect => true, + thread_capable => maps:get(thread_channels_capable, Request, false) =:= true, + thread_viewer => false }, guild_sessions_connect:put_session_ref(S, MRef, State#{ sessions => Sessions0#{S => Entry} @@ -502,6 +504,9 @@ compute_session_connect(GuildId, #{user_id := UserId} = Request, State) when case guild_availability:is_guild_unavailable_for_user(UserId, State) of true -> #{ + thread_viewer => maps:get( + thread_viewer, guild_thread_gate:session_fields(Request, UserId, State) + ), unavailable_response => #{ <<"id">> => integer_to_binary(GuildId), <<"unavailable">> => true @@ -519,9 +524,13 @@ compute_available_connect(GuildId, UserId, Request, State) -> undefined -> #{not_member => true}; _Member -> - GS = guild_data:get_guild_state(UserId, State), + ThreadFields = guild_thread_gate:session_fields(Request, UserId, State), + GS = guild_data:get_guild_state( + UserId, State, guild_thread_gate:state_opts(maps:merge(Request, ThreadFields)) + ), #{ guild_state => GS, + thread_viewer => maps:get(thread_viewer, ThreadFields), initial_last_message_ids => guild_sessions:build_initial_last_message_ids(GS), initial_channel_versions => build_channel_versions(GS), viewable_channels => build_viewable_map( @@ -586,7 +595,9 @@ upsert_session_valid(SessionId, SessionPid, UserId, Request, Computed, State, Jo store_passive_state(SessionId, GuildId, Computed), FinalSD = maybe_mark_synced(GuildId, Computed, SessionData), Sessions = merge_session(SessionId, FinalSD, Existing1, Sessions0), - State1 = reindex_session_ref(SessionId, Existing, MRef, State#{sessions => Sessions}), + State1 = guild_thread_flip:user_flip( + SessionId, reindex_session_ref(SessionId, Existing, MRef, State#{sessions => Sessions}) + ), State2 = update_connected_tracking(UserId, Existing, State1), update_presence_subscription( UserId, Existing, State2, note_joined_user(UserId, State2, Joined) @@ -607,6 +618,8 @@ note_joined_user(UserId, State, {Joined, Fresh}) -> -spec build_session_data(session_id(), integer(), pid(), reference(), map(), map()) -> map(). build_session_data(SessionId, UserId, SessionPid, MRef, Request, Computed) -> #{ + thread_capable => maps:get(thread_channels_capable, Request, false) =:= true, + thread_viewer => maps:get(thread_viewer, Computed, false), session_id => SessionId, user_id => UserId, pid => SessionPid, @@ -838,4 +851,59 @@ enqueue_creates_pending_session_entry_test() -> ?assertNot(maps:is_key(owns_connected_tracking, Entry)), ?assertEqual(UserId, maps:get(user_id, Entry)). +unavailable_connect_of_a_thread_viewer_sends_no_guild_create_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, included_users => #{<<"10">> => true} + }), + try + SessionId = <<"s-unavailable">>, + Request = (pending_connect_request(SessionId, 10))#{thread_channels_capable => true}, + Data = #{ + <<"guild">> => #{ + <<"id">> => <<"42">>, <<"features">> => [<<"UNAVAILABLE_FOR_EVERYONE">>] + }, + thread_gate => #{active => true, version => 1} + }, + Snapshot = #{id => 42, data => Data, sessions => #{}}, + Item = #{guild_id => 42, attempt => 1, request => Request}, + {{ok_unavailable, _}, Computed} = compute_connect_result(Snapshot, Item, Request), + ?assertEqual(true, maps:get(thread_viewer, Computed)), + State0 = (pending_connect_state(#{}))#{ + data => Data, + session_connect_pending => #{SessionId => 1}, + session_connect_inflight => 1, + user_session_counts => #{}, + connected_user_ids => sets:new(), + presence_subscriptions => #{}, + member_presence => #{} + }, + State1 = finalize_session_connect_async( + SessionId, 1, {ok_unavailable, #{}}, maps:merge(Item, Computed), State0 + ), + Session = maps:get(SessionId, maps:get(sessions, State1)), + ?assert(maps:get(thread_viewer, Session)), + _ = guild_thread_flip:resend(SessionId, State1), + ?assertEqual([], unavailable_test_messages()), + receive + {guild_connect_result, 42, 1, _} -> ok + after 0 -> ok + end + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +unavailable_test_messages() -> + receive + {thread_flip_resend, _} = Resend -> + [Resend | unavailable_test_messages()]; + {'$gen_cast', {dispatch, guild_create, _}} = Create -> + [Create | unavailable_test_messages()] + after 0 -> [] + end. + -endif. diff --git a/fluxer_gateway/src/guild/guild_data.erl b/fluxer_gateway/src/guild/guild_data.erl index 0f1b97d62..5dae95c14 100644 --- a/fluxer_gateway/src/guild/guild_data.erl +++ b/fluxer_gateway/src/guild/guild_data.erl @@ -12,7 +12,7 @@ -export([search_guild_members/2]). -export([get_vanity_url_channel/1]). -export([get_first_viewable_text_channel/1]). --export([get_guild_state/2]). +-export([get_guild_state/2, get_guild_state/3]). -export([build_connect_snapshot/2]). -export([fetch_latest_voice_states/1]). -export([find_everyone_viewable_text_channel/2]). @@ -86,7 +86,22 @@ build_auth_guild(Data, State) -> find_channel(null, _Data) -> null; find_channel(ChannelId, Data) -> - maps:get(ChannelId, guild_data_index:channel_index(Data), null). + Index = guild_data_index:channel_index(Data), + case maps:get(ChannelId, Index, null) of + null -> find_thread_parent_channel(ChannelId, Index, Data); + Channel -> Channel + end. + +-spec find_thread_parent_channel(term(), map(), map()) -> map() | null. +find_thread_parent_channel(ChannelId, Index, Data) when is_integer(ChannelId) -> + case guild_thread_gate:thread(ChannelId, Data) of + undefined -> + null; + Thread -> + maps:get(guild_thread_permissions:parent_id(Thread), Index, null) + end; +find_thread_parent_channel(_ChannelId, _Index, _Data) -> + null. -spec get_guild_member(map(), guild_state()) -> guild_reply(map()). get_guild_member(Request, State) -> @@ -130,6 +145,33 @@ find_everyone_viewable_text_channel(Channels, State) -> -spec get_guild_state(user_id(), guild_state()) -> map(). get_guild_state(UserId, State) -> + get_guild_state(UserId, State, #{}). + +-spec get_guild_state(user_id(), guild_state(), map()) -> map(). +get_guild_state(UserId, State, Opts) -> + GuildState = base_guild_state(UserId, State), + case guild_thread_gate:needs_variant(State) of + false -> GuildState; + true -> apply_thread_view(UserId, Opts, State, GuildState) + end. + +-spec apply_thread_view(user_id(), map(), guild_state(), map()) -> map(). +apply_thread_view(UserId, #{thread_viewer := true} = Opts, State, GuildState) -> + Member = guild_data_members:find_member_by_user_id(UserId, State), + GuildState#{<<"threads">> => guild_thread_view:guild_threads(UserId, Opts, Member, State)}; +apply_thread_view(UserId, _Opts, State, GuildState) -> + Member = guild_data_members:find_member_by_user_id(UserId, State), + Channels = guild_thread_view:hide_for_non_viewer( + UserId, Member, State, maps:get(<<"channels">>, GuildState, []) + ), + Roles = [ + guild_thread_gate:mask_role(R) + || R <- maps:get(<<"roles">>, GuildState, []), is_map(R) + ], + GuildState#{<<"channels">> => Channels, <<"roles">> => Roles}. + +-spec base_guild_state(user_id(), guild_state()) -> map(). +base_guild_state(UserId, State) -> Data = guild_data_index:ensure_data_map(State), GuildId = guild_id(State), AllChannels = guild_data_channels:channels_from_data(Data), diff --git a/fluxer_gateway/src/guild/guild_data_channels_order.erl b/fluxer_gateway/src/guild/guild_data_channels_order.erl index 5a5240119..8bb6864e0 100644 --- a/fluxer_gateway/src/guild/guild_data_channels_order.erl +++ b/fluxer_gateway/src/guild/guild_data_channels_order.erl @@ -175,7 +175,8 @@ channel_is_category(Channel) -> -spec channel_is_layout_text(map()) -> boolean(). channel_is_layout_text(Channel) -> ChannelType = channel_type(Channel), - ChannelType =:= 0 orelse ChannelType =:= 5 orelse ChannelType =:= 998. + ChannelType =:= 0 orelse ChannelType =:= 5 orelse ChannelType =:= 998 orelse + ChannelType =:= 15 orelse ChannelType =:= 16. -spec channel_is_layout_voice(map()) -> boolean(). channel_is_layout_voice(Channel) -> diff --git a/fluxer_gateway/src/guild/guild_data_index.erl b/fluxer_gateway/src/guild/guild_data_index.erl index 0a0c7cda0..4a18fad5f 100644 --- a/fluxer_gateway/src/guild/guild_data_index.erl +++ b/fluxer_gateway/src/guild/guild_data_index.erl @@ -53,7 +53,8 @@ normalize_map(Data) -> ), Roles = guild_data_index_roles:role_list(Data0), Channels = guild_data_index_channels:channel_list(Data0), - Data0#{ + Data1 = guild_data_index_channels:put_forum_categories(Channels, Data0), + Data1#{ <<"members">> => MemberMap, members_normalized => MemberMap, member_list_revision => make_ref(), diff --git a/fluxer_gateway/src/guild/guild_data_index_channels.erl b/fluxer_gateway/src/guild/guild_data_index_channels.erl index c19ad19fd..932273194 100644 --- a/fluxer_gateway/src/guild/guild_data_index_channels.erl +++ b/fluxer_gateway/src/guild/guild_data_index_channels.erl @@ -7,6 +7,7 @@ channel_list/1, channel_index/1, put_channels/2, + put_forum_categories/2, build_overwrite_perms_cache/1 ]). @@ -46,14 +47,58 @@ put_channels(Channels, Data) when is_map(Data) -> fun normalize_channel_item/1, guild_data_index:ensure_list(Channels) ), - Data#{ + put_forum_categories(ChannelList, Data#{ <<"channels">> => ChannelList, <<"channel_index">> => guild_data_index:build_id_index(ChannelList), overwrite_perms_cache => build_overwrite_perms_cache(ChannelList) - }; + }); put_channels(_, Data) -> Data. +-spec put_forum_categories([channel()], map()) -> map(). +put_forum_categories(Channels, Data) -> + case forum_categories(Channels) of + Categories when map_size(Categories) =:= 0 -> + maps:remove(thread_forum_categories, Data); + Categories -> + Data#{thread_forum_categories => Categories} + end. + +-spec forum_categories([channel()]) -> #{snowflake_id() => [snowflake_id()]}. +forum_categories(Channels) -> + case [parent_of(C) || #{<<"type">> := Type} = C <- Channels, is_forum_type(Type)] of + [] -> + #{}; + Parents -> + Categories = maps:from_list([{P, []} || P <- Parents, is_integer(P)]), + lists:foldl(fun add_plain_child/2, Categories, Channels) + end. + +-spec add_plain_child(channel(), #{snowflake_id() => [snowflake_id()]}) -> + #{snowflake_id() => [snowflake_id()]}. +add_plain_child(Channel, Categories) -> + case + { + parent_of(Channel), + is_forum_type(maps:get(<<"type">>, Channel, undefined)), + snowflake_id:parse_maybe(maps:get(<<"id">>, Channel, undefined)) + } + of + {Parent, false, ChildId} when is_map_key(Parent, Categories), is_integer(ChildId) -> + Categories#{Parent := [ChildId | maps:get(Parent, Categories)]}; + _ -> + Categories + end. + +-spec parent_of(channel()) -> snowflake_id() | undefined. +parent_of(Channel) -> + snowflake_id:parse_maybe(maps:get(<<"parent_id">>, Channel, undefined)). + +-spec is_forum_type(term()) -> boolean(). +is_forum_type(15) -> true; +is_forum_type(16) -> true; +is_forum_type(_) -> false. + -spec build_overwrite_perms_cache([channel()]) -> #{integer() => [{integer(), integer(), integer()}]}. build_overwrite_perms_cache(Channels) -> @@ -186,6 +231,20 @@ channel_index_from_list_test() -> ?assertEqual(2, map_size(Index)), ?assertEqual(<<"general">>, maps:get(<<"name">>, maps:get(300, Index))). +forum_categories_list_plain_children_of_forum_parents_test() -> + Channels = [ + #{<<"id">> => 1, <<"type">> => 4}, + #{<<"id">> => 2, <<"type">> => 4}, + #{<<"id">> => 10, <<"type">> => 15, <<"parent_id">> => 1}, + #{<<"id">> => 11, <<"type">> => 0, <<"parent_id">> => 1}, + #{<<"id">> => 12, <<"type">> => 0, <<"parent_id">> => 2} + ], + ?assertEqual(#{1 => [11]}, forum_categories(Channels)), + ?assertEqual(#{}, forum_categories(tl(tl(tl(Channels))))), + Data = put_channels(Channels, #{}), + ?assertEqual(#{1 => [11]}, maps:get(thread_forum_categories, Data)), + ?assertNot(maps:is_key(thread_forum_categories, put_channels([], Data))). + put_channels_updates_list_and_index_test() -> Data = #{<<"channels">> => []}, NewChannels = [ diff --git a/fluxer_gateway/src/guild/guild_data_normalize.erl b/fluxer_gateway/src/guild/guild_data_normalize.erl index 73a8a93f8..3c34303f2 100644 --- a/fluxer_gateway/src/guild/guild_data_normalize.erl +++ b/fluxer_gateway/src/guild/guild_data_normalize.erl @@ -7,7 +7,9 @@ guild_data/1, member/1, role/1, - channel/1 + channel/1, + thread/1, + thread_member/1 ]). -spec guild_data(term()) -> term(). @@ -51,16 +53,68 @@ channel(Channel) when is_map(Channel) -> Channel5 = normalize_nick_map(Channel4), Channel6 = normalize_overwrite_list(Channel5), Channel7 = guild_data_normalize_schema:normalize_int_fields( - [<<"type">>, <<"position">>, <<"content_warning_level">>, <<"rate_limit_per_user">>], + [ + <<"type">>, + <<"position">>, + <<"content_warning_level">>, + <<"rate_limit_per_user">>, + <<"flags">>, + <<"default_thread_rate_limit_per_user">>, + <<"default_forum_layout">> + ], Channel6 ), guild_data_normalize_schema:normalize_nullable_int_fields( - [<<"bitrate">>, <<"user_limit">>, <<"voice_connection_limit">>], + [ + <<"bitrate">>, + <<"user_limit">>, + <<"voice_connection_limit">>, + <<"default_auto_archive_duration">>, + <<"default_sort_order">> + ], Channel7 ); channel(Channel) -> Channel. +-spec thread(term()) -> term(). +thread(Thread) when is_map(Thread) -> + Thread1 = channel( + maps:without( + [ + <<"member">>, + <<"newly_created">>, + <<"position">>, + <<"permission_overwrites">>, + <<"nsfw">>, + <<"topic">> + ], + Thread + ) + ), + Thread2 = normalize_snowflake_list_field(<<"applied_tags">>, Thread1), + Thread3 = normalize_snowflake_list_field(<<"member_ids_preview">>, Thread2), + Thread4 = guild_data_normalize_schema:normalize_int_fields( + [<<"message_count">>, <<"total_message_sent">>, <<"member_count">>], + Thread3 + ), + maps:filter(fun(Key, _Value) -> not internal_key(Key) end, Thread4); +thread(Thread) -> + Thread. + +-spec thread_member(term()) -> term(). +thread_member(Member) when is_map(Member) -> + Member1 = normalize_required_snowflake_fields([<<"id">>, <<"user_id">>], Member), + Member2 = guild_data_normalize_schema:normalize_int_fields([<<"flags">>], Member1), + maps:without([<<"member">>, <<"presence">>, <<"guild_id">>], Member2); +thread_member(Member) -> + Member. + +-spec internal_key(term()) -> boolean(). +internal_key(<<"_fluxer_", _/binary>>) -> true; +internal_key(<<"__thread_", _/binary>>) -> true; +internal_key(_) -> false. + -spec guild(map()) -> map(). guild(Guild) -> Guild1 = normalize_required_snowflake_fields([<<"id">>, <<"owner_id">>], Guild), diff --git a/fluxer_gateway/src/guild/guild_data_wire.erl b/fluxer_gateway/src/guild/guild_data_wire.erl index 6e9315516..dfc43d662 100644 --- a/fluxer_gateway/src/guild/guild_data_wire.erl +++ b/fluxer_gateway/src/guild/guild_data_wire.erl @@ -99,6 +99,8 @@ is_fast_snowflake_scalar(_) -> -spec binary_key_kind(binary()) -> key_kind(). binary_key_kind(<> = Key) when First >= $1, First =< $9 -> numeric_binary_key_kind(Key); +binary_key_kind(<<"_fluxer_", _/binary>>) -> + drop; binary_key_kind(Key) -> named_or_suffix_kind(Key). @@ -161,6 +163,9 @@ named_key_kind(<<"member_role_index">>) -> drop; named_key_kind(<<"member_list_revision">>) -> drop; named_key_kind(<<"role_perms_cache">>) -> drop; named_key_kind(<<"overwrite_perms_cache">>) -> drop; +named_key_kind(<<"thread_index">>) -> drop; +named_key_kind(<<"member_ids_preview">>) -> drop; +named_key_kind(<<"applied_tags">>) -> scalar_list; named_key_kind(_) -> unknown. -spec has_suffix(binary(), binary()) -> boolean(). @@ -207,6 +212,19 @@ fast_payload_drops_internal_keys_test() -> Data = #{<<"id">> => 1, <<"role_index">> => #{}, role_perms_cache => #{}}, ?assertEqual(#{<<"id">> => <<"1">>}, fast_payload(Data, false)). +fast_payload_shapes_thread_keys_test() -> + Data = #{ + <<"id">> => 1, + <<"applied_tags">> => [2, 3], + <<"member_ids_preview">> => [4], + <<"_fluxer_thread">> => #{}, + <<"thread_index">> => #{} + }, + ?assertEqual( + #{<<"id">> => <<"1">>, <<"applied_tags">> => [<<"2">>, <<"3">>]}, + fast_payload(Data, false) + ). + pre_encoded_payload_passes_through_unchanged_test() -> Data = {pre_encoded, <<"{}">>}, ?assertEqual(Data, payload(Data)). diff --git a/fluxer_gateway/src/guild/guild_dispatch.erl b/fluxer_gateway/src/guild/guild_dispatch.erl index b58113ae3..d0aec0fc6 100644 --- a/fluxer_gateway/src/guild/guild_dispatch.erl +++ b/fluxer_gateway/src/guild/guild_dispatch.erl @@ -20,15 +20,11 @@ -type event() :: atom(). -type event_data() :: map(). --spec normalize_event(term()) -> event(). +-spec normalize_event(atom() | binary()) -> atom() | binary(). normalize_event(Event) when is_atom(Event) -> Event; normalize_event(Event) when is_binary(Event) -> - normalized_event_atom(event_atoms:normalize(Event)). - --spec normalized_event_atom(atom() | binary()) -> event(). -normalized_event_atom(Event) when is_atom(Event) -> - Event. + event_atoms:normalize(Event). -spec handle_dispatch(term(), event_data(), guild_state()) -> {noreply, guild_state()}. handle_dispatch(Event, EventData, State) -> @@ -36,10 +32,19 @@ handle_dispatch(Event, EventData, State) -> true -> {noreply, State}; false -> - NormalizedEvent = normalize_event(Event), - process_dispatch(NormalizedEvent, EventData, State) + normalized_event_atom(normalize_event(Event), EventData, State) end. +-spec normalized_event_atom(atom() | binary(), event_data(), guild_state()) -> + {noreply, guild_state()}. +normalized_event_atom(Event, EventData, State) when is_atom(Event) -> + process_dispatch(Event, EventData, State); +normalized_event_atom(Event, _EventData, State) when is_binary(Event) -> + ?LOG_WARNING("guild dispatch dropped unknown event: event=~ts guild_id=~p", [ + Event, maps:get(id, State, undefined) + ]), + {noreply, State}. + -spec should_skip_dispatch(term(), guild_state()) -> boolean(). should_skip_dispatch(guild_update, _State) -> false; @@ -52,16 +57,47 @@ should_skip_dispatch(_Event, State) -> -spec process_dispatch(event(), event_data(), guild_state()) -> {noreply, guild_state()}. process_dispatch(Event, EventData, State) -> + case guild_thread_gate:pre_update_filter(Event, EventData, State) of + drop -> + ok = maybe_self_heal(State), + {noreply, guild_thread_flip:hold(Event, EventData, State)}; + {filtered, FilteredData} -> + ok = maybe_self_heal(State), + process_regular_dispatch( + Event, FilteredData, guild_thread_flip:hold(Event, EventData, State) + ); + pass -> + State1 = guild_thread_flip:hold(Event, EventData, State), + case guild_thread_dispatch:handles(Event) of + true -> guild_thread_dispatch:handle(Event, EventData, State1); + false -> process_regular_dispatch(Event, EventData, State1) + end + end. + +-spec maybe_self_heal(guild_state()) -> ok. +maybe_self_heal(#{id := GuildId}) when is_integer(GuildId) -> + case channel_threads_config:guild_active(GuildId) of + true -> channel_threads_flip:self_heal(GuildId); + false -> ok + end; +maybe_self_heal(_State) -> + ok. + +-spec process_regular_dispatch(event(), event_data(), guild_state()) -> + {noreply, guild_state()}. +process_regular_dispatch(Event, EventData, State) -> GuildId = maps:get(id, State), {SessionIdOpt, CleanData} = extract_session_id_if_needed(Event, EventData), DecoratedData = CleanData#{<<"guild_id">> => integer_to_binary(GuildId)}, - FinalData = guild_dispatch_decorate:decorate_member_data(Event, DecoratedData, State), - UpdatedState = guild_state:update_state(Event, FinalData, State), + FinalData0 = guild_dispatch_decorate:decorate_member_data(Event, DecoratedData, State), + UpdatedState0 = guild_state:update_state(Event, FinalData0, State), + UpdatedState = guild_state_threads:after_regular_event(Event, FinalData0, UpdatedState0), FilterState = filter_state_for_event(Event, State, UpdatedState), Sessions = maps:get(sessions, UpdatedState, #{}), - FilteredSessions = guild_dispatch_filter:filter_sessions_for_event( - Event, FinalData, SessionIdOpt, Sessions, FilterState + FilteredSessions = filter_dispatch_sessions( + Event, FinalData0, SessionIdOpt, Sessions, FilterState ), + FinalData = strip_thread_internal(FinalData0, UpdatedState), ?LOG_DEBUG( "process_dispatch: event=~p guild_id=~p total_sessions=~p filtered_sessions=~p", [Event, GuildId, map_size(Sessions), length(FilteredSessions)] @@ -73,6 +109,32 @@ process_dispatch(Event, EventData, State) -> ), {noreply, FinalState}. +-spec filter_dispatch_sessions( + event(), event_data(), binary() | undefined, map(), guild_state() +) -> [{binary(), map()}]. +filter_dispatch_sessions(Event, Data, SessionIdOpt, Sessions, State) -> + case guild_thread_gate:event_scope(Event, Data, State) of + open -> + guild_dispatch_filter:filter_sessions_for_event( + Event, Data, SessionIdOpt, Sessions, State + ); + viewers -> + guild_dispatch_filter:filter_sessions_for_event( + Event, Data, SessionIdOpt, guild_thread_gate:viewer_sessions(Sessions), State + ); + {thread, ThreadId, Thread} -> + guild_thread_gate:thread_sessions( + Event, Data, ThreadId, Thread, SessionIdOpt, State + ) + end. + +-spec strip_thread_internal(event_data(), guild_state()) -> event_data(). +strip_thread_internal(Data, State) -> + case guild_thread_gate:needs_variant(State) of + true -> guild_thread_gate:strip_internal(Data); + false -> Data + end. + -spec filter_state_for_event(event(), guild_state(), guild_state()) -> guild_state(). filter_state_for_event(channel_delete, PreviousState, _UpdatedState) -> PreviousState; @@ -161,6 +223,17 @@ should_skip_dispatch_normal_guild_test() -> }, ?assertEqual(false, should_skip_dispatch(message_create, State)). +unknown_binary_event_is_dropped_without_crashing_test() -> + State = #{id => 1, data => #{<<"guild">> => #{}}, sessions => #{}}, + ?assertEqual( + {noreply, State}, + handle_dispatch(<<"FUTURE_EVENT_THE_GATEWAY_HAS_NEVER_SEEN">>, #{}, State) + ). + +normalize_event_keeps_unknown_binaries_test() -> + ?assertEqual(<<"FUTURE_EVENT_ZZZ">>, normalize_event(<<"FUTURE_EVENT_ZZZ">>)), + ?assertEqual(thread_create, normalize_event(<<"THREAD_CREATE">>)). + should_skip_dispatch_no_features_test() -> State = #{data => #{<<"guild">> => #{}}}, ?assertEqual(false, should_skip_dispatch(message_create, State)). diff --git a/fluxer_gateway/src/guild/guild_dispatch_push.erl b/fluxer_gateway/src/guild/guild_dispatch_push.erl index 3b99373a6..e77eb2194 100644 --- a/fluxer_gateway/src/guild/guild_dispatch_push.erl +++ b/fluxer_gateway/src/guild/guild_dispatch_push.erl @@ -10,6 +10,8 @@ ]). -define(MAX_FORMAT_MEMBERS, 50). +-define(MAX_ROLE_MENTION_THREAD_ADDS, 250). +-define(MAX_THREAD_MEMBERS, 1000). -define(CONCURRENCY_LIMIT_KEY, guild_push_concurrency_limit). -define(DEFAULT_PUSH_CONCURRENCY, 2). -define(MAX_PUSH_CONCURRENCY, 8). @@ -44,6 +46,7 @@ -type grace_sessions() :: #{user_id() => [{binary(), pid()}]}. -type grace_hold() :: {grace_sessions(), integer()} | none. -type held_push() :: {[user_id()], grace_hold(), term(), pid() | undefined}. +-type member_source() :: {ets, ets:tid()} | {map, map()}. -export_type([event/0, event_data/0, guild_state/0, guild_id/0]). -spec maybe_send_push_notifications(event(), event_data(), guild_id(), guild_state()) -> ok. @@ -279,7 +282,9 @@ compact_push_data(Data) -> <<"channels">>, <<"channel_index">>, overwrite_perms_cache, - members_ets + members_ets, + thread_gate, + thread_store ], Data ). @@ -423,14 +428,124 @@ scan_and_send_compact_push(MessageData, GuildId, ChannelId, State) -> Context = compact_scan_context(MessageData, ChannelId, State), MembersTab = eqwalizer:dynamic_cast(maps:get(members_ets, State)), case scan_push_members(MembersTab, Context, State) of - {ok, #{eligible_user_ids := [], held_user_ids := []}} -> - ok; + {ok, #{eligible_user_ids := [], held_user_ids := []} = Scan} -> + send_compact_forum_thread_created( + MessageData, GuildId, ChannelId, MembersTab, Scan, State + ); {ok, Scan} -> - send_compact_scanned_push(MessageData, GuildId, Scan, State); + ok = send_compact_scanned_push(MessageData, GuildId, Scan, State), + send_compact_forum_thread_created( + MessageData, GuildId, ChannelId, MembersTab, Scan, State + ); {error, table_unavailable} -> scan_table_unavailable(GuildId) end. +-spec send_compact_forum_thread_created( + event_data(), guild_id(), integer(), ets:tid(), map(), guild_state() +) -> ok. +send_compact_forum_thread_created(MessageData, GuildId, ChannelId, MembersTab, Scan, State) -> + send_forum_thread_created( + MessageData, + GuildId, + ChannelId, + {ets, MembersTab}, + fun() -> compact_format_data(maps:get(data, State), maps:get(format_members, Scan)) end, + maps:get(session_eligibility, State, #{}), + maps:get(grace_hold, State, none), + State + ). + +-spec send_forum_thread_created( + event_data(), + guild_id(), + integer(), + member_source(), + fun(() -> map()), + map(), + grace_hold(), + guild_state() +) -> ok. +send_forum_thread_created( + MessageData, GuildId, ChannelId, Source, FormatData, SessionEligibility, Hold, State +) -> + case guild_thread_push:forum_starter(MessageData, ChannelId, maps:get(data, State, #{})) of + undefined -> + ok; + {ForumId, ThreadMembers} -> + Skip = maps:merge( + maps:put(message_author_id(MessageData), true, ThreadMembers), + direct_mention_id_set(MessageData) + ), + MentionRoles = mention_role_id_set(maps:get(<<"mention_roles">>, MessageData, [])), + Recipient = guild_thread_gate:recipient_active(ChannelId, State), + HeldSessions = held_sessions(Hold), + Recipients = fold_members( + fun(UserId, Member, Acc) -> + Route = + case maps:is_key(UserId, Skip) of + true -> skip; + false -> forum_push_route(UserId, SessionEligibility, HeldSessions) + end, + case + Route =/= skip andalso + not member_has_mentioned_role(Member, MentionRoles) andalso + not guild_members_common:is_member_bot(Member) andalso + Recipient(UserId) andalso + guild_permissions:can_view_channel(UserId, ForumId, Member, State) + of + true -> [{Route, UserId, extract_role_ids(Member)} | Acc]; + false -> Acc + end + end, + [], + Source + ), + send_forum_thread_created_push( + MessageData, GuildId, Recipients, SessionEligibility, Hold, FormatData(), State + ) + end. + +-spec forum_push_route(user_id(), map(), grace_sessions()) -> + eligible_user_ids | held_user_ids | skip. +forum_push_route(UserId, SessionEligibility, HeldSessions) -> + case maps:get(UserId, SessionEligibility, true) of + true -> eligible_user_ids; + false -> grace_member_route(maps:is_key(UserId, HeldSessions)) + end. + +-spec send_forum_thread_created_push( + event_data(), + guild_id(), + [{eligible_user_ids | held_user_ids, user_id(), [integer()]}], + map(), + grace_hold(), + map(), + guild_state() +) -> ok. +send_forum_thread_created_push( + _MessageData, _GuildId, [], _SessionEligibility, _Hold, _Data, _State +) -> + ok; +send_forum_thread_created_push( + MessageData, GuildId, Recipients, SessionEligibility, Hold, Data, State +) -> + send_push_to_eligible_users( + guild_thread_push:mark_forum_thread_created(MessageData), + GuildId, + [UserId || {eligible_user_ids, UserId, _RoleIds} <- Recipients], + maps:from_list([{UserId, RoleIds} || {_, UserId, RoleIds} <- Recipients]), + SessionEligibility, + Data, + large_guild_meta(State), + { + [UserId || {held_user_ids, UserId, _RoleIds} <- Recipients], + Hold, + maps:get(member_presence, State, undefined), + maps:get(guild_pid, State, undefined) + } + ). + -spec scan_table_unavailable(guild_id()) -> ok. scan_table_unavailable(GuildId) -> count_push_event(scan_table_unavailable), @@ -447,8 +562,11 @@ compact_scan_context(MessageData, ChannelId, State) -> mention_everyone => maps:get(<<"mention_everyone">>, MessageData, false) =:= true, direct_mentions => direct_mention_id_set(MessageData), mention_roles => mention_role_id_set(MentionRoles), + author_id => message_author_id(MessageData), format_members => format_member_id_set(MessageData), channel_id => ChannelId, + thread_recipient => guild_thread_gate:recipient_active(ChannelId, State), + thread_scope => guild_thread_push:scope(ChannelId, State), session_eligibility => maps:get(session_eligibility, State), grace_sessions => held_sessions(maps:get(grace_hold, State, none)) }. @@ -457,7 +575,9 @@ compact_scan_context(MessageData, ChannelId, State) -> {ok, map()} | {error, table_unavailable}. scan_push_members(MembersTab, Context, State) -> try - {ok, fold_push_members(member_id_snapshot(MembersTab), MembersTab, Context, State)} + Context1 = with_thread_role_adds(MembersTab, Context, State), + {ok, + fold_push_members(scan_user_ids(MembersTab, Context1), MembersTab, Context1, State)} catch error:badarg -> {error, table_unavailable} end. @@ -471,6 +591,17 @@ initial_scan_acc() -> format_members => #{} }. +-spec scan_user_ids(ets:tid(), map()) -> [user_id()]. +scan_user_ids(_MembersTab, #{thread_scope := #{members := Members}} = Context) -> + lists:usort( + maps:keys(Members) ++ + maps:keys(maps:get(direct_mentions, Context)) ++ + maps:keys(maps:get(format_members, Context)) ++ + maps:keys(maps:get(role_adds, Context)) + ); +scan_user_ids(MembersTab, _Context) -> + member_id_snapshot(MembersTab). + -spec member_id_snapshot(ets:tid()) -> [user_id()]. member_id_snapshot(MembersTab) -> ets:select(MembersTab, [{{'$1', '_'}, [], ['$1']}]). @@ -505,18 +636,173 @@ scan_push_member(_Row, _Context, _State, Acc) -> -spec scanned_member_route(user_id(), map(), map(), guild_state()) -> eligible_user_ids | held_user_ids | skip. scanned_member_route(UserId, Member, Context, State) -> + case + thread_push_candidate( + UserId, Member, maps:get(thread_scope, Context, undefined), Context + ) + of + true -> member_route(UserId, Member, Context, State); + false -> skip + end. + +-spec thread_push_candidate(user_id(), map(), guild_thread_push:scope(), map()) -> boolean(). +thread_push_candidate(_UserId, _Member, undefined, _Mentions) -> + true; +thread_push_candidate(UserId, _Member, #{members := Members}, Mentions) -> + maps:is_key(UserId, Members) orelse + maps:is_key(UserId, maps:get(direct_mentions, Mentions)) orelse + maps:is_key(UserId, maps:get(role_adds, Mentions)). + +-spec with_thread_role_adds(ets:tid(), map(), guild_state()) -> map(). +with_thread_role_adds(MembersTab, #{thread_scope := Scope} = Context, State) when + Scope =/= undefined +-> + Context#{ + role_adds => thread_role_adds( + Scope, + Context, + {ets, MembersTab}, + thread_role_visible(maps:get(channel_id, Context), State) + ) + }; +with_thread_role_adds(_MembersTab, Context, _State) -> + Context. + +-spec fold_members(fun((user_id(), map(), Acc) -> Acc), Acc, member_source()) -> Acc. +fold_members(Fun, Acc0, {ets, MembersTab}) -> + ets:foldl( + fun + ({UserId, Member}, Acc) when is_integer(UserId), is_map(Member) -> + Fun(UserId, Member, Acc); + (_Row, Acc) -> + Acc + end, + Acc0, + MembersTab + ); +fold_members(Fun, Acc0, {map, Members}) -> + maps:fold( + fun + (UserId, Member, Acc) when is_integer(UserId), is_map(Member) -> + Fun(UserId, Member, Acc); + (_UserId, _Member, Acc) -> + Acc + end, + Acc0, + Members + ). + +-spec lookup_member(user_id(), member_source()) -> map() | undefined. +lookup_member(UserId, {ets, MembersTab}) -> + case ets:lookup(MembersTab, UserId) of + [{_, Member}] when is_map(Member) -> Member; + _ -> undefined + end; +lookup_member(UserId, {map, Members}) -> + case maps:get(UserId, Members, undefined) of + Member when is_map(Member) -> Member; + _ -> undefined + end. + +-spec thread_role_visible(integer(), guild_state()) -> fun((user_id(), map()) -> boolean()). +thread_role_visible(ChannelId, State) -> + Recipient = guild_thread_gate:recipient_active(ChannelId, State), + fun(UserId, Member) -> + Recipient(UserId) andalso + guild_permissions:can_view_channel(UserId, ChannelId, Member, State) + end. + +-spec thread_role_adds( + map(), map(), member_source(), fun((user_id(), map()) -> boolean()) +) -> #{user_id() => true}. +thread_role_adds(#{thread := Thread, members := Members}, Mentions, Source, Visible) -> + MentionRoles = maps:get(mention_roles, Mentions), + case map_size(MentionRoles) > 0 andalso thread_accepts_role_adds(Thread) of + false -> + #{}; + true -> + Direct = maps:get(direct_mentions, Mentions), + AuthorId = maps:get(author_id, Mentions), + Candidates = fold_members( + fun(UserId, Member, Acc) -> + case + UserId =/= AuthorId andalso + not maps:is_key(UserId, Members) andalso + not maps:is_key(UserId, Direct) andalso + member_has_mentioned_role(Member, MentionRoles) andalso + not guild_members_common:is_member_bot(Member) + of + true -> [UserId | Acc]; + false -> Acc + end + end, + [], + Source + ), + Room = max(0, ?MAX_THREAD_MEMBERS - map_size(Members)), + take_role_adds( + lists:sort(Candidates), + Source, + Visible, + min(?MAX_ROLE_MENTION_THREAD_ADDS, Room), + #{} + ) + end. + +-spec thread_accepts_role_adds(map()) -> boolean(). +thread_accepts_role_adds(#{<<"type">> := 12}) -> false; +thread_accepts_role_adds(#{<<"thread_metadata">> := #{<<"archived">> := true}}) -> false; +thread_accepts_role_adds(_Thread) -> true. + +-spec take_role_adds( + [user_id()], + member_source(), + fun((user_id(), map()) -> boolean()), + non_neg_integer(), + map() +) -> #{user_id() => true}. +take_role_adds([], _Source, _Visible, _Left, Acc) -> + Acc; +take_role_adds(_UserIds, _Source, _Visible, 0, Acc) -> + Acc; +take_role_adds([UserId | Rest], Source, Visible, Left, Acc) -> + case lookup_member(UserId, Source) of + Member when is_map(Member) -> + case Visible(UserId, Member) of + true -> take_role_adds(Rest, Source, Visible, Left - 1, Acc#{UserId => true}); + false -> take_role_adds(Rest, Source, Visible, Left, Acc) + end; + undefined -> + take_role_adds(Rest, Source, Visible, Left, Acc) + end. + +-spec message_author_id(event_data()) -> user_id() | undefined. +message_author_id(MessageData) -> + case maps:get(<<"author">>, MessageData, undefined) of + #{<<"id">> := Id} -> snowflake_id:parse_optional(Id); + _ -> undefined + end. + +-spec member_route(user_id(), map(), map(), guild_state()) -> + eligible_user_ids | held_user_ids | skip. +member_route(UserId, Member, Context, State) -> case member_push_route(UserId, Member, Context) of skip -> skip; Key -> + ThreadRecipient = maps:get(thread_recipient, Context, fun all_recipients/1), visible_member_route( Key, guild_permissions:can_view_channel( UserId, maps:get(channel_id, Context), Member, State - ) + ) andalso ThreadRecipient(UserId) ) end. +-spec all_recipients(user_id()) -> true. +all_recipients(_UserId) -> + true. + -spec member_push_route(user_id(), map(), map()) -> eligible_user_ids | held_user_ids | skip. member_push_route(UserId, Member, Context) -> @@ -669,7 +955,7 @@ send_push_notifications(MessageData, GuildId, State) -> undefined -> ok; ChannelId -> - send_to_eligible( + ok = send_to_eligible( MessageData, GuildId, Members, @@ -679,6 +965,16 @@ send_push_notifications(MessageData, GuildId, State) -> Hold, Data, State + ), + send_forum_thread_created( + MessageData, + GuildId, + ChannelId, + {map, Members}, + fun() -> Data end, + SessionEligibility, + Hold, + State ) end. @@ -704,11 +1000,15 @@ send_to_eligible( Data, State ) -> + ThreadCandidates = thread_candidate_filter(ChannelId, Members, MessageData, State), EligibleUserIds = find_eligible_users_for_push( - Members, CandidateUserIds, ChannelId, State + Members, ThreadCandidates(CandidateUserIds), ChannelId, State ), HeldUserIds = find_eligible_users_for_push( - Members, held_candidate_user_ids(Hold, CandidateUserIds), ChannelId, State + Members, + ThreadCandidates(held_candidate_user_ids(Hold, CandidateUserIds)), + ChannelId, + State ), case {EligibleUserIds, HeldUserIds} of {[], []} -> @@ -732,6 +1032,39 @@ send_to_eligible( ) end. +-spec thread_candidate_filter(integer(), map(), event_data(), guild_state()) -> + fun(([user_id()]) -> [user_id()]). +thread_candidate_filter(ChannelId, Members, MessageData, State) -> + case guild_thread_push:scope(ChannelId, State) of + undefined -> + fun(UserIds) -> UserIds end; + Scope -> + Mentions0 = #{ + direct_mentions => direct_mention_id_set(MessageData), + mention_roles => mention_role_id_set( + maps:get(<<"mention_roles">>, MessageData, []) + ), + author_id => message_author_id(MessageData) + }, + Mentions = Mentions0#{ + role_adds => thread_role_adds( + Scope, + Mentions0, + {map, Members}, + thread_role_visible(ChannelId, State) + ) + }, + fun(UserIds) -> + [ + UserId + || UserId <- UserIds, + thread_push_candidate( + UserId, maps:get(UserId, Members, #{}), Scope, Mentions + ) + ] + end + end. + -spec held_candidate_user_ids(grace_hold(), [user_id()]) -> [user_id()]. held_candidate_user_ids(none, _CandidateUserIds) -> []; @@ -875,8 +1208,11 @@ member_roles_include_mentioned([Role | Rest], MentionRoleSet) -> -spec find_eligible_users_for_push(map(), [user_id()], integer(), guild_state()) -> [user_id()]. find_eligible_users_for_push(Members, CandidateUserIds, ChannelId, State) -> + ThreadRecipient = guild_thread_gate:recipient_active(ChannelId, State), lists:filtermap( - fun(UserId) -> is_push_eligible(UserId, Members, ChannelId, State) end, + fun(UserId) -> + ThreadRecipient(UserId) andalso is_push_eligible(UserId, Members, ChannelId, State) + end, CandidateUserIds ). @@ -1045,10 +1381,12 @@ send_push_to_eligible_users( ok; AuthorId -> ChannelIdBin = maps:get(<<"channel_id">>, MessageData), - ChannelName = find_channel_name(ChannelIdBin, Data), + {PushMessageData, ChannelName} = push_message_data( + MessageData, ChannelIdBin, EligibleUserIds ++ element(1, Held), Data + ), RoleNames = build_role_names_map(Data), Params = push_params( - MessageData, + PushMessageData, GuildId, UserRolesMap, ConnectedUsers, @@ -1062,6 +1400,21 @@ send_push_to_eligible_users( hold_push_through_grace(Held, Params) end. +-spec push_message_data(event_data(), term(), [user_id()], map()) -> {event_data(), binary()}. +push_message_data(MessageData, ChannelIdBin, UserIds, Data) -> + ChannelId = guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin), + case thread_push_message_data(MessageData, ChannelId, UserIds, Data) of + undefined -> {MessageData, find_channel_name(ChannelIdBin, Data)}; + ThreadPush -> ThreadPush + end. + +-spec thread_push_message_data(event_data(), integer() | undefined, [user_id()], map()) -> + {event_data(), binary()} | undefined. +thread_push_message_data(_MessageData, undefined, _UserIds, _Data) -> + undefined; +thread_push_message_data(MessageData, ChannelId, UserIds, Data) -> + guild_thread_push:push_message_data(MessageData, ChannelId, UserIds, Data). + -spec push_params( event_data(), guild_id(), @@ -1904,6 +2257,154 @@ compact_push_includes_large_guild_metadata_test() -> ets:delete(Tab) end. +thread_scope_context() -> + #{ + thread_scope => #{thread => #{<<"id">> => 500}, members => #{1 => #{}}}, + direct_mentions => #{2 => true}, + mention_roles => #{}, + role_adds => #{}, + author_id => 99, + format_members => #{4 => true} + }. + +thread_scan_visits_members_mentions_and_format_ids_only_test() -> + Tab = ets:new(test_members, [set, public]), + try + true = ets:insert(Tab, [{Id, #{}} || Id <- [1, 2, 3, 4, 5]]), + Context = thread_scope_context(), + ?assertEqual([1, 2, 4], scan_user_ids(Tab, Context)), + ?assertEqual([1, 2, 3, 4], scan_user_ids(Tab, Context#{role_adds => #{3 => true}})), + ?assertEqual( + [1, 2, 3, 4, 5], lists:sort(scan_user_ids(Tab, Context#{thread_scope => undefined})) + ) + after + ets:delete(Tab) + end. + +thread_push_candidates_are_members_and_mentioned_users_test() -> + Context = thread_scope_context(), + Scope = maps:get(thread_scope, Context), + RoleMember = #{<<"roles">> => [<<"9">>]}, + ?assert(thread_push_candidate(1, #{}, Scope, Context)), + ?assert(thread_push_candidate(2, #{}, Scope, Context)), + ?assertNot(thread_push_candidate(3, #{}, Scope, Context)), + ?assertNot(thread_push_candidate(3, RoleMember, Scope, Context)), + ?assertNot( + thread_push_candidate(3, RoleMember, Scope, Context#{mention_roles => #{9 => true}}) + ), + ?assert(thread_push_candidate(3, RoleMember, Scope, Context#{role_adds => #{3 => true}})), + ?assert(thread_push_candidate(3, #{}, undefined, Context)). + +thread_role_adds_cap_at_first_visible_role_holders_test() -> + Scope = #{thread => #{<<"id">> => 500, <<"type">> => 11}, members => #{1 => #{}}}, + Context = (thread_scope_context())#{mention_roles => #{9 => true}}, + RoleMember = #{<<"roles">> => [<<"9">>]}, + Bot = RoleMember#{<<"user">> => #{<<"bot">> => true}}, + Extra = [{Id, RoleMember} || Id <- lists:seq(1000, 1000 + ?MAX_ROLE_MENTION_THREAD_ADDS)], + Rows = maps:from_list( + lists:reverse(Extra) ++ + [ + {1, RoleMember}, + {2, RoleMember}, + {5, RoleMember}, + {6, #{}}, + {7, Bot}, + {8, RoleMember}, + {99, RoleMember} + ] + ), + Visible = fun(UserId, _Member) -> UserId =/= 8 end, + Adds = thread_role_adds(Scope, Context, {map, Rows}, Visible), + ?assertEqual(?MAX_ROLE_MENTION_THREAD_ADDS, map_size(Adds)), + ?assert(maps:is_key(5, Adds)), + ?assert(maps:is_key(1000, Adds)), + ?assertNot(maps:is_key(1000 + ?MAX_ROLE_MENTION_THREAD_ADDS - 1, Adds)), + ?assertNot(maps:is_key(1000 + ?MAX_ROLE_MENTION_THREAD_ADDS, Adds)), + [ + ?assertNot(maps:is_key(Id, Adds)) + || Id <- [1, 2, 6, 7, 8, 99] + ], + ?assertNot( + thread_push_candidate(1000 + ?MAX_ROLE_MENTION_THREAD_ADDS, RoleMember, Scope, Context#{ + role_adds => Adds + }) + ), + ?assert(thread_push_candidate(5, RoleMember, Scope, Context#{role_adds => Adds})). + +thread_role_adds_cap_at_thread_member_room_test() -> + Existing = maps:from_list([{Id, #{}} || Id <- lists:seq(1, ?MAX_THREAD_MEMBERS - 5)]), + Scope = #{thread => #{<<"id">> => 500, <<"type">> => 11}, members => Existing}, + Context = (thread_scope_context())#{mention_roles => #{9 => true}}, + RoleMember = #{<<"roles">> => [<<"9">>]}, + Rows = maps:from_list([{Id, RoleMember} || Id <- lists:seq(5000, 5099)]), + Visible = fun(_UserId, _Member) -> true end, + Adds = thread_role_adds(Scope, Context, {map, Rows}, Visible), + ?assertEqual(lists:seq(5000, 5004), lists:sort(maps:keys(Adds))), + Full = maps:from_list([{Id, #{}} || Id <- lists:seq(1, ?MAX_THREAD_MEMBERS)]), + ?assertEqual( + #{}, thread_role_adds(Scope#{members => Full}, Context, {map, Rows}, Visible) + ). + +thread_role_adds_skip_private_and_archived_threads_test() -> + Context = (thread_scope_context())#{mention_roles => #{9 => true}}, + Rows = {map, #{5 => #{<<"roles">> => [<<"9">>]}}}, + Visible = fun(_UserId, _Member) -> true end, + Scope = fun(Thread) -> #{thread => Thread, members => #{}} end, + ?assertEqual( + #{5 => true}, thread_role_adds(Scope(#{<<"type">> => 11}), Context, Rows, Visible) + ), + ?assertEqual(#{}, thread_role_adds(Scope(#{<<"type">> => 12}), Context, Rows, Visible)), + ?assertEqual( + #{}, + thread_role_adds( + Scope(#{<<"type">> => 11, <<"thread_metadata">> => #{<<"archived">> => true}}), + Context, + Rows, + Visible + ) + ), + ?assertEqual( + #{}, + thread_role_adds( + Scope(#{<<"type">> => 11}), Context#{mention_roles => #{}}, Rows, Visible + ) + ). + +thread_push_message_data_names_the_thread_test() -> + Tab = ets:new(test_members, [set, public]), + Store = guild_thread_store:new(), + try + guild_thread_store:load( + Store, + [#{<<"id">> => 500, <<"parent_id">> => 10, <<"name">> => <<"ideas">>}], + [#{<<"id">> => <<"500">>, <<"user_id">> => <<"1">>, <<"flags">> => 4}] + ), + Data = (compact_test_data(Tab))#{ + thread_gate => #{active => true, version => 1}, thread_store => Store + }, + Message = #{<<"channel_id">> => <<"500">>}, + {ThreadMessage, Name} = push_message_data(Message, <<"500">>, [1, 2], Data), + ?assertEqual(<<"ideas">>, Name), + ?assertEqual( + #{ + <<"parent_id">> => 10, + <<"parent_name">> => <<"general">>, + <<"category_id">> => undefined, + <<"members">> => #{1 => #{<<"flags">> => 4}} + }, + maps:get(<<"__thread_push">>, ThreadMessage) + ), + Plain = #{<<"channel_id">> => <<"10">>}, + ?assertEqual({Plain, <<"general">>}, push_message_data(Plain, <<"10">>, [1], Data)), + ?assertEqual( + {Message, <<"unknown">>}, + push_message_data(Message, <<"500">>, [1], compact_test_data(Tab)) + ) + after + guild_thread_store:destroy(Store), + ets:delete(Tab) + end. + compact_format_data_restricts_member_map_test() -> Data = #{ <<"guild">> => #{}, @@ -2259,6 +2760,168 @@ reset_push_worker_state() -> erase(push_inflight_workers), ok = application:unset_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY). +forum_thread_created_state(Tab, Store) -> + Forum = #{<<"id">> => <<"300">>, <<"name">> => <<"ideas">>, <<"type">> => 15}, + Base = compact_test_data(Tab), + #{ + id => 7, + data => compact_push_data(Base#{ + <<"channels">> => [Forum], + <<"channel_index">> => #{300 => Forum}, + thread_gate => #{active => true, version => 1}, + thread_store => Store + }), + virtual_channel_access => maps:from_list([ + {UserId, sets:from_list([300])} + || UserId <- [1, 2, 3, 4, 6, 42] + ]), + members_ets => Tab, + session_eligibility => #{3 => false}, + member_count => 10 + }. + +collect_push_params(Acc) -> + receive + {push_params, Params} -> collect_push_params([Params | Acc]) + after 100 -> lists:reverse(Acc) + end. + +forum_thread_created_pushes_forum_watchers_outside_the_thread_test() -> + Self = self(), + Tab = ets:new(test_members, [set, public]), + Store = guild_thread_store:new(), + Previous = persistent_term:get(channel_threads_config, undefined), + {ok, Config} = channel_threads_config:validate_config(#{ + <<"enabled">> => true, <<"user_basis_points">> => 10000 + }), + persistent_term:put(channel_threads_config, Config), + ok = meck:new(push, [passthrough, no_link]), + try + ok = meck:expect(push, handle_message_create, fun(Params) -> + Self ! {push_params, Params}, + ok + end), + Bot = #{<<"roles">> => [], <<"user">> => #{<<"bot">> => true}}, + true = ets:insert( + Tab, [{Id, #{<<"roles">> => []}} || Id <- [1, 2, 3, 4, 5, 42]] ++ [{6, Bot}] + ), + guild_thread_store:load( + Store, + [ + #{ + <<"id">> => 700, + <<"parent_id">> => 300, + <<"type">> => 11, + <<"name">> => <<"post">> + } + ], + [#{<<"id">> => <<"700">>, <<"user_id">> => <<"1">>, <<"flags">> => 0}] + ), + State = forum_thread_created_state(Tab, Store), + Starter = #{ + <<"id">> => <<"700">>, + <<"channel_id">> => <<"700">>, + <<"author">> => #{<<"id">> => <<"42">>}, + <<"mentions">> => [#{<<"id">> => <<"4">>}] + }, + ok = scan_and_send_compact_push(Starter, 7, 700, State), + Forum = [ + Params + || #{message_data := #{<<"__thread_push">> := #{<<"forum_thread_created">> := true}}} = + Params <- collect_push_params([]) + ], + ?assertMatch([#{user_ids := [2], channel_name := <<"post">>}], Forum), + [#{message_data := ForumMessage}] = Forum, + ?assertNot(maps:is_key(<<"__forum_thread_created">>, ForumMessage)), + ?assertMatch( + #{<<"parent_id">> := 300, <<"parent_name">> := <<"ideas">>}, + maps:get(<<"__thread_push">>, ForumMessage) + ), + Reply = Starter#{<<"id">> => <<"701">>}, + ok = scan_and_send_compact_push(Reply, 7, 700, State), + ?assertEqual( + [], + [ + P + || #{ + message_data := #{ + <<"__thread_push">> := #{<<"forum_thread_created">> := true} + } + } = + P <- collect_push_params([]) + ] + ) + after + meck:unload(push), + restore_threads_config(Previous), + guild_thread_store:destroy(Store), + ets:delete(Tab) + end. + +forum_thread_created_holds_suppressed_members_through_grace_test() -> + Self = self(), + Tab = ets:new(test_members, [set, public]), + Store = guild_thread_store:new(), + Previous = persistent_term:get(channel_threads_config, undefined), + {ok, Config} = channel_threads_config:validate_config(#{ + <<"enabled">> => true, <<"user_basis_points">> => 10000 + }), + persistent_term:put(channel_threads_config, Config), + ok = meck:new(push, [passthrough, no_link]), + try + ok = meck:expect(push, handle_message_create, fun(Params) -> + Self ! {push_params, Params}, + ok + end), + true = ets:insert(Tab, [{Id, #{<<"roles">> => []}} || Id <- [2, 3, 4, 42]]), + guild_thread_store:load( + Store, + [ + #{ + <<"id">> => 700, + <<"parent_id">> => 300, + <<"type">> => 11, + <<"name">> => <<"post">> + } + ], + [] + ), + {Ended, EndedRef} = spawn_monitor(fun() -> ok end), + receive + {'DOWN', EndedRef, process, Ended, _} -> ok + end, + Base = forum_thread_created_state(Tab, Store), + State = Base#{ + session_eligibility => #{3 => false, 4 => false}, + grace_hold => {#{3 => [{<<"s3">>, Ended}]}, erlang:monotonic_time(millisecond) + 20} + }, + Starter = #{ + <<"id">> => <<"700">>, + <<"channel_id">> => <<"700">>, + <<"author">> => #{<<"id">> => <<"42">>} + }, + ok = scan_and_send_compact_push(Starter, 7, 700, State), + Forum = [ + UserIds + || #{ + user_ids := UserIds, + message_data := #{<<"__thread_push">> := #{<<"forum_thread_created">> := true}} + } <- collect_push_params([]) + ], + ?assertEqual([[2], [3]], Forum) + after + meck:unload(push), + restore_threads_config(Previous), + guild_thread_store:destroy(Store), + ets:delete(Tab) + end. + +restore_threads_config(undefined) -> + _ = persistent_term:erase(channel_threads_config), + ok; +restore_threads_config(Previous) -> + persistent_term:put(channel_threads_config, Previous). + compact_test_data(Tab) -> #{ <<"guild">> => #{ diff --git a/fluxer_gateway/src/guild/guild_dispatch_send.erl b/fluxer_gateway/src/guild/guild_dispatch_send.erl index b68141741..f72dd3295 100644 --- a/fluxer_gateway/src/guild/guild_dispatch_send.erl +++ b/fluxer_gateway/src/guild/guild_dispatch_send.erl @@ -29,6 +29,44 @@ -spec dispatch_to_sessions([session_pair()], event(), event_data(), guild_state()) -> non_neg_integer(). dispatch_to_sessions(FilteredSessions, Event, FinalData, UpdatedState) -> + case guild_thread_gate:needs_variant(UpdatedState) of + false -> + dispatch_variant(FilteredSessions, Event, FinalData, UpdatedState); + true -> + dispatch_masked_variants(FilteredSessions, Event, FinalData, UpdatedState) + end. + +-spec dispatch_masked_variants([session_pair()], event(), event_data(), guild_state()) -> + non_neg_integer(). +dispatch_masked_variants(FilteredSessions, Event, FinalData, UpdatedState) -> + Masked = guild_thread_gate:mask_payload(Event, FinalData), + {Viewers, Others} = lists:partition( + fun({_Sid, Session}) -> guild_thread_gate:session_viewer(Session) end, FilteredSessions + ), + case {Masked =:= FinalData, Viewers, Others} of + {true, _, _} -> + dispatch_variant(FilteredSessions, Event, FinalData, UpdatedState); + {false, _, []} -> + dispatch_variant(Viewers, Event, FinalData, UpdatedState); + {false, [], _} -> + dispatch_masked(Others, Event, Masked, UpdatedState); + {false, _, _} -> + max( + dispatch_variant(Viewers, Event, FinalData, UpdatedState), + dispatch_masked(Others, Event, Masked, UpdatedState) + ) + end. + +-spec dispatch_masked([session_pair()], event(), event_data(), guild_state()) -> + non_neg_integer(). +dispatch_masked(Sessions, Event, Masked, UpdatedState) -> + Count = dispatch_variant(Sessions, Event, Masked, UpdatedState), + ok = guild_thread_dispatch:count_masked(Event, Count), + Count. + +-spec dispatch_variant([session_pair()], event(), event_data(), guild_state()) -> + non_neg_integer(). +dispatch_variant(FilteredSessions, Event, FinalData, UpdatedState) -> GuildId = maps:get(id, UpdatedState), case guild_dispatch_filter:is_bulk_update_event(Event) of true -> @@ -125,11 +163,34 @@ dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched) -> -spec filter_indexed_for_session(map(), [{integer() | undefined, map()}], guild_state()) -> [map()]. filter_indexed_for_session(SessionData, IndexedChannels, UpdatedState) -> + Visible = filter_permitted_for_session(SessionData, IndexedChannels, UpdatedState), + case guild_thread_gate:needs_variant(UpdatedState) of + false -> + Visible; + true -> + [ + Ch + || Ch <- Visible, + guild_thread_gate:channel_visible( + SessionData, + guild_dispatch_decorate:parse_snowflake( + <<"id">>, maps:get(<<"id">>, Ch, undefined) + ), + UpdatedState + ) + ] + end. + +-spec filter_permitted_for_session(map(), [{integer() | undefined, map()}], guild_state()) -> + [map()]. +filter_permitted_for_session(SessionData, IndexedChannels, UpdatedState) -> case maps:get(viewable_channels, SessionData, undefined) of ViewableMap when is_map(ViewableMap) -> [ Ch - || {ChId, Ch} <- IndexedChannels, is_integer(ChId), maps:is_key(ChId, ViewableMap) + || {ChId, Ch} <- IndexedChannels, + is_integer(ChId), + maps:is_key(ChId, ViewableMap) ]; _ -> UserId = maps:get(user_id, SessionData), diff --git a/fluxer_gateway/src/guild/guild_handoff.erl b/fluxer_gateway/src/guild/guild_handoff.erl index ea726ee06..9fe83d329 100644 --- a/fluxer_gateway/src/guild/guild_handoff.erl +++ b/fluxer_gateway/src/guild/guild_handoff.erl @@ -17,9 +17,13 @@ -spec export_handoff_state(guild_state()) -> map(). export_handoff_state(State) -> - #{ + Threads = maps:merge( + guild_thread_flip:export_handoff(State), + guild_thread_subscriptions:export_handoff(State) + ), + Threads#{ id => maps:get(id, State), - data => maps:without(derived_data_keys(), maps:get(data, State, #{})), + data => maps:without(derived_data_keys(), guild_thread_load:export_handoff(State)), sessions => export_handoff_sessions(maps:get(sessions, State, #{})), voice_states => maps:get(voice_states, State, #{}), virtual_channel_access => maps:get(virtual_channel_access, State, #{}), diff --git a/fluxer_gateway/src/guild/guild_health.erl b/fluxer_gateway/src/guild/guild_health.erl index ad91ed1d7..05e8e8cd1 100644 --- a/fluxer_gateway/src/guild/guild_health.erl +++ b/fluxer_gateway/src/guild/guild_health.erl @@ -7,6 +7,7 @@ -export([ start_link/0, is_overloaded/1, + is_degraded/1, register_guild/1, put_session/2, remove_session/2, @@ -42,6 +43,13 @@ is_overloaded(Pid) -> _:_ -> false end. +-spec is_degraded(pid()) -> boolean(). +is_degraded(Pid) -> + case lookup(Pid) of + {Pid, _GuildId, Degraded, _Targets, _Pending} -> Degraded =:= true; + undefined -> false + end. + -spec register_guild(map()) -> map(). register_guild(State) -> case maps:get(id, State, undefined) of diff --git a/fluxer_gateway/src/guild/guild_init.erl b/fluxer_gateway/src/guild/guild_init.erl index 8aefd6982..fa3602f31 100644 --- a/fluxer_gateway/src/guild/guild_init.erl +++ b/fluxer_gateway/src/guild/guild_init.erl @@ -27,7 +27,8 @@ init_base_state(GuildState) -> Data0 = maps:get(data, TransferSafe, #{}), ExistingVoice = maps:get(voice_states, TransferSafe, #{}), {VoiceStates, Data1} = extract_voice_states_from_data(Data0, ExistingVoice), - NormalizedData = guild_data_index:normalize_map(Data1), + {ThreadExtras, Data2} = guild_thread_load:take_extras(Data1), + NormalizedData = guild_data_index:normalize_map(Data2), MemberTab = ets:new(guild_members_data, [set, public, {read_concurrency, true}]), populate_member_ets(MemberTab, NormalizedData), BaseState = TransferSafe#{ @@ -44,7 +45,15 @@ init_base_state(GuildState) -> ), viewable_channels_cache => ets:new(viewable_channels_cache, [set, public]) }, - guild_handoff:restore_transferred_session_state(BaseState). + guild_thread_flip:converge( + guild_thread_subscriptions:restore_handoff( + guild_thread_flip:restore_handoff( + guild_handoff:restore_transferred_session_state( + guild_thread_load:install(ThreadExtras, BaseState) + ) + ) + ) + ). -spec populate_member_ets(ets:table(), map()) -> ok. populate_member_ets(Tab, Data) -> @@ -129,10 +138,17 @@ handle_reload(NewData, State) -> {ReloadVoiceStates, ReloadData} = extract_voice_states_from_data( NewData, ExistingVoiceStates ), - NormalizedNewData0 = guild_data_index:normalize_map(ReloadData), + {ThreadExtras, ReloadData1} = guild_thread_load:take_extras(ReloadData), + NormalizedNewData0 = guild_data_index:normalize_map(ReloadData1), NormalizedNewData = reuse_members_table(OldData, NormalizedNewData0), NewState0 = guild_member_list_engine_inputs:forget_all( - State#{voice_states => ReloadVoiceStates, data => NormalizedNewData} + guild_thread_flip:converge( + guild_thread_flip:reinstall( + ThreadExtras, State, State#{ + voice_states => ReloadVoiceStates, data => NormalizedNewData + } + ) + ) ), NewState1 = guild_availability:handle_unavailability_transition(State, NewState0), NewState2 = guild_sessions:refresh_all_viewable_channels(NewState1), diff --git a/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl b/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl index efa56bb43..a4788d9f1 100644 --- a/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl +++ b/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl @@ -55,6 +55,11 @@ await_collection(Ref, PendingCollections, Acc, DeadlineMs, RemainingMs) -> {Ref, Collection, {ok, Data}} -> NewPending = lists:delete(Collection, PendingCollections), collect_results(Ref, NewPending, Acc#{Collection => Data}, DeadlineMs); + {Ref, Collection, {ok, Data, Extras}} when is_map(Extras) -> + NewPending = lists:delete(Collection, PendingCollections), + collect_results( + Ref, NewPending, maps:merge(Acc#{Collection => Data}, Extras), DeadlineMs + ); {Ref, Collection, {error, Reason}} -> {error, {guild_collection_fetch_failed, Collection, Reason}}; {'DOWN', _, process, _, _} -> @@ -63,9 +68,17 @@ await_collection(Ref, PendingCollections, Acc, DeadlineMs, RemainingMs) -> {error, {guild_collection_fetch_timeout, PendingCollections}} end. --spec fetch_guild_collection(guild_id(), binary()) -> {ok, term()} | {error, term()}. +-spec fetch_guild_collection(guild_id(), binary()) -> + {ok, term()} | {ok, term(), map()} | {error, term()}. fetch_guild_collection(GuildId, <<"members">>) -> fetch_members_stream(GuildId, undefined, []); +fetch_guild_collection(GuildId, <<"channels">>) -> + RpcRequest = #{ + <<"type">> => <<"guild_collection">>, + <<"guild_id">> => type_conv:to_binary(GuildId), + <<"collection">> => <<"channels">> + }, + parse_channels_response(rpc_client:call(RpcRequest)); fetch_guild_collection(GuildId, Collection) -> RpcRequest = #{ <<"type">> => <<"guild_collection">>, @@ -83,6 +96,21 @@ parse_collection_response(Collection, {ok, Data}) when is_map(Data) -> parse_collection_response(_Collection, {error, Reason}) -> {error, Reason}. +-spec parse_channels_response(term()) -> + {ok, term()} | {ok, [map()], map()} | {error, term()}. +parse_channels_response({ok, #{<<"channels">> := Channels} = Data}) when + is_list(Channels) +-> + case maps:is_key(<<"thread_gate">>, Data) orelse maps:is_key(<<"thread_tainted">>, Data) of + true -> + {Merged, Extras} = guild_thread_load:channels_collection(Data), + {ok, Merged, Extras}; + false -> + {ok, Channels} + end; +parse_channels_response(Result) -> + parse_collection_response(<<"channels">>, Result). + -spec fetch_members_stream(guild_id(), binary() | undefined, [[map()]]) -> {ok, [map()]} | {error, term()}. fetch_members_stream(GuildId, AfterUserId, ChunksAcc) -> @@ -152,6 +180,27 @@ parse_members_result_invalid_cursor_test() -> parse_members_result(42, Members, true, null, []) ). +parse_channels_response_control_is_unchanged_test() -> + Channels = [#{<<"id">> => <<"1">>}], + ?assertEqual( + {ok, Channels}, parse_channels_response({ok, #{<<"channels">> => Channels}}) + ), + ?assertEqual( + {error, {invalid_collection_response, <<"channels">>}}, + parse_channels_response({ok, #{}}) + ). + +parse_channels_response_carries_the_thread_gate_test() -> + Response = #{ + <<"channels">> => [], + <<"thread_gate">> => #{<<"active">> => false, <<"config_version">> => 2}, + <<"thread_tainted">> => true + }, + ?assertEqual( + {ok, [], #{thread_gate => #{active => false, version => 2}, thread_tainted => true}}, + parse_channels_response({ok, Response}) + ). + maybe_put_after_test() -> BaseRequest = #{ <<"type">> => <<"guild_collection">>, diff --git a/fluxer_gateway/src/guild/guild_manager_shard_lifecycle.erl b/fluxer_gateway/src/guild/guild_manager_shard_lifecycle.erl index 8a3e9279c..30cc8a69f 100644 --- a/fluxer_gateway/src/guild/guild_manager_shard_lifecycle.erl +++ b/fluxer_gateway/src/guild/guild_manager_shard_lifecycle.erl @@ -88,7 +88,10 @@ start_new_guild_from_state(GuildId, GuildState, GuildKey, State) -> -spec normalize_transferred_guild_state(guild_id(), map()) -> map(). normalize_transferred_guild_state(GuildId, TransferState) -> - #{ + Threads = maps:with( + [thread_handoff_resends, thread_handoff_syncs, thread_handoff_lists], TransferState + ), + Threads#{ id => GuildId, data => maps:get(data, TransferState, #{}), sessions => maps:get(sessions, TransferState, #{}), diff --git a/fluxer_gateway/src/guild/guild_member_list_connected.erl b/fluxer_gateway/src/guild/guild_member_list_connected.erl index b626db9db..d28c3177a 100644 --- a/fluxer_gateway/src/guild/guild_member_list_connected.erl +++ b/fluxer_gateway/src/guild/guild_member_list_connected.erl @@ -202,6 +202,11 @@ session_can_view_channel(_SessionData, ChannelId, _State) when -> false; session_can_view_channel(SessionData, ChannelId, State) -> + session_can_view_channel_by_permissions(SessionData, ChannelId, State) andalso + guild_thread_gate:channel_visible(SessionData, ChannelId, State). + +-spec session_can_view_channel_by_permissions(map(), channel_id(), guild_state()) -> boolean(). +session_can_view_channel_by_permissions(SessionData, ChannelId, State) -> case { maps:get(user_id, SessionData, undefined), diff --git a/fluxer_gateway/src/guild/guild_members_mutation.erl b/fluxer_gateway/src/guild/guild_members_mutation.erl index d6cd9678c..4efa7713e 100644 --- a/fluxer_gateway/src/guild/guild_members_mutation.erl +++ b/fluxer_gateway/src/guild/guild_members_mutation.erl @@ -28,7 +28,7 @@ resolve_all_mentions(Request, State) -> UserIds = case MentionEveryone of true -> - resolve_mentions(Members, Context); + resolve_mentions(thread_universe_members(Members, Context), Context); false -> CandidateUserIds = candidate_user_ids(Context), resolve_mentions_for_ids(CandidateUserIds, MemberMap, Context) @@ -84,8 +84,9 @@ collect_direct_user_ids(DirectUserIds, AuthorId, ChannelId, State) -> -spec collect_user_mentions([user_id()], user_id(), channel_id(), guild_state()) -> [user_id()]. collect_user_mentions(UserIds, AuthorId, ChannelId, State) -> + ThreadRecipient = guild_thread_gate:recipient_active(ChannelId, State), guild_members_common:collect_mentions_for_user_ids( - UserIds, AuthorId, ChannelId, State, fun(_UserId, _Member) -> true end + UserIds, AuthorId, ChannelId, State, fun(UserId, _Member) -> ThreadRecipient(UserId) end ). -spec resolve_mention_sources_page(map(), guild_state()) -> guild_reply(map()). @@ -151,9 +152,45 @@ build_mention_context( connected_user_ids => guild_members_common:build_connected_user_ids( MentionHere, Sessions ), + thread_universe => guild_thread_push:mention_universe(ChannelId, State), state => State }. +-spec thread_universe_members([member()], map()) -> [member()]. +thread_universe_members(Members, #{thread_universe := Universe}) when is_map(Universe) -> + [ + Member + || Member <- Members, + maps:is_key(guild_members_common:member_user_id(Member), Universe) + ]; +thread_universe_members(Members, _Context) -> + Members. + +-spec thread_here_set(gb_sets:set(user_id()), map()) -> gb_sets:set(user_id()). +thread_here_set(HereSet, #{thread_universe := Universe}) when is_map(Universe) -> + gb_sets:filter(fun(UserId) -> maps:is_key(UserId, Universe) end, HereSet); +thread_here_set(HereSet, _Context) -> + HereSet. + +-spec thread_recipients([user_id()], map()) -> [user_id()]. +thread_recipients(UserIds, #{ + thread_universe := Universe, channel_id := ChannelId, state := State +}) when + is_map(Universe) +-> + Recipient = guild_thread_gate:recipient_active(ChannelId, State), + [UserId || UserId <- UserIds, Recipient(UserId)]; +thread_recipients(UserIds, _Context) -> + UserIds. + +-spec thread_recipient_set(gb_sets:set(user_id()), map()) -> gb_sets:set(user_id()). +thread_recipient_set(UserIdSet, #{thread_universe := Universe} = Context) when + is_map(Universe) +-> + gb_sets:from_list(thread_recipients(gb_sets:to_list(UserIdSet), Context)); +thread_recipient_set(UserIdSet, _Context) -> + UserIdSet. + -spec resolve_mentions([member()], map()) -> [user_id()]. resolve_mentions(Members, Context) -> #{author_id := AuthorId} = Context, @@ -227,7 +264,7 @@ candidate_user_ids(Context) -> } = Context, HereSet = case MentionHere of - true -> ConnectedUserIds; + true -> thread_here_set(ConnectedUserIds, Context); false -> gb_sets:empty() end, RoleUsersSet = @@ -235,18 +272,27 @@ candidate_user_ids(Context) -> true -> RoleIds = gb_sets:to_list(RoleIdSet), RoleUserIds = guild_members_roles:user_ids_for_any_role(RoleIds, State), - gb_sets:from_list(RoleUserIds); + gb_sets:from_list(thread_recipients(RoleUserIds, Context)); false -> gb_sets:empty() end, DirectSet = case HasDirectMentions of - true -> DirectUserIdSet; + true -> thread_recipient_set(DirectUserIdSet, Context); false -> gb_sets:empty() end, gb_sets:to_list(gb_sets:union(HereSet, gb_sets:union(RoleUsersSet, DirectSet))). -spec paged_candidates(map(), map()) -> [user_id()]. +paged_candidates( + #{mention_everyone := true, thread_universe := Universe} = Context, MemberMap +) when + is_map(Universe) +-> + lists:usort( + [UserId || UserId <- maps:keys(Universe), maps:is_key(UserId, MemberMap)] ++ + candidate_user_ids(Context) + ); paged_candidates(#{mention_everyone := true}, MemberMap) -> lists:sort(maps:keys(MemberMap)); paged_candidates(Context, _MemberMap) -> @@ -302,6 +348,10 @@ maybe_channel_mention(ChannelId, ChannelIndex, GuildId, BasePerms) -> end. -spec maybe_channel_mention_for_channel(map(), role_id(), integer()) -> {true, map()} | false. +maybe_channel_mention_for_channel(#{<<"type">> := Type}, _GuildId, _BasePerms) when + Type =:= 15; Type =:= 16 +-> + false; maybe_channel_mention_for_channel(Channel, GuildId, BasePerms) -> case everyone_can_view(Channel, GuildId, BasePerms) of true -> build_channel_mention(Channel); diff --git a/fluxer_gateway/src/guild/guild_mutual_online.erl b/fluxer_gateway/src/guild/guild_mutual_online.erl index ee50345f1..37c8ba8ab 100644 --- a/fluxer_gateway/src/guild/guild_mutual_online.erl +++ b/fluxer_gateway/src/guild/guild_mutual_online.erl @@ -60,7 +60,7 @@ count_mutually_visible_indexed(UserId, ViewerSet, State) -> {non_neg_integer(), memo() | undefined}. count_and_memo(UserId, ViewerSet, State) -> Tab = maps:get(member_presence, State), - Ctx = {viewer_channel_map(ViewerSet), build_viewable_index(State), State}, + Ctx = {viewer_channel_map(ViewerSet, State), build_viewable_index(State), State}, sets:fold( fun(OtherUserId, Acc) -> Presence = guild_state_member:lookup_presence(Tab, OtherUserId), @@ -109,9 +109,29 @@ ensure_memo(undefined, State) -> ensure_memo(Memo, _State) -> Memo. --spec viewer_channel_map(sets:set()) -> map(). -viewer_channel_map(ViewerSet) -> - sets:fold(fun(ChannelId, Acc) -> Acc#{ChannelId => true} end, #{}, ViewerSet). +-spec viewer_channel_map(sets:set(), guild_state()) -> map(). +viewer_channel_map(ViewerSet, State) -> + case guild_thread_gate:needs_variant(State) of + false -> + sets:fold(fun(ChannelId, Acc) -> Acc#{ChannelId => true} end, #{}, ViewerSet); + true -> + Index = guild_data_index:channel_index(maps:get(data, State, #{})), + sets:fold( + fun(ChannelId, Acc) -> put_shared(ChannelId, Index, Acc) end, #{}, ViewerSet + ) + end. + +-spec put_shared(term(), map(), map()) -> map(). +put_shared(ChannelId, Index, Acc) -> + case maps:get(ChannelId, Index, undefined) of + #{<<"type">> := Type} -> + case guild_thread_gate:is_thread_only_type(Type) of + true -> Acc; + false -> Acc#{ChannelId => true} + end; + _ -> + Acc#{ChannelId => true} + end. -spec build_viewable_index(guild_state()) -> viewable_index(). build_viewable_index(State) -> @@ -241,6 +261,31 @@ slow_path_counts_only_mutually_visible_members_test() -> Result = compute_count(10, State), ?assertEqual(2, Result). +forum_shared_only_with_a_member_is_not_counted_in_an_active_guild_test() -> + State = mutual_visibility_state(1, 5000), + Data = maps:get(data, State), + Forum = #{ + <<"id">> => <<"200">>, + <<"type">> => 15, + <<"permission_overwrites">> => [ + user_view_overwrite(<<"10">>), user_view_overwrite(<<"40">>) + ] + }, + Members = maps:get(<<"members">>, Data), + WithMember = Data#{ + <<"members">> => Members#{ + 40 => #{<<"user">> => #{<<"id">> => <<"40">>}, <<"roles">> => []} + } + }, + WithForum = WithMember#{<<"channels">> => [Forum | maps:get(<<"channels">>, Data)]}, + Gate = #{thread_gate => #{active => true, version => 1}}, + ?assertEqual(3, compute_count(10, State#{data => WithForum})), + ?assertEqual( + compute_count(10, State#{data => maps:merge(WithMember, Gate)}), + compute_count(10, State#{data => maps:merge(WithForum, Gate)}) + ), + ?assertEqual(2, compute_count(10, State#{data => maps:merge(WithForum, Gate)})). + mutual_visibility_state(GuildId, BotRoleId) -> #{ id => GuildId, diff --git a/fluxer_gateway/src/guild/guild_passive_sync.erl b/fluxer_gateway/src/guild/guild_passive_sync.erl index d380de1f6..08a4eb731 100644 --- a/fluxer_gateway/src/guild/guild_passive_sync.erl +++ b/fluxer_gateway/src/guild/guild_passive_sync.erl @@ -143,9 +143,28 @@ process_single_passive_session(SessionId, SessionData, GuildId, Channels, State) UserId = maps:get(user_id, SessionData), Member = guild_permissions:find_member_by_user_id(UserId, State), RegState = passive_sync_registry:lookup(SessionId, GuildId), - Diffs = compute_passive_diffs(Channels, UserId, Member, GuildId, State, RegState), + Visible = + session_channels(SessionData, Channels, State) ++ + guild_thread_push:passive_threads(SessionData, UserId, State), + Diffs = compute_passive_diffs(Visible, UserId, Member, GuildId, State, RegState), dispatch_passive_diffs(SessionId, GuildId, Pid, Diffs). +-spec session_channels(map(), [map()], guild_state()) -> [map()]. +session_channels(SessionData, Channels, State) -> + case + guild_thread_gate:needs_variant(State) andalso + not guild_thread_gate:session_viewer(SessionData) + of + true -> + [ + C + || C <- Channels, + not guild_thread_gate:is_thread_only_type(maps:get(<<"type">>, C, undefined)) + ]; + false -> + Channels + end. + -spec compute_passive_diffs( [map()], integer(), map() | undefined, integer(), guild_state(), map() ) -> map(). diff --git a/fluxer_gateway/src/guild/guild_permission_cache.erl b/fluxer_gateway/src/guild/guild_permission_cache.erl index 161de1a5e..e65898a26 100644 --- a/fluxer_gateway/src/guild/guild_permission_cache.erl +++ b/fluxer_gateway/src/guild/guild_permission_cache.erl @@ -211,14 +211,15 @@ strip_data(Data) when is_map(Data) -> ChannelIndex = strip_channel_index(maps:get(<<"channel_index">>, Data, #{})), RolePermsCache = maps:get(role_perms_cache, Data, #{}), OverwritePermsCache = maps:get(overwrite_perms_cache, Data, #{}), - with_member_source(Data, #{ + Base = maps:merge(maps:with([thread_gate, thread_store, thread_forum_categories], Data), #{ <<"guild">> => Guild, <<"roles">> => Roles, <<"channels">> => Channels, <<"channel_index">> => ChannelIndex, role_perms_cache => RolePermsCache, overwrite_perms_cache => OverwritePermsCache - }); + }), + with_member_source(Data, Base); strip_data(Data) -> Data. diff --git a/fluxer_gateway/src/guild/guild_permissions.erl b/fluxer_gateway/src/guild/guild_permissions.erl index c1591c326..92922eca7 100644 --- a/fluxer_gateway/src/guild/guild_permissions.erl +++ b/fluxer_gateway/src/guild/guild_permissions.erl @@ -55,9 +55,14 @@ get_member_permissions(UserId, ChannelId, State) -> -spec compute_member_permissions(user_id(), maybe_channel_id(), maybe_member(), guild_state()) -> permission(). compute_member_permissions(UserId, ChannelId, ProvidedMember, State) -> - channel_permissions( - member_base_permissions(UserId, ProvidedMember, State), UserId, ChannelId, State - ). + case thread_for_channel(ChannelId, State) of + undefined -> + overwrite_permissions( + member_base_permissions(UserId, ProvidedMember, State), UserId, ChannelId, State + ); + Thread -> + guild_thread_permissions:resolve(UserId, Thread, ProvidedMember, State) + end. -spec member_base_permissions(user_id(), maybe_member(), guild_state()) -> base_permissions(). member_base_permissions(UserId, ProvidedMember, State) when is_integer(UserId) -> @@ -72,13 +77,40 @@ member_base_permissions(_, _, _) -> -spec channel_permissions(base_permissions(), user_id(), maybe_channel_id(), guild_state()) -> permission(). -channel_permissions({Permissions, MemberRoles, GuildId}, UserId, ChannelId, State) -> +channel_permissions(Base, UserId, ChannelId, State) -> + case thread_for_channel(ChannelId, State) of + undefined -> + overwrite_permissions(Base, UserId, ChannelId, State); + Thread -> + guild_thread_permissions:resolve( + UserId, Thread, find_member_by_user_id(UserId, State), State + ) + end. + +-spec overwrite_permissions(base_permissions(), user_id(), maybe_channel_id(), guild_state()) -> + permission(). +overwrite_permissions({Permissions, MemberRoles, GuildId}, UserId, ChannelId, State) -> guild_permissions_overwrites:maybe_apply_channel_overwrites( Permissions, UserId, MemberRoles, ChannelId, GuildId, State ); -channel_permissions(Permissions, _UserId, _ChannelId, _State) -> +overwrite_permissions(Permissions, _UserId, _ChannelId, _State) -> Permissions. +-spec thread_for_channel(maybe_channel_id(), guild_state()) -> map() | undefined. +thread_for_channel(ChannelId, State) when is_integer(ChannelId) -> + case guild_thread_gate:store(State) of + undefined -> + undefined; + Tab -> + try + guild_thread_store:get_thread(Tab, ChannelId) + catch + error:badarg -> undefined + end + end; +thread_for_channel(_ChannelId, _State) -> + undefined. + -spec base_permissions_for_data(user_id(), maybe_member(), guild_state(), guild_data()) -> base_permissions(). base_permissions_for_data(UserId, ProvidedMember, State, Data) -> diff --git a/fluxer_gateway/src/guild/guild_permissions_overwrites.erl b/fluxer_gateway/src/guild/guild_permissions_overwrites.erl index e0cf06098..f90d311bb 100644 --- a/fluxer_gateway/src/guild/guild_permissions_overwrites.erl +++ b/fluxer_gateway/src/guild/guild_permissions_overwrites.erl @@ -84,10 +84,17 @@ maybe_apply_channel_overwrites( ) -> permission(). apply_from_channel_lookup(Permissions, UserId, MemberRoles, ChannelId, GuildId, State) -> case guild_permissions_check:find_channel_by_id(ChannelId, State) of - undefined -> Permissions; + undefined -> unknown_channel_permissions(Permissions, State); Channel -> apply_channel_overwrites(Permissions, UserId, MemberRoles, Channel, GuildId) end. +-spec unknown_channel_permissions(permission(), guild_state()) -> permission(). +unknown_channel_permissions(Permissions, State) -> + case guild_thread_gate:active(State) of + true -> 0; + false -> Permissions + end. + -spec overwrite_cache_from_data(map() | undefined) -> map(). overwrite_cache_from_data(D) when is_map(D) -> maps:get(overwrite_perms_cache, D, #{}); diff --git a/fluxer_gateway/src/guild/guild_presence.erl b/fluxer_gateway/src/guild/guild_presence.erl index 21574c381..a7a8827ae 100644 --- a/fluxer_gateway/src/guild/guild_presence.erl +++ b/fluxer_gateway/src/guild/guild_presence.erl @@ -100,8 +100,9 @@ process_presence_change(UserId, PresenceMap, PresenceMap, Status, _ListSync, Sta process_presence_change(UserId, OldPresence, PresenceMap, Status, ListSync, State) -> StateWithPresence = store_member_presence(UserId, PresenceMap, State), ok = guild_presence_sync:sync_online_status(UserId, StateWithPresence), + StateWithThreads = guild_thread_subscriptions:presence_changed(UserId, StateWithPresence), StateAfterBroadcast = spawn_presence_broadcast( - UserId, OldPresence, PresenceMap, State, StateWithPresence, ListSync + UserId, OldPresence, PresenceMap, State, StateWithThreads, ListSync ), StateAfterOffline = maybe_handle_offline(Status, UserId, StateAfterBroadcast), {noreply, StateAfterOffline}. diff --git a/fluxer_gateway/src/guild/guild_public_online.erl b/fluxer_gateway/src/guild/guild_public_online.erl index 0aa679727..48141cffb 100644 --- a/fluxer_gateway/src/guild/guild_public_online.erl +++ b/fluxer_gateway/src/guild/guild_public_online.erl @@ -28,7 +28,12 @@ compute_count(State) -> -spec compute_count_for_data(map(), guild_state()) -> non_neg_integer(). compute_count_for_data(Data, State) -> - Channels = [C || C <- ensure_list(maps:get(<<"channels">>, Data, [])), is_map(C)], + Channels = [ + C + || C <- ensure_list(maps:get(<<"channels">>, Data, [])), + is_map(C), + not guild_thread_gate:is_thread_only_type(maps:get(<<"type">>, C, undefined)) + ], case classify_everyone_viewable(Channels, State) of {open, _ChannelIds} -> guild_member_list:get_online_count(State); {restricted, ChannelIdSet} -> count_online_with_access(ChannelIdSet, State); diff --git a/fluxer_gateway/src/guild/guild_query_handler.erl b/fluxer_gateway/src/guild/guild_query_handler.erl index 19237228e..f4e546419 100644 --- a/fluxer_gateway/src/guild/guild_query_handler.erl +++ b/fluxer_gateway/src/guild/guild_query_handler.erl @@ -80,6 +80,7 @@ caller_timeout_ms(get_large_guild_metadata) -> 200; caller_timeout_ms(get_user_counts) -> 2000; caller_timeout_ms(get_viewer_counts) -> 2000; caller_timeout_ms(get_channel_member_counts) -> 2000; +caller_timeout_ms(get_forum_unread_scope) -> 2000; caller_timeout_ms(check_permission) -> 5000; caller_timeout_ms(get_guild_members_batch) -> 5000; caller_timeout_ms(list_guild_members) -> 10000; @@ -126,6 +127,9 @@ handle_query({get_channel_member_counts, Request}, _From, State) when is_map(Req handle_get_channel_member_counts(Request, State); handle_query({get_large_guild_metadata}, _From, State) -> handle_get_large_guild_metadata(State); +handle_query({get_forum_unread_scope, Request}, _From, State) when is_map(Request) -> + Session = request_session_data(Request, State), + {reply, guild_request_forum_unreads:scope(Session, Request, State), State}; handle_query(Msg, From, State) -> handle_call_dispatch(Msg, From, State). @@ -197,7 +201,7 @@ channel_member_count_entry(ChannelId, SessionData, State) when case guild_member_list_connected:session_can_view_channel_members( SessionData, ChannelId, State - ) + ) andalso not guild_thread_gate:is_thread_id(ChannelId, State) of true -> channel_member_count_entry_for_visible_channel(ChannelId, State); false -> false @@ -549,6 +553,16 @@ monotonic_deadline_ignores_the_legacy_wall_clock_test() -> ), ?assertNot(is_expired({get_data, #{deadline => 0}}, From)). +expired_forum_unread_scope_is_shed_test() -> + From = {self(), make_ref()}, + State = #{id => 1}, + Request = #{ + channel_id => 2, + thread_ids => [3], + deadline_monotonic => erlang:monotonic_time(millisecond) - 1 + }, + ?assertEqual({noreply, State}, handle_call({get_forum_unread_scope, Request}, From, State)). + deadline_less_queries_expire_once_every_known_caller_has_given_up_test() -> Self = self(), Ref = make_ref(), @@ -563,6 +577,7 @@ deadline_less_queries_expire_once_every_known_caller_has_given_up_test() -> fresh => Fresh, metadata => is_expired({get_large_guild_metadata}, From), counts => is_expired({get_user_counts, 1}, From), + forum_unreads => is_expired({get_forum_unread_scope, #{}}, From), unknown_tag => is_expired({get_sessions}, From), explicit_deadline => is_expired({get_large_guild_metadata, #{deadline_monotonic => Future}}, From) @@ -582,6 +597,7 @@ deadline_less_queries_expire_once_every_known_caller_has_given_up_test() -> fresh => false, metadata => true, counts => false, + forum_unreads => false, unknown_tag => false, explicit_deadline => false, disabled => false diff --git a/fluxer_gateway/src/guild/guild_read_model.erl b/fluxer_gateway/src/guild/guild_read_model.erl index 049044edf..9165f73e7 100644 --- a/fluxer_gateway/src/guild/guild_read_model.erl +++ b/fluxer_gateway/src/guild/guild_read_model.erl @@ -17,7 +17,11 @@ <<"stickers">>, role_perms_cache, overwrite_perms_cache, - members_ets + members_ets, + thread_gate, + thread_tainted, + thread_store, + thread_forum_categories ]). -spec put_state(map()) -> ok. diff --git a/fluxer_gateway/src/guild/guild_request_forum_unreads.erl b/fluxer_gateway/src/guild/guild_request_forum_unreads.erl new file mode 100644 index 000000000..d8e3174f4 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_request_forum_unreads.erl @@ -0,0 +1,408 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_request_forum_unreads). +-typing([eqwalizer]). + +-export([handle_request/3, scope/3]). + +-define(MAX_THREADS, 40). +-define(MAX_COUNT, 25). +-define(GUILD_CALL_TIMEOUT_MS, 2000). + +-type thread_request() :: {integer(), integer() | undefined}. +-type thread_scope() :: {stored, integer() | undefined} | unstored. + +-spec handle_request(map(), pid(), map()) -> ok. +handle_request(Data, SocketPid, SessionState) -> + UserId = snowflake_id:parse_maybe(maps:get(user_id, SessionState, undefined)), + case session_viewer(UserId, SessionState) of + false -> + SocketPid ! forum_unreads_unknown_opcode, + ok; + true -> + handle_viewer_request(Data, UserId, SessionState) + end. + +-spec session_viewer(integer() | undefined, map()) -> boolean(). +session_viewer(UserId, SessionState) -> + Bot = maps:get(bot, SessionState, false) =:= true, + Capable = maps:get(thread_channels_capable, SessionState, false) =:= true, + case channel_threads_config:loaded() of + false -> (Bot orelse Capable) andalso is_integer(UserId); + true -> guild_thread_gate:compute_viewer(true, Bot, Capable, UserId) + end. + +-spec handle_viewer_request(map(), integer() | undefined, map()) -> ok. +handle_viewer_request(Data, UserId, SessionState) -> + GuildId = snowflake_id:parse_maybe(maps:get(<<"guild_id">>, Data, undefined)), + ChannelId = snowflake_id:parse_maybe(maps:get(<<"channel_id">>, Data, undefined)), + SessionPid = maps:get(session_pid, SessionState, undefined), + SessionId = maps:get(session_id, SessionState, undefined), + case + {GuildId, ChannelId, parse_threads(maps:get(<<"threads">>, Data, undefined)), + SessionPid} + of + {G, C, [_ | _] = Threads, Pid} when + is_integer(G), is_integer(C), is_integer(UserId), is_pid(Pid), is_binary(SessionId) + -> + Request = #{ + session_id => SessionId, + user_id => UserId, + channel_id => C, + thread_ids => [ThreadId || {ThreadId, _Ack} <- Threads] + }, + case guild_scope(G, Request, maps:get(guilds, SessionState, #{})) of + {ok, Scopes} -> + Entries = resolve(G, C, UserId, Threads, Scopes), + dispatch(Pid, G, C, Entries); + denied -> + ok + end; + _ -> + ok + end. + +-spec parse_threads(term()) -> [thread_request()]. +parse_threads(Threads) when is_list(Threads) -> + take_threads(Threads, #{}, []); +parse_threads(_Threads) -> + []. + +-spec take_threads([term()], #{integer() => true}, [thread_request()]) -> [thread_request()]. +take_threads(_Rest, _Seen, Acc) when length(Acc) >= ?MAX_THREADS -> + lists:reverse(Acc); +take_threads([], _Seen, Acc) -> + lists:reverse(Acc); +take_threads([#{<<"thread_id">> := RawId} = Entry | Rest], Seen, Acc) -> + case snowflake_id:parse_maybe(RawId) of + ThreadId when is_integer(ThreadId), not is_map_key(ThreadId, Seen) -> + Ack = snowflake_id:parse_maybe(maps:get(<<"ack_message_id">>, Entry, undefined)), + take_threads(Rest, Seen#{ThreadId => true}, [{ThreadId, Ack} | Acc]); + _ -> + take_threads(Rest, Seen, Acc) + end; +take_threads([_ | Rest], Seen, Acc) -> + take_threads(Rest, Seen, Acc). + +-spec guild_scope(integer(), map(), map()) -> {ok, #{integer() => thread_scope()}} | denied. +guild_scope(GuildId, Request, Guilds) -> + case maps:get(GuildId, Guilds, undefined) of + {GuildPid, _Ref} when is_pid(GuildPid) -> + try + guild_query_handler:call( + GuildPid, {get_forum_unread_scope, Request}, ?GUILD_CALL_TIMEOUT_MS + ) + of + {ok, Scopes} when is_map(Scopes) -> {ok, Scopes}; + _ -> denied + catch + _:_ -> denied + end; + _ -> + denied + end. + +-spec scope(map() | undefined, map(), map()) -> {ok, #{integer() => thread_scope()}} | denied. +scope(undefined, _Request, _State) -> + denied; +scope(Session, #{channel_id := ForumId, thread_ids := ThreadIds}, State) when + is_integer(ForumId), is_list(ThreadIds) +-> + case + guild_thread_gate:session_viewer(Session) andalso is_forum(ForumId, State) andalso + guild_member_list_connected:session_can_view_channel(Session, ForumId, State) + of + true -> {ok, thread_scopes(ForumId, ThreadIds, State)}; + false -> denied + end; +scope(_Session, _Request, _State) -> + denied. + +-spec is_forum(integer(), map()) -> boolean(). +is_forum(ChannelId, State) -> + case + maps:get( + ChannelId, guild_data_index:channel_index(maps:get(data, State, #{})), undefined + ) + of + #{<<"type">> := Type} -> guild_thread_gate:is_thread_only_type(Type); + _ -> false + end. + +-spec thread_scopes(integer(), [term()], map()) -> #{integer() => thread_scope()}. +thread_scopes(ForumId, ThreadIds, State) -> + maps:from_list([ + {ThreadId, Scope} + || ThreadId <- ThreadIds, + is_integer(ThreadId), + Scope <- [thread_scope(ThreadId, ForumId, State)], + Scope =/= foreign + ]). + +-spec thread_scope(integer(), integer(), map()) -> thread_scope() | foreign. +thread_scope(ThreadId, ForumId, State) -> + case guild_thread_gate:thread(ThreadId, State) of + undefined -> + unstored; + Thread -> + case snowflake_id:parse_maybe(maps:get(<<"parent_id">>, Thread, undefined)) of + ForumId -> + {stored, + snowflake_id:parse_maybe( + maps:get(<<"last_message_id">>, Thread, undefined) + )}; + _ -> + foreign + end + end. + +-spec resolve(integer(), integer(), integer(), [thread_request()], #{ + integer() => thread_scope() +}) -> + [map()]. +resolve(GuildId, ChannelId, UserId, Threads, Scopes) -> + Local = [ + {ThreadId, local_entry(Ack, maps:get(ThreadId, Scopes))} + || {ThreadId, Ack} <- Threads, is_map_key(ThreadId, Scopes) + ], + Remote = remote_entries( + GuildId, ChannelId, UserId, [{ThreadId, Ack} || {ThreadId, {remote, Ack}} <- Local] + ), + lists:filtermap( + fun + ({ThreadId, {remote, _Ack}}) -> + case maps:get(ThreadId, Remote, undefined) of + undefined -> false; + Entry -> {true, Entry} + end; + ({ThreadId, Fields}) -> + {true, Fields#{<<"thread_id">> => integer_to_binary(ThreadId)}} + end, + Local + ). + +-spec local_entry(integer() | undefined, thread_scope()) -> + map() | {remote, integer() | undefined}. +local_entry(undefined, {stored, _LastMessageId}) -> + #{<<"missing">> => true}; +local_entry(Ack, {stored, LastMessageId}) when + is_integer(Ack), (LastMessageId =:= undefined orelse LastMessageId =< Ack) +-> + #{<<"count">> => 0}; +local_entry(Ack, _Scope) -> + {remote, Ack}. + +-spec remote_entries(integer(), integer(), integer(), [thread_request()]) -> + #{integer() => map()}. +remote_entries(_GuildId, _ChannelId, _UserId, []) -> + #{}; +remote_entries(GuildId, ChannelId, UserId, Threads) -> + Request = #{ + <<"type">> => <<"forum_unreads">>, + <<"guild_id">> => integer_to_binary(GuildId), + <<"channel_id">> => integer_to_binary(ChannelId), + <<"user_id">> => integer_to_binary(UserId), + <<"threads">> => [thread_rpc_entry(ThreadId, Ack) || {ThreadId, Ack} <- Threads] + }, + case rpc_client:call(Request) of + {ok, #{<<"threads">> := Entries}} when is_list(Entries) -> + Requested = maps:from_keys([ThreadId || {ThreadId, _Ack} <- Threads], true), + maps:from_list(lists:filtermap(fun(E) -> rpc_entry(E, Requested) end, Entries)); + _ -> + #{} + end. + +-spec thread_rpc_entry(integer(), integer() | undefined) -> map(). +thread_rpc_entry(ThreadId, undefined) -> + #{<<"thread_id">> => integer_to_binary(ThreadId)}; +thread_rpc_entry(ThreadId, Ack) -> + #{ + <<"thread_id">> => integer_to_binary(ThreadId), + <<"ack_message_id">> => integer_to_binary(Ack) + }. + +-spec rpc_entry(term(), #{integer() => true}) -> {true, {integer(), map()}} | false. +rpc_entry(#{<<"thread_id">> := RawId} = Entry, Requested) -> + case snowflake_id:parse_maybe(RawId) of + ThreadId when is_integer(ThreadId), is_map_key(ThreadId, Requested) -> + case rpc_fields(Entry) of + undefined -> + false; + Fields -> + {true, {ThreadId, Fields#{<<"thread_id">> => integer_to_binary(ThreadId)}}} + end; + _ -> + false + end; +rpc_entry(_Entry, _Requested) -> + false. + +-spec rpc_fields(map()) -> map() | undefined. +rpc_fields(#{<<"missing">> := true}) -> + #{<<"missing">> => true}; +rpc_fields(#{<<"count">> := Count}) when is_integer(Count), Count >= 0 -> + #{<<"count">> => min(Count, ?MAX_COUNT)}; +rpc_fields(_Entry) -> + undefined. + +-spec dispatch(pid(), integer(), integer(), [map()]) -> ok. +dispatch(SessionPid, GuildId, ChannelId, Entries) -> + gateway_dispatch_relay:dispatch(SessionPid, forum_unreads, #{ + <<"guild_id">> => integer_to_binary(GuildId), + <<"channel_id">> => integer_to_binary(ChannelId), + <<"threads">> => Entries + }). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +thread_entry(Id) -> + #{<<"thread_id">> => integer_to_binary(Id), <<"ack_message_id">> => <<"7">>}. + +parse_threads_dedupes_and_caps_at_forty_test() -> + Raw = [thread_entry(Id) || Id <- lists:seq(1, 60)], + Parsed = parse_threads([thread_entry(1), #{<<"thread_id">> => <<"x">>}, 5 | Raw]), + ?assertEqual(?MAX_THREADS, length(Parsed)), + ?assertEqual({1, 7}, hd(Parsed)), + ?assertEqual(lists:seq(1, 40), [Id || {Id, _} <- Parsed]), + ?assertEqual([{9, undefined}], parse_threads([#{<<"thread_id">> => 9}])), + ?assertEqual([], parse_threads(null)). + +local_entries_skip_the_rpc_when_the_store_answers_test() -> + ?assertEqual(#{<<"missing">> => true}, local_entry(undefined, {stored, 10})), + ?assertEqual(#{<<"count">> => 0}, local_entry(10, {stored, 10})), + ?assertEqual(#{<<"count">> => 0}, local_entry(10, {stored, undefined})), + ?assertEqual({remote, 10}, local_entry(10, {stored, 11})), + ?assertEqual({remote, undefined}, local_entry(undefined, unstored)). + +rpc_entries_are_clamped_and_limited_to_requested_threads_test() -> + Requested = #{1 => true}, + ?assertEqual( + {true, {1, #{<<"thread_id">> => <<"1">>, <<"count">> => 25}}}, + rpc_entry(#{<<"thread_id">> => <<"1">>, <<"count">> => 90}, Requested) + ), + ?assertEqual( + {true, {1, #{<<"thread_id">> => <<"1">>, <<"missing">> => true}}}, + rpc_entry(#{<<"thread_id">> => <<"1">>, <<"missing">> => true}, Requested) + ), + ?assertEqual(false, rpc_entry(#{<<"thread_id">> => <<"2">>, <<"count">> => 1}, Requested)), + ?assertEqual(false, rpc_entry(#{<<"thread_id">> => <<"1">>, <<"count">> => -1}, Requested)). + +non_viewer_sessions_get_the_unknown_opcode_close_test() -> + ok = handle_request(#{}, self(), #{user_id => <<"5">>, thread_channels_capable => false}), + receive + forum_unreads_unknown_opcode -> ok + after 100 -> ?assert(false) + end. + +with_forum_state(Fun) -> + Tab = guild_thread_store:new(), + try + guild_thread_store:load( + Tab, + [ + #{ + <<"id">> => 500, + <<"parent_id">> => 200, + <<"type">> => 11, + <<"last_message_id">> => 510 + }, + #{<<"id">> => 600, <<"parent_id">> => 300, <<"type">> => 11} + ], + [] + ), + Fun(#{ + data => #{ + thread_gate => #{active => true, version => 1}, + thread_store => Tab, + <<"channels">> => [ + #{<<"id">> => 200, <<"type">> => 15}, + #{<<"id">> => 300, <<"type">> => 0} + ] + } + }) + after + guild_thread_store:destroy(Tab) + end. + +viewer(ViewableChannels) -> + #{user_id => 1, thread_viewer => true, viewable_channels => ViewableChannels}. + +session_viewer_admits_capable_sessions_before_the_config_loads_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, channel_threads_config:default_config()), + try + ?assert(session_viewer(7, #{thread_channels_capable => true})), + ?assert(session_viewer(7, #{bot => true})), + ?assertNot(session_viewer(7, #{})), + ?assertNot(session_viewer(undefined, #{thread_channels_capable => true})) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +scope_requires_a_viewer_session_on_a_visible_forum_test() -> + with_forum_state(fun(State) -> + Request = #{channel_id => 200, thread_ids => [500, 600, 700]}, + ?assertEqual( + {ok, #{500 => {stored, 510}, 700 => unstored}}, + scope(viewer(#{200 => true}), Request, State) + ), + ?assertEqual(denied, scope(undefined, Request, State)), + ?assertEqual( + denied, scope((viewer(#{200 => true}))#{thread_viewer => false}, Request, State) + ), + ?assertEqual( + denied, + scope(viewer(#{300 => true}), Request#{channel_id => 300}, State) + ) + end). + +resolve_asks_the_api_only_for_threads_with_newer_messages_test() -> + Self = self(), + ok = meck:new(rpc_client, [passthrough, no_link]), + try + ok = meck:expect(rpc_client, call, fun(Request) -> + Self ! {rpc, Request}, + {ok, #{ + <<"threads">> => [ + #{<<"thread_id">> => <<"500">>, <<"count">> => 3}, + #{<<"thread_id">> => <<"700">>, <<"missing">> => true} + ] + }} + end), + Entries = resolve( + 1, + 200, + 9, + [{500, 505}, {501, 520}, {502, undefined}, {700, undefined}, {800, 1}], + #{500 => {stored, 510}, 501 => {stored, 510}, 502 => {stored, 1}, 700 => unstored} + ), + ?assertEqual( + [ + #{<<"thread_id">> => <<"500">>, <<"count">> => 3}, + #{<<"thread_id">> => <<"501">>, <<"count">> => 0}, + #{<<"thread_id">> => <<"502">>, <<"missing">> => true}, + #{<<"thread_id">> => <<"700">>, <<"missing">> => true} + ], + Entries + ), + receive + {rpc, #{<<"type">> := <<"forum_unreads">>, <<"threads">> := Threads}} -> + ?assertEqual( + [ + #{<<"thread_id">> => <<"500">>, <<"ack_message_id">> => <<"505">>}, + #{<<"thread_id">> => <<"700">>} + ], + Threads + ) + after 100 -> ?assert(false) + end + after + meck:unload(rpc_client) + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_sessions.erl b/fluxer_gateway/src/guild/guild_sessions.erl index fb449e15a..6c1f0a34a 100644 --- a/fluxer_gateway/src/guild/guild_sessions.erl +++ b/fluxer_gateway/src/guild/guild_sessions.erl @@ -42,7 +42,8 @@ -type session_pair() :: {session_id(), session_data()}. -type perm_memo() :: #{user_id() => non_neg_integer()}. -type view_memo() :: #{user_id() => boolean()}. --type message_ctx() :: {channel_id(), binary(), session_id() | undefined, guild_state()}. +-type message_ctx() :: + {channel_id(), binary(), session_id() | undefined, boolean(), guild_state()}. -export_type([ guild_state/0, session_id/0, @@ -174,9 +175,10 @@ handle_send_members_chunk(SessionId, ChunkData, State) -> sessions_map(), channel_id(), session_id() | undefined, guild_state() ) -> [session_pair()]. filter_sessions_for_channel(Sessions, ChannelId, SessionIdOpt, State) -> + Gated = guild_thread_gate:needs_variant(State), {Acc, _Memo} = maps:fold( fun(Sid, S, In) -> - collect_channel_session(Sid, S, ChannelId, SessionIdOpt, State, In) + collect_channel_session(Sid, S, ChannelId, SessionIdOpt, Gated, State, In) end, {[], #{}}, Sessions @@ -188,15 +190,16 @@ filter_sessions_for_channel(Sessions, ChannelId, SessionIdOpt, State) -> session_data(), channel_id(), session_id() | undefined, + boolean(), guild_state(), {[session_pair()], view_memo()} ) -> {[session_pair()], view_memo()}. -collect_channel_session(Sid, S, ChannelId, SessionIdOpt, State, {Acc, Memo}) -> +collect_channel_session(Sid, S, ChannelId, SessionIdOpt, Gated, State, {Acc, Memo}) -> case is_pending_or_excluded(Sid, S, SessionIdOpt) of true -> {Acc, Memo}; false -> - {Visible, Memo1} = memo_session_can_view_channel(S, ChannelId, State, Memo), + {Visible, Memo1} = memo_session_can_view_channel(Gated, S, ChannelId, State, Memo), {prepend_session(Visible, Sid, S, Acc), Memo1} end. @@ -210,7 +213,7 @@ filter_sessions_for_message(Sessions, ChannelId, MessageId, SessionIdOpt, State) sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state() ) -> [session_pair()]. filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State) -> - Ctx = {ChannelId, MessageId, SessionIdOpt, State}, + Ctx = {ChannelId, MessageId, SessionIdOpt, guild_thread_gate:needs_variant(State), State}, {Acc, _ViewMemo, _PermMemo} = maps:fold( fun(Sid, S, In) -> collect_message_session(Sid, S, Ctx, In) end, {[], #{}, #{}}, @@ -222,7 +225,7 @@ filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State) -> session_id(), session_data(), message_ctx(), {[session_pair()], view_memo(), perm_memo()} ) -> {[session_pair()], view_memo(), perm_memo()}. collect_message_session(Sid, S, Ctx, {Acc, ViewMemo, PermMemo}) -> - {ChannelId, _MessageId, SessionIdOpt, State} = Ctx, + {ChannelId, _MessageId, SessionIdOpt, Gated, State} = Ctx, case is_pending_or_excluded(Sid, S, SessionIdOpt) of true -> {Acc, ViewMemo, PermMemo}; @@ -232,7 +235,7 @@ collect_message_session(Sid, S, Ctx, {Acc, ViewMemo, PermMemo}) -> S, Ctx, Acc, - memo_session_can_view_channel(S, ChannelId, State, ViewMemo), + memo_session_can_view_channel(Gated, S, ChannelId, State, ViewMemo), PermMemo ) end. @@ -254,7 +257,7 @@ collect_visible_message_session(_Sid, _S, _Ctx, Acc, {false, ViewMemo}, PermMemo -spec memo_message_session( session_id(), session_data(), message_ctx(), [session_pair()], perm_memo() ) -> {[session_pair()], perm_memo()}. -memo_message_session(Sid, S, {ChannelId, MessageId, _SessionIdOpt, State}, Acc, Memo) -> +memo_message_session(Sid, S, {ChannelId, MessageId, _SessionIdOpt, _Gated, State}, Acc, Memo) -> case maps:get(user_id, S, undefined) of UserId when is_integer(UserId) -> {Perms, Memo1} = memo_member_permissions(UserId, ChannelId, State, Memo), @@ -418,9 +421,25 @@ refresh_session_viewable(SessionId, SessionData, AccState) -> AccState end. --spec memo_session_can_view_channel(session_data(), channel_id(), guild_state(), view_memo()) -> +-spec memo_session_can_view_channel( + boolean(), session_data(), channel_id(), guild_state(), view_memo() +) -> {boolean(), view_memo()}. -memo_session_can_view_channel(SessionData, ChannelId, State, Memo) -> +memo_session_can_view_channel(false, SessionData, ChannelId, State, Memo) -> + memo_session_can_view_channel_by_permissions(SessionData, ChannelId, State, Memo); +memo_session_can_view_channel(true, SessionData, ChannelId, State, Memo) -> + case guild_thread_gate:channel_visible(SessionData, ChannelId, State) of + true -> + memo_session_can_view_channel_by_permissions(SessionData, ChannelId, State, Memo); + false -> + {false, Memo} + end. + +-spec memo_session_can_view_channel_by_permissions( + session_data(), channel_id(), guild_state(), view_memo() +) -> + {boolean(), view_memo()}. +memo_session_can_view_channel_by_permissions(SessionData, ChannelId, State, Memo) -> UserId = maps:get(user_id, SessionData, undefined), case {UserId, maps:get(viewable_channels, SessionData, undefined)} of {Uid, ViewableChannels} when is_integer(Uid), is_map(ViewableChannels) -> @@ -535,6 +554,14 @@ reference_session_can_access_message(SessionData, ChannelId, MessageId, State) - -spec reference_session_can_view_channel(session_data(), channel_id(), guild_state()) -> boolean(). reference_session_can_view_channel(SessionData, ChannelId, State) -> + guild_thread_gate:channel_visible(SessionData, ChannelId, State) andalso + reference_session_can_view_channel_by_permissions(SessionData, ChannelId, State). + +-spec reference_session_can_view_channel_by_permissions( + session_data(), channel_id(), guild_state() +) -> + boolean(). +reference_session_can_view_channel_by_permissions(SessionData, ChannelId, State) -> UserId = maps:get(user_id, SessionData, undefined), case {UserId, maps:get(viewable_channels, SessionData, undefined)} of {Uid, ViewableChannels} when is_integer(Uid), is_map(ViewableChannels) -> @@ -572,11 +599,11 @@ memo_member_channel_access_miss_test() -> memo_session_can_view_channel_listed_skips_memo_test() -> Session = #{user_id => 1001, viewable_channels => #{10 => true}}, - ?assertEqual({true, #{}}, memo_session_can_view_channel(Session, 10, #{}, #{})). + ?assertEqual({true, #{}}, memo_session_can_view_channel(false, Session, 10, #{}, #{})). memo_session_can_view_channel_without_user_test() -> Session = #{viewable_channels => #{10 => true}}, - ?assertEqual({false, #{}}, memo_session_can_view_channel(Session, 10, #{}, #{})). + ?assertEqual({false, #{}}, memo_session_can_view_channel(false, Session, 10, #{}, #{})). channel_filters_match_per_session_reference_test() -> State = visibility_fixture_state(), diff --git a/fluxer_gateway/src/guild/guild_sessions_connect.erl b/fluxer_gateway/src/guild/guild_sessions_connect.erl index 2742969c2..295544ca0 100644 --- a/fluxer_gateway/src/guild/guild_sessions_connect.erl +++ b/fluxer_gateway/src/guild/guild_sessions_connect.erl @@ -42,7 +42,8 @@ handle_session_connect(Request, Pid, State) -> Sessions = maps:get(sessions, State, #{}), case maps:is_key(SessionId, Sessions) of true -> - {reply, {ok, guild_data:get_guild_state(UserId, State)}, State}; + Opts = guild_thread_gate:state_opts(maps:get(SessionId, Sessions)), + {reply, {ok, guild_data:get_guild_state(UserId, State, Opts)}, State}; false -> register_new_session(Request, Pid, UserId, SessionId, State) end. @@ -78,7 +79,10 @@ do_register_new_session(Request, Pid, UserId, SessionId, Sessions, State) -> reject_not_member -> {reply, {error, not_member}, State}; admit -> - GuildState = guild_data:get_guild_state(UserId, State), + ThreadFields = guild_thread_gate:session_fields(Request, UserId, State), + GuildState = guild_data:get_guild_state( + UserId, State, guild_thread_gate:state_opts(maps:merge(Request, ThreadFields)) + ), register_admitted_session( Request, Pid, UserId, SessionId, GuildId, GuildState, Sessions, State ) @@ -152,6 +156,9 @@ user_session_count_increment(UserId, SData) -> ) -> session_data(). build_session_data(Request, Pid, UserId, SessionId, State) -> UserRoles = session_passive:get_user_roles_for_guild(UserId, State), + #{thread_capable := Capable, thread_viewer := Viewer} = guild_thread_gate:session_fields( + Request, UserId, State + ), #{ session_id => SessionId, user_id => UserId, @@ -161,7 +168,9 @@ build_session_data(Request, Pid, UserId, SessionId, State) -> user_roles => UserRoles, bot => maps:get(bot, Request, false), is_staff => maps:get(is_staff, Request, false), - viewable_channels => get_viewable_channels_cached(UserId, UserRoles, State) + viewable_channels => get_viewable_channels_cached(UserId, UserRoles, State), + thread_capable => Capable, + thread_viewer => Viewer }. -spec store_initial_passive_state(session_id(), guild_id(), map()) -> ok. diff --git a/fluxer_gateway/src/guild/guild_sessions_passive.erl b/fluxer_gateway/src/guild/guild_sessions_passive.erl index a428a2638..7b4d83522 100644 --- a/fluxer_gateway/src/guild/guild_sessions_passive.erl +++ b/fluxer_gateway/src/guild/guild_sessions_passive.erl @@ -45,7 +45,9 @@ set_session_passive_guild(SessionId, GuildId, State) -> set_passive_for_session(SessionId, ValidGuildId, State) -> update_session(SessionId, State, fun(SD) -> NewSD = session_passive:set_passive(ValidGuildId, SD), - session_passive:clear_guild_synced(ValidGuildId, NewSD) + guild_thread_subscriptions:clear_on_passive( + session_passive:clear_guild_synced(ValidGuildId, NewSD) + ) end). -spec is_session_active(session_id(), guild_state()) -> boolean(). @@ -133,7 +135,9 @@ guild_id(State) -> dispatch_guild_sync(SessionId, SessionData, GuildId, Sessions, State) -> case {session_user_id(SessionData), maps:get(pid, SessionData, undefined)} of {UserId, SessionPid} when is_integer(UserId), UserId > 0, is_pid(SessionPid) -> - GuildData = guild_data:get_guild_state(UserId, State), + GuildData = guild_data:get_guild_state( + UserId, State, guild_thread_gate:state_opts(SessionData) + ), Encoded = {pre_encoded, iolist_to_binary( diff --git a/fluxer_gateway/src/guild/guild_state_channels.erl b/fluxer_gateway/src/guild/guild_state_channels.erl index 5215d2a1d..ac60252d3 100644 --- a/fluxer_gateway/src/guild/guild_state_channels.erl +++ b/fluxer_gateway/src/guild/guild_state_channels.erl @@ -55,6 +55,8 @@ handle_channel_delete(EventData, Data) -> guild_data_index:put_channels(FilteredChannels, Data). -spec handle_message_create(event_data(), guild_data()) -> guild_data(). +handle_message_create(#{<<"type">> := 18}, Data) -> + Data; handle_message_create(EventData, Data) -> ChannelId = snowflake_id:parse_optional( maps:get(<<"channel_id">>, EventData, undefined) diff --git a/fluxer_gateway/src/guild/guild_state_threads.erl b/fluxer_gateway/src/guild/guild_state_threads.erl new file mode 100644 index 000000000..a7828c563 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_state_threads.erl @@ -0,0 +1,526 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_state_threads). +-typing([eqwalizer]). + +-export([ + apply_event/3, + after_regular_event/3, + after_replayed_event/3, + thread_context/2 +]). + +-type guild_state() :: map(). + +-define(STORE_EVENT_KEYS, [ + <<"id">>, <<"channel_id">>, <<"type">>, <<"ids">>, <<"last_pin_timestamp">>, <<"user">> +]). + +-spec apply_event(atom(), map(), guild_state()) -> guild_state(). +apply_event(Event, Data, State) -> + case guild_thread_gate:store(State) of + undefined -> State; + Tab -> apply_store_event(Event, Data, Tab, State) + end. + +-spec apply_store_event(atom(), map(), ets:table(), guild_state()) -> guild_state(). +apply_store_event(thread_create, Data, Tab, State) -> + Thread = normalize_thread(Data), + case archived(Thread) of + true -> + State; + false -> + ok = guild_thread_store:put_thread(Tab, Thread), + ok = put_members(Tab, thread_members(Data), State), + bump_forum_parent(Thread, State) + end; +apply_store_event(thread_update, Data, Tab, State) -> + Thread = normalize_thread(Data), + ThreadId = maps:get(<<"id">>, Thread), + case archived(Thread) of + true -> + ok = guild_thread_store:remove_thread(Tab, ThreadId), + State; + false -> + Known = guild_thread_store:get_thread(Tab, ThreadId) =/= undefined, + ok = guild_thread_store:put_thread(Tab, Thread), + case Known of + true -> State; + false -> restore_members(Tab, Data, State) + end + end; +apply_store_event(thread_delete, Data, Tab, State) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)) of + ThreadId when is_integer(ThreadId) -> + ok = guild_thread_store:remove_thread(Tab, ThreadId); + _ -> + ok + end, + State; +apply_store_event(thread_member_update, Data, Tab, State) -> + Member = guild_data_normalize:thread_member(Data), + case member_key(Member) of + {ThreadId, UserId} -> + case guild_thread_store:get_thread(Tab, ThreadId) of + undefined -> ok; + _ -> guild_thread_store:put_member(Tab, ThreadId, UserId, Member) + end; + undefined -> + ok + end, + State; +apply_store_event(thread_members_update, Data, Tab, State) -> + ThreadId = snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)), + ok = ensure_thread(Tab, ThreadId, Data), + case + is_integer(ThreadId) andalso guild_thread_store:get_thread(Tab, ThreadId) =/= undefined + of + true -> + ok = put_members(Tab, list_field(<<"added_members">>, Data), State), + lists:foreach( + fun(UserId) -> guild_thread_store:remove_member(Tab, ThreadId, UserId) end, + parse_ids(maps:get(<<"removed_member_ids">>, Data, [])) + ), + ok = update_member_count(Tab, ThreadId, Data); + false -> + ok + end, + State; +apply_store_event(_Event, _Data, _Tab, State) -> + State. + +-spec ensure_thread(ets:table(), integer() | undefined, map()) -> ok. +ensure_thread(Tab, ThreadId, Data) when is_integer(ThreadId) -> + case + { + guild_thread_store:get_thread(Tab, ThreadId), + maps:get(<<"_fluxer_thread">>, Data, undefined) + } + of + {undefined, Context} when is_map(Context) -> + Thread = normalize_thread(Context), + case archived(Thread) of + true -> ok; + false -> guild_thread_store:put_thread(Tab, Thread) + end; + {Existing, Context} when is_map(Existing), is_map(Context) -> + guild_thread_store:put_thread(Tab, maps:merge(Existing, normalize_thread(Context))); + _ -> + ok + end; +ensure_thread(_Tab, _ThreadId, _Data) -> + ok. + +-spec update_member_count(ets:table(), integer(), map()) -> ok. +update_member_count(Tab, ThreadId, Data) -> + case + { + guild_thread_store:get_thread(Tab, ThreadId), + maps:get(<<"member_count">>, Data, undefined) + } + of + {Thread, Count} when is_map(Thread), is_integer(Count) -> + guild_thread_store:put_thread(Tab, Thread#{<<"member_count">> => Count}); + _ -> + ok + end. + +-spec restore_members(ets:table(), map(), guild_state()) -> guild_state(). +restore_members(Tab, Data, State) -> + ok = put_members(Tab, list_field(<<"_fluxer_members">>, Data), State), + State. + +-spec put_members(ets:table(), [map()], guild_state()) -> ok. +put_members(Tab, Members, _State) -> + lists:foreach( + fun(Raw) -> + Member = guild_data_normalize:thread_member(Raw), + case member_key(Member) of + {ThreadId, UserId} -> + case guild_thread_store:get_thread(Tab, ThreadId) of + undefined -> ok; + _ -> guild_thread_store:put_member(Tab, ThreadId, UserId, Member) + end; + undefined -> + ok + end + end, + Members + ). + +-spec member_key(term()) -> {integer(), integer()} | undefined. +member_key(#{<<"id">> := ThreadId, <<"user_id">> := UserId}) when + is_integer(ThreadId), is_integer(UserId) +-> + {ThreadId, UserId}; +member_key(_) -> + undefined. + +-spec thread_members(map()) -> [map()]. +thread_members(Data) -> + Members = list_field(<<"_fluxer_members">>, Data), + case maps:get(<<"member">>, Data, undefined) of + #{<<"user_id">> := _} = Own -> [Own#{<<"id">> => maps:get(<<"id">>, Data)} | Members]; + _ -> Members + end. + +-spec normalize_thread(map()) -> map(). +normalize_thread(Data) -> + Preview = maps:get(<<"_fluxer_member_ids_preview">>, Data, undefined), + WithPreview = + case Preview of + List when is_list(List) -> Data#{<<"member_ids_preview">> => List}; + _ -> Data + end, + case guild_data_normalize:thread(WithPreview) of + Thread when is_map(Thread) -> Thread; + _ -> WithPreview + end. + +-spec archived(map()) -> boolean(). +archived(#{<<"thread_metadata">> := #{<<"archived">> := true}}) -> true; +archived(_) -> false. + +-spec bump_forum_parent(map(), guild_state()) -> guild_state(). +bump_forum_parent(Thread, State) -> + Data = maps:get(data, State), + ParentId = guild_thread_permissions:parent_id(Thread), + Index = guild_data_index:channel_index(Data), + case maps:get(ParentId, Index, undefined) of + #{<<"type">> := Type} = Parent when Type =:= 15; Type =:= 16 -> + Updated = Parent#{<<"last_message_id">> => maps:get(<<"id">>, Thread)}, + State#{ + data => Data#{ + <<"channel_index">> => Index#{ParentId => Updated}, channels_stale => true + } + }; + _ -> + State + end. + +-spec after_regular_event(atom(), map(), guild_state()) -> guild_state(). +after_regular_event(Event, Data, State) -> + case guild_thread_gate:store(State) of + undefined -> State; + Tab -> after_store_event(Event, Data, Tab, queue_while_loading(Event, Data, State)) + end. + +-spec after_replayed_event(atom(), map(), guild_state()) -> guild_state(). +after_replayed_event(message_create, Data, State) -> + case guild_thread_gate:store(State) of + undefined -> + State; + Tab -> + MessageId = snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)), + with_thread(Data, Tab, fun(ThreadId, Thread) -> + case newer(MessageId, Thread) of + true -> count_message(ThreadId, Data, Thread); + false -> Thread + end + end), + State + end; +after_replayed_event(Event, Data, State) -> + after_regular_event(Event, Data, State). + +-spec queue_while_loading(atom(), map(), guild_state()) -> guild_state(). +queue_while_loading(Event, Data, State) -> + case + store_event(Event) andalso guild_thread_load:loading(State) andalso + not regular_channel_event(Event, Data, State) + of + true -> guild_thread_load:queue_event(Event, maps:with(?STORE_EVENT_KEYS, Data), State); + false -> State + end. + +-spec regular_channel_event(atom(), map(), guild_state()) -> boolean(). +regular_channel_event(Event, Data, State) when + Event =:= message_create; + Event =:= message_delete; + Event =:= message_delete_bulk; + Event =:= channel_pins_update +-> + case snowflake_id:parse_maybe(maps:get(<<"channel_id">>, Data, undefined)) of + ChannelId when is_integer(ChannelId) -> + maps:is_key(ChannelId, guild_data_index:channel_index(maps:get(data, State, #{}))); + _ -> + true + end; +regular_channel_event(_Event, _Data, _State) -> + false. + +-spec store_event(atom()) -> boolean(). +store_event(message_create) -> true; +store_event(message_delete) -> true; +store_event(message_delete_bulk) -> true; +store_event(channel_pins_update) -> true; +store_event(channel_delete) -> true; +store_event(guild_member_remove) -> true; +store_event(_) -> false. + +-spec after_store_event(atom(), map(), ets:table(), guild_state()) -> guild_state(). +after_store_event(message_create, Data, Tab, State) -> + with_thread(Data, Tab, fun(ThreadId, Thread) -> count_message(ThreadId, Data, Thread) end), + State; +after_store_event(message_delete, Data, Tab, State) -> + with_thread(Data, Tab, fun(ThreadId, Thread) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)) of + ThreadId -> Thread; + _ -> decrement(Thread, 1) + end + end), + State; +after_store_event(message_delete_bulk, Data, Tab, State) -> + with_thread(Data, Tab, fun(ThreadId, Thread) -> + Ids = parse_ids(maps:get(<<"ids">>, Data, [])), + decrement(Thread, length([Id || Id <- Ids, Id =/= ThreadId])) + end), + State; +after_store_event(channel_pins_update, Data, Tab, State) -> + with_thread(Data, Tab, fun(_ThreadId, Thread) -> + Thread#{<<"last_pin_timestamp">> => maps:get(<<"last_pin_timestamp">>, Data, null)} + end), + State; +after_store_event(channel_delete, Data, Tab, State) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)) of + ParentId when is_integer(ParentId) -> + lists:foreach( + fun(ThreadId) -> guild_thread_store:remove_thread(Tab, ThreadId) end, + guild_thread_store:parent_thread_ids(Tab, ParentId) + ); + _ -> + ok + end, + State; +after_store_event(guild_member_remove, Data, Tab, State) -> + _ = + case guild_state_member:extract_user_id(Data) of + UserId when is_integer(UserId) -> guild_thread_store:remove_user(Tab, UserId); + _ -> [] + end, + State; +after_store_event(_Event, _Data, _Tab, State) -> + State. + +-spec with_thread(map(), ets:table(), fun((integer(), map()) -> map())) -> ok. +with_thread(Data, Tab, Fun) -> + case snowflake_id:parse_maybe(maps:get(<<"channel_id">>, Data, undefined)) of + ChannelId when is_integer(ChannelId) -> + case guild_thread_store:get_thread(Tab, ChannelId) of + undefined -> ok; + Thread -> guild_thread_store:put_thread(Tab, Fun(ChannelId, Thread)) + end; + _ -> + ok + end. + +-spec count_message(integer(), map(), map()) -> map(). +count_message(ThreadId, Data, Thread) -> + MessageId = snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)), + case MessageId =:= ThreadId orelse maps:get(<<"type">>, Data, 0) =:= 21 of + true -> + Thread; + false -> + Last = + case newer(MessageId, Thread) of + true -> MessageId; + false -> maps:get(<<"last_message_id">>, Thread) + end, + Thread#{ + <<"last_message_id">> => Last, + <<"message_count">> => counter(Thread, <<"message_count">>) + 1, + <<"total_message_sent">> => counter(Thread, <<"total_message_sent">>) + 1 + } + end. + +-spec newer(term(), map()) -> boolean(). +newer(MessageId, Thread) when is_integer(MessageId) -> + case snowflake_id:parse_maybe(maps:get(<<"last_message_id">>, Thread, null)) of + LastId when is_integer(LastId) -> MessageId > LastId; + _ -> true + end; +newer(_MessageId, _Thread) -> + true. + +-spec decrement(map(), non_neg_integer()) -> map(). +decrement(Thread, N) -> + Thread#{<<"message_count">> => max(0, counter(Thread, <<"message_count">>) - N)}. + +-spec counter(map(), binary()) -> non_neg_integer(). +counter(Thread, Key) -> + case maps:get(Key, Thread, 0) of + N when is_integer(N), N >= 0 -> N; + _ -> 0 + end. + +-spec thread_context(integer(), map()) -> map() | undefined. +thread_context(ThreadId, Data) -> + case maps:get(<<"_fluxer_thread">>, Data, undefined) of + Context when is_map(Context) -> + Thread = normalize_thread(Context), + case maps:get(<<"id">>, Thread, undefined) of + ThreadId -> Thread; + _ -> undefined + end; + _ -> + undefined + end. + +-spec parse_ids(term()) -> [integer()]. +parse_ids(Ids) when is_list(Ids) -> + [Id || Raw <- Ids, Id <- [snowflake_id:parse_maybe(Raw)], is_integer(Id)]; +parse_ids(_) -> + []. + +-spec list_field(binary(), map()) -> [map()]. +list_field(Key, Map) -> + case maps:get(Key, Map, []) of + List when is_list(List) -> [Item || Item <- List, is_map(Item)]; + _ -> [] + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +state() -> + Tab = guild_thread_store:new(), + Data = guild_data_index:normalize_map(#{ + <<"guild">> => #{<<"id">> => <<"1">>, <<"owner_id">> => <<"2">>}, + <<"channels">> => [ + #{<<"id">> => <<"100">>, <<"type">> => 0}, + #{<<"id">> => <<"200">>, <<"type">> => 15} + ], + <<"roles">> => [], + <<"members">> => [] + }), + #{ + id => 1, + data => Data#{thread_store => Tab, thread_gate => #{active => true, version => 1}} + }. + +thread_payload(Id, Parent) -> + #{ + <<"id">> => integer_to_binary(Id), + <<"parent_id">> => integer_to_binary(Parent), + <<"guild_id">> => <<"1">>, + <<"type">> => 11, + <<"name">> => <<"t">>, + <<"message_count">> => 0, + <<"total_message_sent">> => 0, + <<"member_count">> => 1, + <<"thread_metadata">> => #{<<"archived">> => false}, + <<"_fluxer_member_ids_preview">> => [<<"5">>], + <<"_fluxer_members">> => [ + #{<<"id">> => integer_to_binary(Id), <<"user_id">> => <<"5">>, <<"flags">> => 1} + ] + }. + +counters_exclude_the_starter_and_type_21_test() -> + State0 = state(), + Tab = guild_thread_gate:store(State0), + State1 = apply_event(thread_create, thread_payload(300, 100), State0), + Msg = fun(Id, Type) -> + #{<<"id">> => integer_to_binary(Id), <<"channel_id">> => <<"300">>, <<"type">> => Type} + end, + State2 = after_regular_event(message_create, Msg(300, 21), State1), + State3 = after_regular_event(message_create, Msg(301, 0), State2), + State4 = after_regular_event(message_create, Msg(302, 0), State3), + Thread = guild_thread_store:get_thread(Tab, 300), + ?assertEqual(2, maps:get(<<"message_count">>, Thread)), + ?assertEqual(2, maps:get(<<"total_message_sent">>, Thread)), + ?assertEqual(302, maps:get(<<"last_message_id">>, Thread)), + ?assertEqual([5], maps:get(<<"member_ids_preview">>, Thread)), + State5 = after_regular_event(message_delete, Msg(300, 0), State4), + State6 = after_regular_event( + message_delete_bulk, + #{<<"channel_id">> => <<"300">>, <<"ids">> => [<<"300">>, <<"301">>, <<"302">>]}, + State5 + ), + Thread2 = guild_thread_store:get_thread(Tab, 300), + ?assertEqual(0, maps:get(<<"message_count">>, Thread2)), + ?assertEqual(2, maps:get(<<"total_message_sent">>, Thread2)), + _ = State6, + guild_thread_store:destroy(Tab). + +out_of_order_live_messages_still_count_test() -> + State0 = state(), + Tab = guild_thread_gate:store(State0), + State1 = apply_event(thread_create, thread_payload(300, 100), State0), + Msg = fun(Id) -> + #{<<"id">> => integer_to_binary(Id), <<"channel_id">> => <<"300">>, <<"type">> => 0} + end, + State2 = after_regular_event(message_create, Msg(302), State1), + State3 = after_regular_event(message_create, Msg(301), State2), + Thread = guild_thread_store:get_thread(Tab, 300), + ?assertEqual(2, maps:get(<<"message_count">>, Thread)), + ?assertEqual(2, maps:get(<<"total_message_sent">>, Thread)), + ?assertEqual(302, maps:get(<<"last_message_id">>, Thread)), + State4 = after_replayed_event(message_create, Msg(301), State3), + ?assertEqual(2, maps:get(<<"message_count">>, guild_thread_store:get_thread(Tab, 300))), + State5 = after_replayed_event(message_create, Msg(303), State4), + Thread2 = guild_thread_store:get_thread(Tab, 300), + ?assertEqual(3, maps:get(<<"message_count">>, Thread2)), + ?assertEqual(303, maps:get(<<"last_message_id">>, Thread2)), + _ = State5, + guild_thread_store:destroy(Tab). + +forum_parent_last_message_id_follows_new_posts_test() -> + State0 = state(), + State1 = apply_event(thread_create, thread_payload(300, 200), State0), + Index = guild_data_index:channel_index(maps:get(data, State1)), + ?assertEqual(300, maps:get(<<"last_message_id">>, maps:get(200, Index))), + guild_thread_store:destroy(guild_thread_gate:store(State0)). + +archive_removes_and_parent_delete_purges_test() -> + State0 = state(), + Tab = guild_thread_gate:store(State0), + State1 = apply_event(thread_create, thread_payload(300, 100), State0), + State2 = apply_event(thread_create, thread_payload(301, 100), State1), + Archived = (thread_payload(300, 100))#{<<"thread_metadata">> => #{<<"archived">> => true}}, + State3 = apply_event(thread_update, Archived, State2), + ?assertEqual(undefined, guild_thread_store:get_thread(Tab, 300)), + ?assertEqual([], guild_thread_store:user_thread_ids(Tab, 5) -- [301]), + _ = after_regular_event(channel_delete, #{<<"id">> => <<"100">>}, State3), + ?assertEqual(0, guild_thread_store:thread_count(Tab)), + guild_thread_store:destroy(Tab). + +unarchive_restores_members_from_the_event_only_test() -> + State0 = state(), + Tab = guild_thread_gate:store(State0), + Archived = (thread_payload(300, 100))#{<<"thread_metadata">> => #{<<"archived">> => true}}, + State1 = apply_event(thread_update, Archived, State0), + State2 = apply_event(thread_update, thread_payload(300, 100), State1), + ?assertEqual([5], guild_thread_store:member_ids(Tab, 300)), + State3 = apply_event(thread_update, Archived, State2), + _ = apply_event( + thread_update, maps:remove(<<"_fluxer_members">>, thread_payload(300, 100)), State3 + ), + ?assertNotEqual(undefined, guild_thread_store:get_thread(Tab, 300)), + ?assertEqual([], guild_thread_store:member_ids(Tab, 300)), + guild_thread_store:destroy(Tab). + +members_update_adds_and_removes_test() -> + State0 = state(), + Tab = guild_thread_gate:store(State0), + State1 = apply_event(thread_create, thread_payload(300, 100), State0), + _ = apply_event( + thread_members_update, + #{ + <<"id">> => <<"300">>, + <<"member_count">> => 2, + <<"added_members">> => [ + #{<<"id">> => <<"300">>, <<"user_id">> => <<"6">>, <<"flags">> => 0} + ], + <<"removed_member_ids">> => [<<"5">>] + }, + State1 + ), + ?assertEqual([6], guild_thread_store:member_ids(Tab, 300)), + ?assertEqual(2, maps:get(<<"member_count">>, guild_thread_store:get_thread(Tab, 300))), + _ = after_regular_event( + guild_member_remove, #{<<"user">> => #{<<"id">> => <<"6">>}}, State1 + ), + ?assertEqual([], guild_thread_store:member_ids(Tab, 300)), + guild_thread_store:destroy(Tab). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_subscription_handler.erl b/fluxer_gateway/src/guild/guild_subscription_handler.erl index 698be61ed..0da4de60d 100644 --- a/fluxer_gateway/src/guild/guild_subscription_handler.erl +++ b/fluxer_gateway/src/guild/guild_subscription_handler.erl @@ -272,7 +272,8 @@ process_lazy_subscribe_for_guild( ) andalso guild_permissions:can_view_channel_members( SessionUserId, ChannelId, undefined, State - ), + ) andalso + member_list_channel_allowed(SessionId, ChannelId, State), case CanView of true -> ListId = guild_member_list:calculate_list_id(ChannelId, State), @@ -283,6 +284,17 @@ process_lazy_subscribe_for_guild( State end. +-spec member_list_channel_allowed(session_id(), channel_id(), guild_state()) -> boolean(). +member_list_channel_allowed(SessionId, ChannelId, State) -> + case guild_thread_gate:needs_variant(State) of + false -> + true; + true -> + Session = maps:get(SessionId, maps:get(sessions, State, #{}), #{}), + not guild_thread_gate:is_thread_id(ChannelId, State) andalso + guild_thread_gate:channel_visible(Session, ChannelId, State) + end. + -spec subscribe_member_list_ranges( guild_member_list:list_id() | undefined, integer(), @@ -367,12 +379,12 @@ filter_user_ids(UserIds) -> -spec handle_update_member_subscriptions_local( integer(), session_id(), [user_id()], guild_state() ) -> guild_state(). -handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) -> +handle_update_member_subscriptions_local(_GuildId, SessionId, MemberIds, State) -> MemberSubs = maps:get(member_subscriptions, State, guild_subscriptions:init_state()), Sessions = maps:get(sessions, State, #{}), SessionUserId = get_session_user_id(SessionId, Sessions), FilteredMemberIds = filter_member_ids_for_subscription( - GuildId, SessionUserId, MemberIds, State + maps:get(SessionId, Sessions, #{}), SessionUserId, MemberIds, State ), {NewMemberSubs, Added, Removed} = guild_subscriptions:update_subscriptions_with_delta( SessionId, FilteredMemberIds, MemberSubs @@ -382,13 +394,15 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) - handle_removed_subscriptions(Removed, State2). -spec filter_member_ids_for_subscription( - integer(), user_id() | undefined, [user_id()], guild_state() + map(), user_id() | undefined, [user_id()], guild_state() ) -> [user_id()]. -filter_member_ids_for_subscription(_GuildId, undefined, _MemberIds, _State) -> +filter_member_ids_for_subscription(_Session, undefined, _MemberIds, _State) -> []; -filter_member_ids_for_subscription(_GuildId, SessionUserId, MemberIds, State) -> - guild_subscription_mutual_channels:filter_member_ids(SessionUserId, MemberIds, State). +filter_member_ids_for_subscription(Session, SessionUserId, MemberIds, State) -> + guild_subscription_mutual_channels:filter_member_ids( + Session, SessionUserId, MemberIds, State + ). -spec handle_added_subscriptions([user_id()], session_id(), guild_state()) -> guild_state(). handle_added_subscriptions(Added, SessionId, State) -> diff --git a/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl index 533a6c1cd..ab6180b48 100644 --- a/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl +++ b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl @@ -3,7 +3,7 @@ -module(guild_subscription_mutual_channels). -typing([eqwalizer]). --export([filter_member_ids/3, filter_session_member_ids/2, filter_session_member_ids/3]). +-export([filter_member_ids/4, filter_session_member_ids/2, filter_session_member_ids/3]). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -16,11 +16,13 @@ -type view_memo() :: #{exceptions => sets:set(user_id()), views => view_cache()}. -export_type([guild_state/0, user_id/0, view_memo/0]). --spec filter_member_ids(user_id(), [user_id()], guild_state()) -> [user_id()]. -filter_member_ids(_SessionUserId, [], _State) -> +-spec filter_member_ids(map(), user_id(), [user_id()], guild_state()) -> [user_id()]. +filter_member_ids(_Session, _SessionUserId, [], _State) -> []; -filter_member_ids(SessionUserId, MemberIds, State) -> - SessionMap = session_channel_map(SessionUserId, State), +filter_member_ids(Session, SessionUserId, MemberIds, State) -> + SessionMap = visible_session_map( + Session#{user_id => SessionUserId}, session_channel_map(SessionUserId, State), State + ), {Kept, _Memo} = lists:foldl( fun(MemberId, Acc) -> keep_member_with_mutual_channel(MemberId, SessionUserId, SessionMap, State, Acc) @@ -71,6 +73,36 @@ is_role_key(_Key) -> false. -spec request_session_map(term(), user_id(), map(), guild_state()) -> map(). request_session_map(SessionId, SessionUserId, Sessions, State) -> + Session = + case maps:get(SessionId, Sessions, undefined) of + Found when is_map(Found) -> Found; + _ -> #{} + end, + visible_session_map( + Session#{user_id => SessionUserId}, + raw_request_session_map(SessionId, SessionUserId, Sessions, State), + State + ). + +-spec visible_session_map(map(), map(), guild_state()) -> map(). +visible_session_map(Session, Map, State) -> + case + guild_thread_gate:needs_variant(State) andalso + not guild_thread_gate:session_viewer(Session) + of + true -> + maps:filter( + fun(ChannelId, _) -> + guild_thread_gate:channel_visible(Session, ChannelId, State) + end, + Map + ); + false -> + Map + end. + +-spec raw_request_session_map(term(), user_id(), map(), guild_state()) -> map(). +raw_request_session_map(SessionId, SessionUserId, Sessions, State) -> case maps:get(SessionId, Sessions, undefined) of #{user_id := SessionUserId, viewable_channels := Map} = SessionData when is_map(Map) -> case maps:get(pending_connect, SessionData, false) of @@ -350,10 +382,10 @@ exceptions_cover_owner_virtual_and_overwrite_targets_test() -> filter_member_ids_matches_unmemoised_test() -> State = test_state(), Ids = candidate_ids(), - ?assertEqual([20, 21, 7, 99, 4242, 20], filter_member_ids(10, Ids, State)), + ?assertEqual([20, 21, 7, 99, 4242, 20], filter_member_ids(#{}, 10, Ids, State)), ?assertEqual( [reference_filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]], - [filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]] + [filter_member_ids(#{}, Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]] ). memo_holds_one_entry_per_roles_term_test() -> @@ -376,7 +408,7 @@ exception_user_is_not_answered_from_its_role_set_test() -> non_member_candidate_is_dropped_test() -> State = test_state(), - ?assertEqual([], filter_member_ids(10, [777], State)), + ?assertEqual([], filter_member_ids(#{}, 10, [777], State)), ?assertEqual([], reference_filter_member_ids(10, [777], State)). filter_session_member_ids_matches_filter_member_ids_test() -> @@ -385,7 +417,10 @@ filter_session_member_ids_matches_filter_member_ids_test() -> Viewers = [10, 30, 7, 99], Requests = [{Viewer, Viewer, Ids} || Viewer <- Viewers], ?assertEqual( - maps:from_list([{Viewer, filter_member_ids(Viewer, Ids, State)} || Viewer <- Viewers]), + maps:from_list([ + {Viewer, filter_member_ids(#{}, Viewer, Ids, State)} + || Viewer <- Viewers + ]), filter_session_member_ids(Requests, State) ). diff --git a/fluxer_gateway/src/guild/guild_thread_dispatch.erl b/fluxer_gateway/src/guild/guild_thread_dispatch.erl new file mode 100644 index 000000000..1e2d9bfed --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_dispatch.erl @@ -0,0 +1,602 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_dispatch). +-typing([eqwalizer]). + +-include_lib("kernel/include/logger.hrl"). + +-export([ + handles/1, + handle/3, + dispatch_counts/0, + count_masked/2, + init_counters/0 +]). + +-define(RECENT_CREATE_CAP, 256). +-define(COUNTS_KEY, {?MODULE, variant_counts}). +-define(EVENTS, [ + thread_create, + thread_update, + thread_delete, + thread_member_update, + thread_members_update, + thread_list_sync, + thread_member_list_update, + forum_unreads +]). +-define(MASKED_EVENTS, [ + message_create, + message_update, + channel_create, + channel_update, + channel_update_bulk, + guild_role_create, + guild_role_update, + guild_role_update_bulk +]). + +-type guild_state() :: map(). +-type session_pair() :: {binary(), map()}. +-type delivery() :: {atom(), pid(), map()}. + +-spec handles(atom()) -> boolean(). +handles(thread_create) -> true; +handles(thread_update) -> true; +handles(thread_delete) -> true; +handles(thread_member_update) -> true; +handles(thread_members_update) -> true; +handles(thread_list_sync) -> true; +handles(thread_member_list_update) -> true; +handles(forum_unreads) -> true; +handles(_) -> false. + +-spec handle(atom(), map(), guild_state()) -> {noreply, guild_state()}. +handle(Event, EventData, #{id := GuildId} = State) when is_integer(GuildId) -> + Data = EventData#{<<"guild_id">> => integer_to_binary(GuildId)}, + try + State1 = + case guild_thread_load:loading(State) of + true -> guild_thread_load:queue_event(Event, Data, State); + false -> State + end, + {Deliveries, State2} = handle_event(Event, Data, State1), + ok = deliver(Deliveries, GuildId), + {noreply, after_delivery(Event, Data, State2)} + catch + Class:Reason:Stack -> + ?LOG_WARNING( + "guild_thread_dispatch_failed: event=~p guild_id=~p class=~p reason=~p stack=~p", + [ + Event, GuildId, Class, Reason, Stack + ] + ), + {noreply, State} + end; +handle(_Event, _EventData, State) -> + {noreply, State}. + +-spec after_delivery(atom(), map(), guild_state()) -> guild_state(). +after_delivery(thread_members_update, Data, State) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)) of + ThreadId when is_integer(ThreadId) -> + guild_thread_subscriptions:member_lists_changed(ThreadId, State); + _ -> + State + end; +after_delivery(_Event, _Data, State) -> + State. + +-spec handle_event(atom(), map(), guild_state()) -> {[delivery()], guild_state()}. +handle_event(thread_create, Data, State) -> + thread_create(Data, State); +handle_event(thread_update, Data, State) -> + thread_update(Data, State); +handle_event(thread_delete, Data, State) -> + thread_delete(Data, State); +handle_event(thread_member_update, Data, State) -> + thread_member_update(Data, State); +handle_event(thread_members_update, Data, State) -> + thread_members_update(Data, State); +handle_event(_Event, _Data, State) -> + {[], State}. + +-spec thread_create(map(), guild_state()) -> {[delivery()], guild_state()}. +thread_create(Data, State0) -> + ThreadId = snowflake_id:parse(maps:get(<<"id">>, Data)), + State = guild_state_threads:apply_event(thread_create, Data, State0), + Thread = thread_or_context(ThreadId, Data, State), + Initial = [ + UserId + || M <- list_field(<<"_fluxer_members">>, Data), + UserId <- [snowflake_id:parse_maybe(maps:get(<<"user_id">>, M, undefined))], + is_integer(UserId) + ], + InitialSet = maps:from_keys(Initial, true), + Base = public_payload(Data), + {Recipients, _Memo} = recipients( + Thread, + viewer_sessions(State), + State, + fun(Session, Passive) -> + is_map_key(maps:get(user_id, Session, undefined), InitialSet) orelse + (not Passive andalso interested(Session, ThreadId, State)) + end, + #{} + ), + Deliveries = [ + {thread_create, Pid, with_member(Base, ThreadId, UserId, State)} + || {_Sid, #{pid := Pid, user_id := UserId}} <- Recipients + ], + count(thread_create, length(Deliveries)), + {Deliveries, remember_create(ThreadId, Initial, State)}. + +-spec thread_update(map(), guild_state()) -> {[delivery()], guild_state()}. +thread_update(Data, State0) -> + ThreadId = snowflake_id:parse(maps:get(<<"id">>, Data)), + WasActive = guild_thread_gate:thread(ThreadId, State0) =/= undefined, + Viewers = viewer_sessions(State0), + Before = thread_or_context(ThreadId, Data, State0), + {BeforeRecipients, Memo} = recipients( + Before, Viewers, State0, joined_or_interested(ThreadId, State0), #{} + ), + State = guild_state_threads:apply_event(thread_update, Data, State0), + After = thread_or_context(ThreadId, Data, State), + {AfterRecipients, AfterMemo} = recipients( + After, + Viewers, + State, + joined_or_interested(ThreadId, State), + shared_memo(Before, After, Memo) + ), + Payload = public_payload(Data), + Deliveries = [ + {thread_update, Pid, Payload} + || {_Sid, #{pid := Pid}} <- union(BeforeRecipients, AfterRecipients) + ], + count(thread_update, length(Deliveries)), + MemberDeliveries = + case {WasActive, guild_thread_gate:thread(ThreadId, State)} of + {false, Restored} when is_map(Restored) -> + unarchive_member_updates(ThreadId, Restored, Viewers, State, AfterMemo); + _ -> + [] + end, + count(thread_member_update, length(MemberDeliveries)), + {Deliveries ++ MemberDeliveries, State}. + +-spec shared_memo(map() | undefined, map() | undefined, map()) -> map(). +shared_memo(Before, After, Memo) when is_map(Before), is_map(After) -> + ParentId = guild_thread_permissions:parent_id(Before), + case + guild_thread_permissions:parent_id(After) =:= ParentId andalso + maps:get(<<"type">>, Before, undefined) =/= 12 andalso + maps:get(<<"type">>, After, undefined) =/= 12 + of + true -> Memo; + false -> #{} + end; +shared_memo(_Before, _After, _Memo) -> + #{}. + +-spec unarchive_member_updates(integer(), map(), [session_pair()], guild_state(), map()) -> + [delivery()]. +unarchive_member_updates(ThreadId, Thread, Viewers, #{id := GuildId} = State, Memo) -> + case guild_thread_gate:store(State) of + undefined -> + []; + Tab -> + {Targets, _Memo} = recipients( + Thread, + Viewers, + State, + fun(Session, _Passive) -> is_member(Session, ThreadId, State) end, + Memo + ), + [ + {thread_member_update, Pid, Member#{ + <<"guild_id">> => integer_to_binary(GuildId) + }} + || {_Sid, #{pid := Pid, user_id := UserId}} <- Targets, + Member <- [guild_thread_store:get_member(Tab, ThreadId, UserId)], + is_map(Member) + ] + end. + +-spec thread_delete(map(), guild_state()) -> {[delivery()], guild_state()}. +thread_delete(Data, State0) -> + ThreadId = snowflake_id:parse(maps:get(<<"id">>, Data)), + Thread = thread_or_context(ThreadId, Data, State0), + Joined = joined_or_interested(ThreadId, State0), + Listed = maps:from_keys( + snowflake_id:parse_list(maps:get(<<"_fluxer_member_ids">>, Data, [])), true + ), + Wants = fun(Session, Passive) -> + Joined(Session, Passive) orelse + is_map_key(maps:get(user_id, Session, undefined), Listed) + end, + {Recipients, _Memo} = recipients(Thread, viewer_sessions(State0), State0, Wants, #{}), + Payload = maps:with([<<"id">>, <<"guild_id">>, <<"parent_id">>, <<"type">>], Data), + State = guild_state_threads:apply_event(thread_delete, Data, State0), + Deliveries = [{thread_delete, Pid, Payload} || {_Sid, #{pid := Pid}} <- Recipients], + count(thread_delete, length(Deliveries)), + {Deliveries, State}. + +-spec thread_member_update(map(), guild_state()) -> {[delivery()], guild_state()}. +thread_member_update(Data, State0) -> + State = guild_state_threads:apply_event(thread_member_update, Data, State0), + ThreadId = snowflake_id:parse_maybe(maps:get(<<"id">>, Data, undefined)), + TargetId = snowflake_id:parse_maybe(maps:get(<<"user_id">>, Data, undefined)), + Thread = member_update_thread(ThreadId, TargetId, Data, State), + Payload = guild_thread_gate:strip_internal(Data), + Deliveries = [ + {thread_member_update, Pid, Payload} + || Thread =/= undefined, + {_Sid, #{pid := Pid, user_id := UserId} = Session} <- viewer_sessions(State), + UserId =:= TargetId, + can_view(Session, Thread, State) + ], + count(thread_member_update, length(Deliveries)), + {Deliveries, State}. + +-spec member_update_thread(term(), term(), map(), guild_state()) -> map() | undefined. +member_update_thread(ThreadId, TargetId, Data, State) when + is_integer(ThreadId), is_integer(TargetId) +-> + case thread_or_context(ThreadId, Data, State) of + undefined -> parent_context(ThreadId, TargetId, Data); + Thread -> Thread + end; +member_update_thread(_ThreadId, _TargetId, _Data, _State) -> + undefined. + +-spec parent_context(integer(), integer(), map()) -> map() | undefined. +parent_context(ThreadId, TargetId, #{<<"_fluxer_parent_id">> := ParentId}) -> + #{ + <<"id">> => ThreadId, + <<"parent_id">> => ParentId, + <<"_fluxer_member_ids">> => [integer_to_binary(TargetId)] + }; +parent_context(_ThreadId, _TargetId, _Data) -> + undefined. + +-spec thread_members_update(map(), guild_state()) -> {[delivery()], guild_state()}. +thread_members_update(Data, State0) -> + ThreadId = snowflake_id:parse(maps:get(<<"id">>, Data)), + State = guild_state_threads:apply_event(thread_members_update, Data, State0), + Thread = thread_or_context(ThreadId, Data, State), + Added = maps:from_keys( + [ + UserId + || M <- list_field(<<"added_members">>, Data), + UserId <- [snowflake_id:parse_maybe(maps:get(<<"user_id">>, M, undefined))], + is_integer(UserId) + ], + true + ), + Removed = maps:from_keys( + [ + UserId + || Raw <- list_value(maps:get(<<"removed_member_ids">>, Data, [])), + UserId <- [snowflake_id:parse_maybe(Raw)], + is_integer(UserId) + ], + true + ), + {Suppressed, State1} = take_recent_create(ThreadId, State), + Viewers = viewer_sessions(State1), + {Synthesized, Memo} = synthesized_creates( + Thread, ThreadId, maps:without(Suppressed, Added), Viewers, State1 + ), + Payload = members_update_payload(Data, State1), + Affected = [ + Pair + || {_Sid, #{user_id := UserId}} = Pair <- Viewers, + is_map_key(UserId, Added) orelse is_map_key(UserId, Removed) + ], + {Interested, _Memo} = recipients( + Thread, + Viewers, + State1, + fun(Session, Passive) -> + not Passive andalso interested(Session, ThreadId, State1) + end, + Memo + ), + Deliveries = [ + {thread_members_update, Pid, Payload} + || {_Sid, #{pid := Pid}} <- union(Affected, Interested) + ], + count(thread_members_update, length(Deliveries)), + {Synthesized ++ Deliveries, State1}. + +-spec synthesized_creates( + map() | undefined, integer(), #{integer() => true}, [session_pair()], guild_state() +) -> + {[delivery()], map()}. +synthesized_creates(undefined, _ThreadId, _Users, _Viewers, _State) -> + {[], #{}}; +synthesized_creates(_Thread, _ThreadId, Users, _Viewers, _State) when map_size(Users) =:= 0 -> + {[], #{}}; +synthesized_creates(Thread, ThreadId, Users, Viewers, State) -> + Base = guild_thread_view:thread_payload(Thread), + {Targets, Memo} = recipients( + Thread, + Viewers, + State, + fun(Session, _Passive) -> is_map_key(maps:get(user_id, Session, undefined), Users) end, + #{} + ), + Deliveries = [ + {thread_create, Pid, with_member(Base, ThreadId, UserId, State)} + || {_Sid, #{pid := Pid, user_id := UserId}} <- Targets + ], + count(thread_create, length(Deliveries)), + {Deliveries, Memo}. + +-spec members_update_payload(map(), guild_state()) -> map(). +members_update_payload(Data, State) -> + Base = guild_thread_gate:strip_internal(Data), + case maps:get(<<"added_members">>, Base, undefined) of + Added when is_list(Added) -> + Ctx = guild_member_list_connected:presence_context(State), + Base#{ + <<"added_members">> => [decorate_added(M, Ctx, State) || M <- Added, is_map(M)] + }; + _ -> + Base + end. + +-spec decorate_added(map(), map(), guild_state()) -> map(). +decorate_added(Member, PresenceCtx, State) -> + Stripped = maps:without([<<"muted">>, <<"mute_config">>], Member), + case snowflake_id:parse_maybe(maps:get(<<"user_id">>, Member, undefined)) of + UserId when is_integer(UserId) -> + GuildMember = guild_data_members:find_member_by_user_id(UserId, State), + Stripped#{ + <<"member">> => null_if_undefined(GuildMember), + <<"presence">> => presence(UserId, PresenceCtx) + }; + _ -> + Stripped#{<<"member">> => null, <<"presence">> => null} + end. + +-spec presence(integer(), map()) -> map() | null. +presence(UserId, Ctx) -> + #{<<"presence">> := Presence} = guild_member_list_connected:add_presence_to_member( + #{}, UserId, Ctx + ), + case maps:get(<<"status">>, Presence, <<"offline">>) of + <<"offline">> -> null; + _ -> Presence#{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}} + end. + +-spec null_if_undefined(map() | undefined) -> map() | null. +null_if_undefined(undefined) -> null; +null_if_undefined(Value) -> Value. + +-spec recipients( + map() | undefined, + [session_pair()], + guild_state(), + fun((map(), boolean()) -> boolean()), + map() +) -> {[session_pair()], map()}. +recipients(undefined, _Viewers, _State, _Wants, Memo) -> + {[], Memo}; +recipients(Thread, Viewers, State, Wants, Memo0) -> + GuildId = maps:get(id, State), + Small = session_passive:is_small_guild(State), + {Pairs, Memo} = lists:foldl( + fun({_Sid, Session} = Pair, {Acc, Memo1}) -> + case + Wants(Session, (not Small) andalso session_passive:is_passive(GuildId, Session)) + of + true -> + case guild_thread_gate:thread_view_access(Session, Thread, State, Memo1) of + {true, Memo2} -> {[Pair | Acc], Memo2}; + {false, Memo2} -> {Acc, Memo2} + end; + false -> + {Acc, Memo1} + end + end, + {[], Memo0}, + Viewers + ), + {lists:reverse(Pairs), Memo}. + +-spec joined_or_interested(integer(), guild_state()) -> fun((map(), boolean()) -> boolean()). +joined_or_interested(ThreadId, State) -> + fun(Session, Passive) -> + is_member(Session, ThreadId, State) orelse + (not Passive andalso interested(Session, ThreadId, State)) + end. + +-spec interested(map(), integer(), guild_state()) -> boolean(). +interested(Session, ThreadId, State) -> + is_bot(Session) orelse is_member(Session, ThreadId, State) orelse + maps:get(thread_subscribed, Session, false) =:= true. + +-spec is_member(map(), integer(), guild_state()) -> boolean(). +is_member(#{user_id := UserId}, ThreadId, State) -> + case guild_thread_gate:store(State) of + undefined -> false; + Tab -> guild_thread_store:is_member(Tab, ThreadId, UserId) + end; +is_member(_Session, _ThreadId, _State) -> + false. + +-spec is_bot(map()) -> boolean(). +is_bot(Session) -> + maps:get(bot, Session, false) =:= true. + +-spec can_view(map(), map(), guild_state()) -> boolean(). +can_view(#{user_id := UserId}, Thread, State) when is_integer(UserId) -> + Member = guild_permissions:find_member_by_user_id(UserId, State), + Member =/= undefined andalso guild_thread_view:can_view(UserId, Thread, Member, State); +can_view(_Session, _Thread, _State) -> + false. + +-spec viewer_sessions(guild_state()) -> [session_pair()]. +viewer_sessions(State) -> + [ + {Sid, Session} + || {Sid, Session} <- maps:to_list(maps:get(sessions, State, #{})), + is_map(Session), + guild_thread_gate:session_viewer(Session), + maps:get(pending_connect, Session, false) =/= true, + is_pid(maps:get(pid, Session, undefined)) + ]. + +-spec union([session_pair()], [session_pair()]) -> [session_pair()]. +union(A, B) -> + maps:to_list(maps:merge(maps:from_list(A), maps:from_list(B))). + +-spec thread_or_context(integer(), map(), guild_state()) -> map() | undefined. +thread_or_context(ThreadId, Data, State) -> + case guild_thread_gate:thread(ThreadId, State) of + undefined -> + case guild_state_threads:thread_context(ThreadId, Data) of + undefined -> fallback_context(ThreadId, Data); + Context -> Context + end; + Thread -> + Thread + end. + +-spec fallback_context(integer(), map()) -> map() | undefined. +fallback_context(ThreadId, #{<<"parent_id">> := _, <<"type">> := _} = Data) -> + (maps:with([<<"parent_id">>, <<"type">>, <<"_fluxer_member_ids">>], Data))#{ + <<"id">> => ThreadId + }; +fallback_context(_ThreadId, _Data) -> + undefined. + +-spec public_payload(map()) -> map(). +public_payload(Data) -> + maps:without( + [<<"member">>, <<"member_ids_preview">>], guild_thread_gate:strip_internal(Data) + ). + +-spec with_member(map(), integer(), term(), guild_state()) -> map(). +with_member(Base, ThreadId, UserId, State) when is_integer(UserId) -> + case guild_thread_gate:store(State) of + undefined -> + Base; + Tab -> + case guild_thread_store:get_member(Tab, ThreadId, UserId) of + undefined -> Base; + Member -> Base#{<<"member">> => Member} + end + end; +with_member(Base, _ThreadId, _UserId, _State) -> + Base. + +-spec remember_create(integer(), [integer()], guild_state()) -> guild_state(). +remember_create(_ThreadId, [], State) -> + State; +remember_create(ThreadId, Initial, State) -> + Recent0 = maps:get(thread_recent_create, State, #{}), + Recent = + case map_size(Recent0) >= ?RECENT_CREATE_CAP of + true -> #{}; + false -> Recent0 + end, + State#{thread_recent_create => Recent#{ThreadId => Initial}}. + +-spec take_recent_create(integer(), guild_state()) -> {[integer()], guild_state()}. +take_recent_create(ThreadId, State) -> + case maps:get(thread_recent_create, State, #{}) of + #{ThreadId := Initial} = Recent -> + {Initial, State#{thread_recent_create => maps:remove(ThreadId, Recent)}}; + _ -> + {[], State} + end. + +-spec deliver([delivery()], integer()) -> ok. +deliver(Deliveries, GuildId) -> + Grouped = lists:foldl( + fun({Event, Pid, Payload}, Acc) -> + Key = {Event, Payload}, + Acc#{Key => [Pid | maps:get(Key, Acc, [])]} + end, + #{}, + Deliveries + ), + Ordered = lists:sort( + fun({{EventA, _}, _}, {{EventB, _}, _}) -> order(EventA) =< order(EventB) end, + maps:to_list(Grouped) + ), + lists:foreach( + fun({{Event, Payload}, Pids}) -> + Encoded = + {pre_encoded, + iolist_to_binary( + json:encode(guild_data_wire:payload(Payload), fun json:encode_value/2) + )}, + gateway_dispatch_relay:dispatch_many(lists:reverse(Pids), Event, Encoded, GuildId) + end, + Ordered + ). + +-spec order(atom()) -> non_neg_integer(). +order(thread_create) -> 0; +order(thread_update) -> 1; +order(thread_delete) -> 1; +order(thread_member_update) -> 2; +order(thread_members_update) -> 3; +order(_) -> 4. + +-spec list_field(binary(), map()) -> [map()]. +list_field(Key, Map) -> + [M || M <- list_value(maps:get(Key, Map, [])), is_map(M)]. + +-spec list_value(term()) -> list(). +list_value(List) when is_list(List) -> List; +list_value(_) -> []. + +-spec count(atom(), non_neg_integer()) -> ok. +count(Event, N) -> + count_key({Event, viewer}, N). + +-spec count_masked(atom(), non_neg_integer()) -> ok. +count_masked(Event, N) -> + count_key({Event, masked}, N). + +-spec count_keys() -> [{atom(), viewer | masked}]. +count_keys() -> + [{E, viewer} || E <- ?EVENTS] ++ [{E, masked} || E <- ?MASKED_EVENTS]. + +-spec count_key({atom(), viewer | masked}, non_neg_integer()) -> ok. +count_key(_Key, 0) -> + ok; +count_key(Key, N) -> + case key_index(Key, count_keys(), 1) of + undefined -> + ok; + Index -> + guild_thread_store:add_counter(?COUNTS_KEY, Index, N) + end. + +-spec init_counters() -> ok. +init_counters() -> + guild_thread_store:ensure_counter(?COUNTS_KEY, length(count_keys())). + +-spec key_index(term(), [term()], pos_integer()) -> pos_integer() | undefined. +key_index(Key, [Key | _], Index) -> Index; +key_index(Key, [_ | Rest], Index) -> key_index(Key, Rest, Index + 1); +key_index(_Key, [], _Index) -> undefined. + +-spec dispatch_counts() -> #{{atom(), viewer | masked} => non_neg_integer()}. +dispatch_counts() -> + Keys = count_keys(), + case persistent_term:get(?COUNTS_KEY, undefined) of + undefined -> + maps:from_list([{K, 0} || K <- Keys]); + Counters -> + maps:from_list([ + {K, counters:get(Counters, I)} + || {K, I} <- lists:zip(Keys, lists:seq(1, length(Keys))) + ]) + end. diff --git a/fluxer_gateway/src/guild/guild_thread_flip.erl b/fluxer_gateway/src/guild/guild_thread_flip.erl new file mode 100644 index 000000000..20dabd508 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_flip.erl @@ -0,0 +1,833 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_flip). +-typing([eqwalizer]). + +-export([ + start/2, + converge/1, + handle_result/3, + hold/3, + apply_flip/2, + reinstall/3, + user_flip/2, + resend/2, + export_handoff/1, + restore_handoff/1, + resend_count/0, + init_counters/0 +]). + +-define(RESEND_JITTER_MS, 10000). +-define(RESEND_COUNT_KEY, {?MODULE, resend}). +-define(RETRY_BACKOFF_MS, [30000, 120000, 600000]). +-define(HOLD_CAP, 10000). + +-type guild_state() :: map(). + +-spec start(non_neg_integer(), guild_state()) -> guild_state(). +start(Version, #{thread_flip := {_Ref, InFlight}} = State) when InFlight >= Version -> + State; +start(Version, #{id := GuildId} = State) when is_integer(GuildId) -> + case + adopted_version(State) >= Version orelse + (not is_map_key(thread_flip, State) andalso settled(GuildId, State)) + of + true -> + recompute_viewers(State); + false -> + Self = self(), + Ref = make_ref(), + _ = spawn(fun() -> + Self ! + {thread_flip_result, Ref, + guild_thread_load:fetch_flip_data(GuildId, Version)} + end), + State#{thread_flip => {Ref, Version}, thread_flip_held => {0, []}} + end; +start(_Version, State) -> + State. + +-spec settled(integer(), guild_state()) -> boolean(). +settled(GuildId, State) -> + guild_thread_gate:active(State) andalso channel_threads_config:guild_active(GuildId). + +-spec recompute_viewers(guild_state()) -> guild_state(). +recompute_viewers(State) -> + State1 = guild_thread_gate:recompute_session_viewers(State), + schedule_resends(changed_sessions(State, State1), State1). + +-spec converge(guild_state()) -> guild_state(). +converge(#{id := GuildId} = State) when is_integer(GuildId) -> + Loaded = channel_threads_config:enabled() orelse channel_threads_config:version() > 0, + _ = + case + Loaded andalso + channel_threads_config:guild_active(GuildId) =/= guild_thread_gate:active(State) + of + true -> self() ! {thread_gate_flip, channel_threads_config:version()}; + false -> ok + end, + State; +converge(State) -> + State. + +-spec adopted_version(guild_state()) -> integer(). +adopted_version(State) -> + case maps:get(data, State, #{}) of + #{thread_gate := #{active := true, version := Version}} -> Version; + _ -> -1 + end. + +-spec handle_result(reference(), term(), guild_state()) -> guild_state(). +handle_result(Ref, {ok, Data}, #{thread_flip := {Ref, Version}} = State) when is_map(Data) -> + case maps:get(thread_flip_held, State, {0, []}) of + {_Count, Held} -> + State0 = maps:without([thread_flip, thread_flip_held], State), + State1 = replay_held(Held, apply_flip(Data, Held, State0)), + case maps:get(<<"config_version">>, Data, Version) of + ApiVersion when is_integer(ApiVersion), ApiVersion < Version -> + retry(Version, State1); + _ -> + clear_retry(State1) + end; + overflow -> + logger:warning("guild_thread_flip_hold_overflow: guild_id=~p", [ + maps:get(id, State, undefined) + ]), + retry(Version, maps:without([thread_flip, thread_flip_held], State)) + end; +handle_result(Ref, Error, #{thread_flip := {Ref, Version}} = State) -> + logger:warning("guild_thread_flip_failed: guild_id=~p reason=~p", [ + maps:get(id, State, undefined), Error + ]), + retry(Version, maps:without([thread_flip, thread_flip_held], State)); +handle_result(_Ref, _Result, State) -> + State. + +-spec retry(non_neg_integer(), guild_state()) -> guild_state(). +retry(Version, #{id := GuildId} = State) when is_integer(GuildId) -> + case channel_threads_config:guild_active(GuildId) =:= guild_thread_gate:active(State) of + true -> + clear_retry(State); + false -> + Attempt = + case State of + #{thread_flip_retry := {_Timer, Previous}} -> Previous + 1; + _ -> 1 + end, + State1 = clear_retry(State), + Delay = lists:nth(min(Attempt, length(?RETRY_BACKOFF_MS)), ?RETRY_BACKOFF_MS), + Timer = erlang:send_after(Delay, self(), {thread_gate_flip, Version}), + State1#{thread_flip_retry => {Timer, Attempt}} + end; +retry(_Version, State) -> + State. + +-spec clear_retry(guild_state()) -> guild_state(). +clear_retry(#{thread_flip_retry := {Timer, _Attempt}} = State) -> + _ = erlang:cancel_timer(Timer), + maps:remove(thread_flip_retry, State); +clear_retry(State) -> + State. + +-spec hold(atom(), map(), guild_state()) -> guild_state(). +hold(Event, Data, #{thread_flip_held := {Count, Held}, id := GuildId} = State) when + is_integer(GuildId) +-> + case held_entry(Event, Data) of + undefined -> + State; + _Entry when Count >= ?HOLD_CAP -> + State#{thread_flip_held => overflow}; + Entry -> + Tagged = Entry#{<<"guild_id">> => integer_to_binary(GuildId)}, + State#{thread_flip_held => {Count + 1, [{Event, Tagged} | Held]}} + end; +hold(_Event, _Data, State) -> + State. + +-spec held_entry(atom(), map()) -> map() | undefined. +held_entry(Event, Data) when + Event =:= channel_create; Event =:= channel_update; Event =:= channel_delete +-> + case thread_only_channel(Data) of + true -> Data; + false -> undefined + end; +held_entry(channel_update_bulk, #{<<"channels">> := Channels} = Data) when is_list(Channels) -> + case [C || C <- Channels, thread_only_channel(C)] of + [] -> undefined; + Gated -> Data#{<<"channels">> => Gated} + end; +held_entry(Event, Data) -> + case guild_thread_dispatch:handles(Event) of + true -> Data; + false -> undefined + end. + +-spec thread_only_channel(term()) -> boolean(). +thread_only_channel(Channel) when is_map(Channel) -> + guild_thread_gate:is_thread_only_type(maps:get(<<"type">>, Channel, undefined)); +thread_only_channel(_Channel) -> + false. + +-spec replay_held([{atom(), map()}], guild_state()) -> guild_state(). +replay_held(Held, State) -> + lists:foldl( + fun({Event, Data}, Acc) -> + case guild_thread_dispatch:handles(Event) of + true -> replay_event(Event, Data, Acc); + false -> Acc + end + end, + State, + lists:reverse(Held) + ). + +-spec replay_event(atom(), map(), guild_state()) -> guild_state(). +replay_event(Event, Data, State) -> + try + guild_state_threads:apply_event(Event, Data, State) + catch + Class:Reason -> + logger:warning( + "guild_thread_flip_replay_failed: event=~p guild_id=~p class=~p reason=~p", [ + Event, maps:get(id, State, undefined), Class, Reason + ] + ), + State + end. + +-spec apply_flip(map(), guild_state()) -> guild_state(). +apply_flip(FlipData, OldState) -> + apply_flip(FlipData, [], OldState). + +-spec apply_flip(map(), [{atom(), map()}], guild_state()) -> guild_state(). +apply_flip(FlipData, Held, OldState) -> + Gate = guild_thread_load:parse_gate(maps:get(<<"thread_gate">>, FlipData, undefined)), + Extras0 = #{ + thread_gate => Gate, + thread_tainted => maps:get(<<"thread_tainted">>, FlipData, false) =:= true + }, + {Extras, Held1} = + case Gate of + #{active := true} -> + {Extras0#{thread_load => {ok, guild_thread_load:load_payload(FlipData)}}, Held}; + _ -> + {Extras0, []} + end, + OldData = maps:get(data, OldState), + Data = merge_channels(FlipData, Held1, OldData), + ChannelsChanged = thread_only_ids(OldData) =/= thread_only_ids(Data), + State = reinstall(Extras, OldState, OldState#{data => Data}, ChannelsChanged), + ok = guild_maintenance:maybe_put_permission_cache(State), + State. + +-spec merge_channels(map(), [{atom(), map()}], map()) -> map(). +merge_channels(FlipData, Held, Data) -> + case maps:get(<<"channels">>, FlipData, undefined) of + Channels when is_list(Channels) -> + Live = [C || C <- guild_data_index:channel_list(Data), not thread_only_channel(C)], + Gated = lists:foldl( + fun replay_channel/2, + [C || C <- Channels, thread_only_channel(C)], + lists:reverse(Held) + ), + guild_data_index:put_channels(Live ++ Gated, Data); + _ -> + Data + end. + +-spec thread_only_ids(map()) -> [integer() | undefined]. +thread_only_ids(Data) -> + lists:sort([ + snowflake_id:parse_optional(maps:get(<<"id">>, C, undefined)) + || C <- guild_data_index:channel_list(Data), thread_only_channel(C) + ]). + +-spec replay_channel({atom(), map()}, [map()]) -> [map()]. +replay_channel({channel_delete, Channel}, Channels) -> + without_channel(Channel, Channels); +replay_channel({channel_update_bulk, #{<<"channels">> := Bulk}}, Channels) -> + lists:foldl(fun(C, Acc) -> [C | without_channel(C, Acc)] end, Channels, Bulk); +replay_channel({Event, Channel}, Channels) when + Event =:= channel_create; Event =:= channel_update +-> + [Channel | without_channel(Channel, Channels)]; +replay_channel(_Held, Channels) -> + Channels. + +-spec without_channel(map(), [map()]) -> [map()]. +without_channel(Channel, Channels) -> + Id = snowflake_id:parse_optional(maps:get(<<"id">>, Channel, undefined)), + [C || C <- Channels, snowflake_id:parse_optional(maps:get(<<"id">>, C, undefined)) =/= Id]. + +-spec reinstall(map(), guild_state(), guild_state()) -> guild_state(). +reinstall(Extras, OldState, NewState0) -> + reinstall(Extras, OldState, NewState0, false). + +-spec reinstall(map(), guild_state(), guild_state(), boolean()) -> guild_state(). +reinstall(Extras0, OldState, NewState0, ChannelsChanged) -> + OldStore = guild_thread_gate:store(OldState), + Extras = keep_store(Extras0, OldStore, OldState), + NewState1 = release_store(Extras, OldStore, NewState0), + NewState2 = guild_thread_load:install(Extras, NewState1), + SameGate = gate_shape(OldState) =:= gate_shape(NewState2), + case changed_sessions(OldState, NewState2) of + [] when not ChannelsChanged andalso (OldStore =:= undefined orelse SameGate) -> + NewState2; + Changed -> + NewState3 = guild_sessions:refresh_all_viewable_channels(NewState2), + schedule_resends(Changed, NewState3) + end. + +-spec keep_store(map(), ets:table() | undefined, guild_state()) -> map(). +keep_store(#{thread_gate := #{active := true} = Gate} = Extras, Tab, OldState) when + Tab =/= undefined, not is_map_key(thread_rows, Extras), not is_map_key(thread_load, Extras) +-> + case + not guild_thread_load:loading(OldState) andalso + maps:get(thread_gate, maps:get(data, OldState), undefined) =:= Gate + of + true -> Extras#{thread_kept => Tab}; + false -> Extras + end; +keep_store(Extras, _Tab, _OldState) -> + Extras. + +-spec release_store(map(), ets:table() | undefined, guild_state()) -> guild_state(). +release_store(#{thread_kept := _}, _OldStore, #{data := Data} = State) -> + guild_thread_load:deactivate(State#{data => maps:remove(thread_store, Data)}); +release_store(_Extras, OldStore, State) -> + State1 = guild_thread_load:deactivate(State), + ok = guild_thread_store:retire(OldStore), + State1. + +-spec gate_shape(guild_state()) -> {boolean(), boolean()}. +gate_shape(State) -> + {guild_thread_gate:active(State), guild_thread_gate:needs_variant(State)}. + +-spec changed_sessions(guild_state(), guild_state()) -> [binary()]. +changed_sessions(OldState, NewState) -> + OldSessions = maps:get(sessions, OldState, #{}), + maps:fold( + fun(SessionId, Session, Acc) -> + Old = maps:get(SessionId, OldSessions, #{}), + case + guild_thread_gate:session_viewer(Old) =/= + guild_thread_gate:session_viewer(Session) + of + true -> [SessionId | Acc]; + false -> Acc + end + end, + [], + maps:get(sessions, NewState, #{}) + ). + +-spec user_flip(binary(), guild_state()) -> guild_state(). +user_flip(SessionId, State) -> + Sessions = maps:get(sessions, State, #{}), + case maps:get(SessionId, Sessions, undefined) of + Session when is_map(Session) -> + Viewer = guild_thread_gate:compute_viewer( + guild_thread_gate:active(State), + maps:get(bot, Session, false) =:= true, + maps:get(thread_capable, Session, false) =:= true, + maps:get(user_id, Session, undefined) + ), + case Viewer =:= guild_thread_gate:session_viewer(Session) of + true -> + State; + false -> + schedule_resends([SessionId], State#{ + sessions => Sessions#{ + SessionId => clear_subscriptions(Session#{thread_viewer => Viewer}) + } + }) + end; + _ -> + State + end. + +-spec clear_subscriptions(map()) -> map(). +clear_subscriptions(#{thread_viewer := true} = Session) -> + Session; +clear_subscriptions(Session) -> + maps:without([thread_subscribed, thread_member_lists], Session). + +-spec schedule_resends([binary()], guild_state()) -> guild_state(). +schedule_resends(SessionIds, State) -> + lists:foldl(fun schedule_resend/2, State, SessionIds). + +-spec schedule_resend(binary(), guild_state()) -> guild_state(). +schedule_resend(SessionId, State) -> + Pending = maps:get(thread_resend_pending, State, #{}), + case maps:is_key(SessionId, Pending) of + true -> + State; + false -> + _ = erlang:send_after( + rand:uniform(?RESEND_JITTER_MS) - 1, self(), {thread_flip_resend, SessionId} + ), + State#{thread_resend_pending => Pending#{SessionId => true}} + end. + +-spec export_handoff(guild_state()) -> map(). +export_handoff(#{thread_resend_pending := Pending}) -> + #{thread_handoff_resends => maps:keys(Pending)}; +export_handoff(_State) -> + #{}. + +-spec restore_handoff(guild_state()) -> guild_state(). +restore_handoff(#{thread_handoff_resends := SessionIds} = State) when is_list(SessionIds) -> + schedule_resends( + [Sid || Sid <- SessionIds, is_binary(Sid)], maps:remove(thread_handoff_resends, State) + ); +restore_handoff(State) -> + maps:remove(thread_handoff_resends, State). + +-spec resend(binary(), guild_state()) -> guild_state(). +resend(SessionId, #{id := GuildId} = State0) when is_integer(GuildId) -> + State = clear_pending_resend(SessionId, State0), + case maps:get(SessionId, maps:get(sessions, State, #{}), undefined) of + #{pending_connect := true} -> + State; + #{user_id := UserId} = Session -> + case guild_availability:is_guild_unavailable_for_user(UserId, State) of + true -> State; + false -> resend_guild_create(Session, GuildId, State) + end; + _ -> + State + end; +resend(SessionId, State) -> + clear_pending_resend(SessionId, State). + +-spec clear_pending_resend(binary(), guild_state()) -> guild_state(). +clear_pending_resend(SessionId, #{thread_resend_pending := Pending0} = State) -> + case maps:remove(SessionId, Pending0) of + Pending when map_size(Pending) =:= 0 -> maps:remove(thread_resend_pending, State); + Pending -> State#{thread_resend_pending => Pending} + end; +clear_pending_resend(_SessionId, State) -> + State. + +-spec resend_guild_create(map(), integer(), guild_state()) -> guild_state(). +resend_guild_create(Session, GuildId, State) -> + count_resend(), + ok = guild_availability:send_guild_create_to_session( + Session, GuildId, State, [], resend_health() + ), + State. + +-spec resend_health() -> map(). +resend_health() -> + case guild_health:is_degraded(self()) of + true -> #{<<"degraded">> => true}; + false -> #{} + end. + +-spec count_resend() -> ok. +count_resend() -> + guild_thread_store:add_counter(?RESEND_COUNT_KEY, 1, 1). + +-spec init_counters() -> ok. +init_counters() -> + guild_thread_store:ensure_counter(?RESEND_COUNT_KEY, 1). + +-spec resend_count() -> non_neg_integer(). +resend_count() -> + case persistent_term:get(?RESEND_COUNT_KEY, undefined) of + undefined -> 0; + Counter -> counters:get(Counter, 1) + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +start_ignores_a_repeat_flip_while_one_is_in_flight_test() -> + Parent = self(), + meck:new(guild_thread_load, [passthrough, no_link]), + meck:expect(guild_thread_load, fetch_flip_data, fun(_GuildId, Version) -> + Parent ! {fetch_started, Version}, + timer:sleep(1000), + {error, timeout} + end), + try + State0 = start(3, #{id => 5}), + State1 = start(3, State0), + State2 = start(2, State1), + ?assertEqual(maps:get(thread_flip, State0), maps:get(thread_flip, State2)), + #{thread_flip := {Superseded, 3}} = State2, + State3 = start(4, State2), + ?assertMatch(#{thread_flip := {_, 4}}, State3), + ?assertEqual([3, 4], collect_fetches([])), + ?assertEqual(State3, handle_result(Superseded, {ok, #{}}, State3)) + after + meck:unload(guild_thread_load) + end. + +settled_config_supersedes_an_in_flight_flip_test() -> + Parent = self(), + meck:new(guild_thread_load, [passthrough, no_link]), + meck:expect(guild_thread_load, fetch_flip_data, fun(_GuildId, Version) -> + Parent ! {fetch_started, Version}, + {error, timeout} + end), + meck:new(channel_threads_config, [passthrough, no_link]), + meck:expect(channel_threads_config, guild_active, fun(_GuildId) -> true end), + try + Stale = make_ref(), + State0 = #{ + id => 5, + data => #{thread_gate => #{active => true, version => 4}}, + sessions => #{}, + thread_flip => {Stale, 5}, + thread_flip_held => {0, []} + }, + State1 = start(6, State0), + ?assertMatch(#{thread_flip := {Ref, 6}} when Ref =/= Stale, State1), + ?assertEqual([6], collect_fetches([])), + Inactive = #{<<"thread_gate">> => #{<<"active">> => false, <<"config_version">> => 5}}, + ?assertEqual(State1, handle_result(Stale, {ok, Inactive}, State1)), + receive + {thread_flip_result, _, _} -> ok + after 1000 -> ok + end + after + meck:unload(channel_threads_config), + meck:unload(guild_thread_load) + end. + +failed_flip_rearms_while_local_config_disagrees_test() -> + meck:new(guild_thread_load, [passthrough, no_link]), + meck:expect(guild_thread_load, fetch_flip_data, fun(_GuildId, _Version) -> + {error, timeout} + end), + meck:new(channel_threads_config, [passthrough, no_link]), + meck:expect(channel_threads_config, guild_active, fun(_GuildId) -> true end), + try + State1 = run_failed_flip(3, #{id => 5, data => #{}}), + #{thread_flip_retry := {Timer1, 1}} = State1, + ?assert(erlang:read_timer(Timer1) > 0), + State2 = run_failed_flip(3, State1), + #{thread_flip_retry := {Timer2, 2}} = State2, + ?assertEqual(false, erlang:read_timer(Timer1)), + ?assert(erlang:read_timer(Timer2) > 30000), + meck:expect(channel_threads_config, guild_active, fun(_GuildId) -> false end), + State3 = run_failed_flip(3, State2), + ?assertNot(maps:is_key(thread_flip_retry, State3)), + ?assertEqual(false, erlang:read_timer(Timer2)) + after + meck:unload(channel_threads_config), + meck:unload(guild_thread_load) + end. + +run_failed_flip(Version, State) -> + #{thread_flip := {Ref, Version}} = State1 = start(Version, State), + receive + {thread_flip_result, Ref, Result} -> handle_result(Ref, Result, State1) + after 1000 -> error(no_flip_result) + end. + +start_skips_a_version_the_active_gate_already_adopted_test() -> + State = #{ + id => 5, data => #{thread_gate => #{active => true, version => 4}}, sessions => #{} + }, + ?assertEqual(State, start(4, State)), + ?assertEqual(State, start(3, State)). + +adopted_flip_recomputes_viewers_stored_before_the_config_loaded_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:erase(Key), + try + Gate = #{id => 5, data => #{thread_gate => #{active => true, version => 4}}}, + Request = #{thread_channels_capable => true}, + Session = maps:merge( + #{user_id => 7}, guild_thread_gate:session_fields(Request, 7, Gate) + ), + ?assertMatch(#{thread_viewer := false}, Session), + State = Gate#{sessions => #{<<"s">> => Session}}, + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, config_version => 4, included_users => #{<<"7">> => true} + }), + ok = seed_short_jitter(), + State1 = start(4, State), + ?assertMatch(#{sessions := #{<<"s">> := #{thread_viewer := true}}}, State1), + receive + {thread_flip_resend, <<"s">>} -> ok + after 1000 -> error(no_resend) + end, + ?assertEqual(State1, start(4, State1)), + receive + {thread_flip_resend, _} -> error(unexpected_resend) + after 1000 -> ok + end + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +activation_without_viewer_changes_refreshes_viewable_channels_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, config_version => 4 + }), + ok = guild_thread_store:init(), + Cache = ets:new(viewable_channels_cache, [set, public]), + ets:insert(Cache, {{}, #{200 => true}}), + Text = #{<<"id">> => <<"200">>, <<"type">> => 0, <<"permission_overwrites">> => []}, + Forum = #{<<"id">> => <<"300">>, <<"type">> => 15, <<"permission_overwrites">> => []}, + Data = guild_data_index:put_channels([Text], #{ + <<"guild">> => #{<<"owner_id">> => <<"999">>}, + <<"roles">> => [ + #{ + <<"id">> => <<"5">>, + <<"permissions">> => integer_to_binary(constants:view_channel_permission()) + } + ], + <<"members">> => #{ + 7 => #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []} + } + }), + Session = #{ + session_id => <<"s">>, + user_id => 7, + pid => self(), + thread_capable => true, + thread_viewer => false, + viewable_channels => #{200 => true} + }, + State0 = #{ + id => 5, + data => Data, + sessions => #{<<"s">> => Session}, + viewable_channels_cache => Cache, + disable_permission_cache_updates => true + }, + try + State1 = apply_flip( + #{ + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 4}, + <<"channels">> => [Text, Forum] + }, + State0 + ), + ?assertMatch(#{sessions := #{<<"s">> := #{thread_viewer := false}}}, State1), + ?assertEqual([], ets:tab2list(Cache)), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, config_version => 5, included_users => #{<<"7">> => true} + }), + State2 = user_flip(<<"s">>, State1), + ?assertMatch(#{sessions := #{<<"s">> := #{thread_viewer := true}}}, State2), + #{data := Data2, sessions := Sessions2} = State2, + Deleted = State2#{data => guild_data_index:put_channels([Text], Data2)}, + ?assertMatch( + [{<<"s">>, _}], + guild_sessions:filter_sessions_for_channel(Sessions2, 300, undefined, Deleted) + ) + after + ets:delete(Cache), + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +user_only_config_change_recomputes_an_active_guild_without_a_refetch_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + try + Session = #{user_id => 7, thread_capable => true, thread_viewer => true}, + State = #{ + id => 5, + data => #{thread_gate => #{active => true, version => 4}}, + sessions => #{<<"s">> => Session} + }, + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, + config_version => 6, + enabled_guilds => #{<<"5">> => true}, + included_users => #{<<"7">> => true}, + excluded_users => #{<<"7">> => true} + }), + State1 = start(6, State), + ?assertNot(maps:is_key(thread_flip, State1)), + ?assertMatch(#{sessions := #{<<"s">> := #{thread_viewer := false}}}, State1), + ?assertMatch(#{thread_resend_pending := #{<<"s">> := true}}, State1) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +seed_short_jitter() -> + Seed = hd([N || N <- lists:seq(1, 100000), short_jitter_seed(N)]), + _ = rand:seed(exsss, {Seed, Seed, Seed}), + ok. + +short_jitter_seed(N) -> + _ = rand:seed(exsss, {N, N, N}), + rand:uniform(?RESEND_JITTER_MS) < 200. + +hold_records_thread_events_only_while_a_flip_is_in_flight_test() -> + Idle = #{id => 5}, + ?assertEqual(Idle, hold(thread_create, #{}, Idle)), + InFlight = Idle#{thread_flip_held => {0, []}}, + ?assertEqual(InFlight, hold(message_create, #{}, InFlight)), + ?assertMatch( + #{thread_flip_held := {1, [{thread_create, #{<<"guild_id">> := <<"5">>}}]}}, + hold(thread_create, #{}, InFlight) + ). + +hold_keeps_forum_channel_events_and_flags_overflow_test() -> + InFlight = #{id => 5, thread_flip_held => {0, []}}, + Text = #{<<"id">> => <<"1">>, <<"type">> => 0}, + Forum = #{<<"id">> => <<"2">>, <<"type">> => 15}, + ?assertEqual(InFlight, hold(channel_create, Text, InFlight)), + ?assertEqual(InFlight, hold(guild_role_update, #{}, InFlight)), + ?assertMatch( + #{thread_flip_held := {1, [{channel_delete, #{<<"id">> := <<"2">>}}]}}, + hold(channel_delete, Forum, InFlight) + ), + #{thread_flip_held := {1, [{channel_update_bulk, Bulk}]}} = + hold(channel_update_bulk, #{<<"channels">> => [Text, Forum]}, InFlight), + ?assertEqual([Forum], maps:get(<<"channels">>, Bulk)), + Full = InFlight#{thread_flip_held => {?HOLD_CAP, []}}, + Overflow = hold(thread_create, #{}, Full), + ?assertMatch(#{thread_flip_held := overflow}, Overflow), + ?assertEqual(Overflow, hold(thread_create, #{}, Overflow)), + ?assertEqual(Full, hold(message_create, #{}, Full)). + +overflowed_flip_is_discarded_and_rearmed_test() -> + meck:new(channel_threads_config, [passthrough, no_link]), + meck:expect(channel_threads_config, guild_active, fun(_GuildId) -> true end), + try + Ref = make_ref(), + State0 = #{id => 5, data => #{}, thread_flip => {Ref, 3}, thread_flip_held => overflow}, + State1 = handle_result(Ref, {ok, #{<<"config_version">> => 3}}, State0), + ?assertNot(maps:is_key(thread_flip, State1)), + ?assertNot(maps:is_key(thread_flip_held, State1)), + ?assertEqual(#{}, maps:get(data, State1)), + #{thread_flip_retry := {Timer, 1}} = State1, + ?assert(erlang:read_timer(Timer) > 0), + _ = erlang:cancel_timer(Timer) + after + meck:unload(channel_threads_config) + end. + +replay_skips_a_malformed_held_event_and_keeps_going_test() -> + ok = guild_thread_store:init(), + Tab = guild_thread_store:new(), + try + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 9, <<"parent_id">> => 1}), + State = #{id => 5, data => #{thread_store => Tab, thread_gate => #{active => true}}}, + Held = [ + {thread_delete, #{<<"id">> => <<"9">>}}, + {thread_update, #{<<"name">> => <<"no-id">>}} + ], + ?assertEqual(State, replay_held(Held, State)), + ?assertEqual(undefined, guild_thread_store:get_thread(Tab, 9)) + after + guild_thread_store:destroy(Tab) + end. + +merge_channels_keeps_live_channels_and_replays_forum_events_test() -> + Channel = fun(Id, Type, Name) -> + #{<<"id">> => integer_to_binary(Id), <<"type">> => Type, <<"name">> => Name} + end, + Live = guild_data_index:put_channels( + [Channel(1, 0, <<"live">>), Channel(2, 15, <<"old-forum">>)], #{} + ), + Snapshot = #{ + <<"channels">> => [ + Channel(1, 0, <<"stale">>), + Channel(3, 0, <<"deleted-live">>), + Channel(2, 15, <<"forum">>), + Channel(4, 16, <<"media">>) + ] + }, + Held = [ + {channel_delete, Channel(4, 16, <<"media">>)}, + {channel_create, Channel(5, 15, <<"new-forum">>)}, + {channel_update, Channel(2, 15, <<"renamed-forum">>)} + ], + Merged = merge_channels(Snapshot, Held, Live), + Names = maps:from_list([ + {maps:get(<<"id">>, C), maps:get(<<"name">>, C)} + || C <- guild_data_index:channel_list(Merged) + ]), + ?assertEqual( + #{1 => <<"live">>, 2 => <<"renamed-forum">>, 5 => <<"new-forum">>}, + maps:fold(fun(K, V, Acc) -> Acc#{snowflake_id:parse_optional(K) => V} end, #{}, Names) + ). + +collect_fetches(Acc) -> + receive + {fetch_started, Version} -> collect_fetches([Version | Acc]) + after 200 -> lists:sort(Acc) + end. + +pending_resends_and_syncs_survive_a_handoff_test_() -> + {spawn, fun pending_resends_and_syncs_survive_a_handoff/0}. + +pending_resends_and_syncs_survive_a_handoff() -> + State0 = schedule_resends([<<"a">>, <<"b">>, <<"a">>], #{ + id => 5, + data => #{}, + sessions => #{}, + thread_sync_pending => #{<<"c">> => queued}, + thread_list_dirty => #{10 => true, 11 => true}, + thread_list_flush => true + }), + ?assertEqual(#{<<"a">> => true, <<"b">> => true}, maps:get(thread_resend_pending, State0)), + Exported = guild_manager_shard_lifecycle:normalize_transferred_guild_state( + 5, guild_handoff:export_handoff_state(State0) + ), + ?assertNot(maps:is_key(thread_resend_pending, Exported)), + ?assertNot(maps:is_key(thread_sync_pending, Exported)), + ?assertNot(maps:is_key(thread_list_dirty, Exported)), + ?assertNot(maps:is_key(thread_list_flush, Exported)), + Imported = guild_thread_subscriptions:restore_handoff(restore_handoff(Exported)), + ?assertEqual( + #{<<"a">> => true, <<"b">> => true}, maps:get(thread_resend_pending, Imported) + ), + ?assertEqual(#{<<"c">> => scheduled}, maps:get(thread_sync_pending, Imported)), + ?assertNot(maps:is_key(thread_handoff_resends, Imported)), + ?assertNot(maps:is_key(thread_handoff_syncs, Imported)), + ?assertNot(maps:is_key(thread_handoff_lists, Imported)), + ?assertMatch( + #{ + thread_list_dirty := #{10 := true, 11 := true}, + thread_list_refresh := true, + thread_list_flush := true + }, + Imported + ), + receive + thread_member_list_flush -> ok + after 1000 -> error(list_flush_not_armed) + end, + Loading = guild_thread_subscriptions:handle_list_flush(Imported#{ + thread_load => #{status => loading, queue => [], queued => 0} + }), + ?assertMatch( + #{thread_list_dirty := #{10 := true, 11 := true}, thread_list_refresh := true}, Loading + ), + ?assertNot(maps:is_key(thread_list_flush, Loading)), + ?assertMatch(#{thread_resend_pending := #{<<"b">> := true}}, resend(<<"a">>, Imported)), + ?assertNot(maps:is_key(thread_resend_pending, resend(<<"b">>, resend(<<"a">>, Imported)))), + ?assertEqual(#{id => 5, data => #{}}, restore_handoff(#{id => 5, data => #{}})), + Plain = guild_handoff:export_handoff_state(#{id => 5, data => #{}, sessions => #{}}), + ?assertNot(maps:is_key(thread_handoff_resends, Plain)), + ?assertNot(maps:is_key(thread_handoff_syncs, Plain)), + ?assertNot(maps:is_key(thread_handoff_lists, Plain)). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_gate.erl b/fluxer_gateway/src/guild/guild_thread_gate.erl new file mode 100644 index 000000000..cc1b51ae4 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_gate.erl @@ -0,0 +1,754 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_gate). +-typing([eqwalizer]). + +-export([ + active/1, + store/1, + compute_viewer/4, + session_viewer/1, + session_fields/3, + recompute_session_viewers/1, + state_opts/1, + is_thread_only_type/1, + thread/2, + is_thread_id/2, + channel_visible/3, + pre_update_filter/3, + event_scope/3, + viewer_sessions/1, + thread_sessions/6, + thread_view_access/4, + needs_variant/1, + mask_payload/2, + strip_channel_surface/1, + mask_role/1, + mask_overwrites/1, + strip_internal/1, + recipient_active/2 +]). + +-export_type([guild_state/0]). + +-type guild_state() :: map(). +-type session_pair() :: {binary(), map()}. +-type event_scope() :: open | viewers | {thread, integer(), map()}. +-type thread_ctx() :: {map(), integer() | undefined, binary() | undefined, guild_state()}. +-type access_memo() :: #{integer() => boolean()}. + +-define(THREAD_MESSAGE_FLAGS, 16#120). +-define(MAX_ACCESS_MEMO_ENTRIES, 8192). +-define(THREAD_PERMISSION_BITS, + (16#400000000 bor 16#800000000 bor 16#1000000000 bor 16#4000000000) +). +-define(SURFACE_KEYS, [ + <<"thread_metadata">>, + <<"applied_tags">>, + <<"available_tags">>, + <<"default_auto_archive_duration">>, + <<"default_thread_rate_limit_per_user">>, + <<"default_reaction_emoji">>, + <<"default_sort_order">>, + <<"default_forum_layout">>, + <<"default_tag_setting">>, + <<"message_count">>, + <<"total_message_sent">>, + <<"member_ids_preview">>, + <<"member_count">>, + <<"flags">> +]). + +-spec data(guild_state()) -> map(). +data(#{data := Data}) when is_map(Data) -> + Data; +data(Data) -> + Data. + +-spec active(guild_state()) -> boolean(). +active(State) -> + case data(State) of + #{thread_gate := #{active := true}} -> true; + _ -> false + end. + +-spec tainted(guild_state()) -> boolean(). +tainted(State) -> + case data(State) of + #{thread_tainted := true} -> true; + _ -> false + end. + +-spec store(guild_state()) -> ets:table() | undefined. +store(State) -> + case data(State) of + #{thread_store := Tab, thread_gate := #{active := true}} when Tab =/= undefined -> Tab; + _ -> undefined + end. + +-spec compute_viewer(boolean(), boolean(), boolean(), term()) -> boolean(). +compute_viewer(true, true, _Capable, UserId) when is_integer(UserId) -> + not channel_threads_config:user_excluded(UserId); +compute_viewer(true, false, true, UserId) when is_integer(UserId) -> + channel_threads_config:user_active(UserId); +compute_viewer(_Active, _Bot, _Capable, _UserId) -> + false. + +-spec session_viewer(map()) -> boolean(). +session_viewer(#{thread_viewer := true}) -> true; +session_viewer(_) -> false. + +-spec session_fields(map(), term(), guild_state()) -> map(). +session_fields(Request, UserId, State) -> + Capable = maps:get(thread_channels_capable, Request, false) =:= true, + Bot = maps:get(bot, Request, false) =:= true, + #{ + thread_capable => Capable, + thread_viewer => compute_viewer(active(State), Bot, Capable, UserId) + }. + +-spec recompute_session_viewers(guild_state()) -> guild_state(). +recompute_session_viewers(State) -> + Active = active(State), + Loaded = channel_threads_config:enabled() orelse channel_threads_config:version() > 0, + Sessions = maps:get(sessions, State, #{}), + State#{ + sessions => maps:map( + fun(_SessionId, Session) -> recompute_session_viewer(Active, Loaded, Session) end, + Sessions + ) + }. + +-spec recompute_session_viewer(boolean(), boolean(), term()) -> term(). +recompute_session_viewer(true, false, #{thread_viewer := _} = Session) -> + Session; +recompute_session_viewer(Active, _Loaded, Session) when is_map(Session) -> + Viewer = compute_viewer( + Active, + maps:get(bot, Session, false) =:= true, + maps:get(thread_capable, Session, false) =:= true, + maps:get(user_id, Session, undefined) + ), + case Viewer of + true -> + Session#{thread_viewer => true}; + false -> + maps:without([thread_subscribed, thread_member_lists], Session#{ + thread_viewer => false + }) + end; +recompute_session_viewer(_Active, _Loaded, Session) -> + Session. + +-spec state_opts(map()) -> map(). +state_opts(Session) -> + #{bot => maps:get(bot, Session, false) =:= true, thread_viewer => session_viewer(Session)}. + +-spec is_thread_type(term()) -> boolean(). +is_thread_type(10) -> true; +is_thread_type(11) -> true; +is_thread_type(12) -> true; +is_thread_type(_) -> false. + +-spec is_thread_only_type(term()) -> boolean(). +is_thread_only_type(15) -> true; +is_thread_only_type(16) -> true; +is_thread_only_type(_) -> false. + +-spec is_gated_channel_type(term()) -> boolean(). +is_gated_channel_type(Type) -> + is_thread_type(Type) orelse is_thread_only_type(Type). + +-spec channel_type(integer(), guild_state()) -> integer() | undefined. +channel_type(ChannelId, State) -> + case maps:get(ChannelId, guild_data_index:channel_index(data(State)), undefined) of + #{<<"type">> := Type} when is_integer(Type) -> Type; + _ -> undefined + end. + +-spec thread(integer(), guild_state()) -> map() | undefined. +thread(ThreadId, State) -> + case store(State) of + undefined -> undefined; + Tab -> safe_get_thread(Tab, ThreadId) + end. + +-spec safe_get_thread(ets:table(), integer()) -> map() | undefined. +safe_get_thread(Tab, ThreadId) -> + try + guild_thread_store:get_thread(Tab, ThreadId) + catch + error:badarg -> undefined + end. + +-spec is_thread_id(integer(), guild_state()) -> boolean(). +is_thread_id(ChannelId, State) -> + thread(ChannelId, State) =/= undefined. + +-spec is_gated_channel_id(integer(), guild_state()) -> boolean(). +is_gated_channel_id(ChannelId, State) -> + case active(State) orelse tainted(State) of + false -> + false; + true -> + case channel_type(ChannelId, State) of + undefined -> true; + Type -> is_gated_channel_type(Type) + end + end. + +-spec channel_visible(map(), term(), guild_state()) -> boolean(). +channel_visible(Session, ChannelId, State) when is_integer(ChannelId) -> + session_viewer(Session) orelse non_viewer_visible(Session, ChannelId, State); +channel_visible(_Session, _ChannelId, _State) -> + true. + +-spec non_viewer_visible(map(), integer(), guild_state()) -> boolean(). +non_viewer_visible(Session, ChannelId, State) -> + case active(State) orelse tainted(State) of + false -> + true; + true -> + case channel_type(ChannelId, State) of + 4 -> + category_visible(Session, ChannelId, State); + undefined -> + not is_thread_id(ChannelId, State); + Type -> + not is_gated_channel_type(Type) + end + end. + +-spec category_visible(map(), integer(), guild_state()) -> boolean(). +category_visible(Session, CategoryId, State) -> + case maps:get(thread_forum_categories, data(State), #{}) of + #{CategoryId := PlainChildren} -> + plain_category_visible(Session, CategoryId, PlainChildren, State); + _ -> + true + end. + +-spec plain_category_visible(map(), integer(), [integer()], guild_state()) -> boolean(). +plain_category_visible(#{user_id := UserId}, CategoryId, PlainChildren, State) when + is_integer(UserId) +-> + Member = guild_permissions:find_member_by_user_id(UserId, State), + Member =/= undefined andalso + (guild_permissions:can_view_channel_by_permissions(UserId, CategoryId, Member, State) orelse + lists:any( + fun(ChildId) -> + guild_permissions:can_view_channel(UserId, ChildId, Member, State) + end, + PlainChildren + )); +plain_category_visible(_Session, _CategoryId, _PlainChildren, _State) -> + false. + +-spec pre_update_filter(atom(), map(), guild_state()) -> pass | drop | {filtered, map()}. +pre_update_filter(Event, Data, State) -> + case active(State) of + true -> pass; + false -> inactive_filter(Event, Data) + end. + +-spec inactive_filter(atom(), map()) -> pass | drop | {filtered, map()}. +inactive_filter(Event, _Data) when + Event =:= thread_create; + Event =:= thread_update; + Event =:= thread_delete; + Event =:= thread_list_sync; + Event =:= thread_member_update; + Event =:= thread_members_update; + Event =:= thread_member_list_update; + Event =:= forum_unreads +-> + drop; +inactive_filter(Event, Data) when + Event =:= channel_create; Event =:= channel_update; Event =:= channel_delete +-> + case is_gated_channel_type(maps:get(<<"type">>, Data, undefined)) of + true -> drop; + false -> pass + end; +inactive_filter(channel_update_bulk, #{<<"channels">> := Channels} = Data) when + is_list(Channels) +-> + case + [ + C + || C <- Channels, + not (is_map(C) andalso is_gated_channel_type(maps:get(<<"type">>, C, undefined))) + ] + of + Channels -> pass; + [] -> drop; + Kept -> {filtered, Data#{<<"channels">> => Kept}} + end; +inactive_filter(message_create, #{<<"type">> := Type}) when Type =:= 18; Type =:= 21 -> + drop; +inactive_filter(_Event, _Data) -> + pass. + +-spec event_scope(atom(), map(), guild_state()) -> event_scope(). +event_scope(Event, Data, State) -> + case active(State) orelse tainted(State) of + false -> + open; + true -> + case viewer_only_event(Event, Data, State) of + true -> viewers; + false -> open; + {thread, _ThreadId, _Thread} = Thread -> Thread + end + end. + +-spec viewer_sessions(map()) -> map(). +viewer_sessions(Sessions) -> + maps:filter( + fun(_Sid, Session) -> is_map(Session) andalso session_viewer(Session) end, Sessions + ). + +-spec thread_sessions(atom(), map(), integer(), map(), binary() | undefined, guild_state()) -> + [session_pair()]. +thread_sessions(Event, Data, ThreadId, Thread, SessionIdOpt, State) -> + MessageId = + case guild_dispatch_filter:is_message_access_filtered_event(Event) of + true -> guild_dispatch_filter:extract_message_id(Data); + false -> undefined + end, + Ctx = {Thread, guild_thread_permissions:parent_id(Thread), MessageId, State}, + {Pairs, _Memo} = maps:fold( + fun(Sid, Session, {Acc, Memo}) -> + case + Sid =/= SessionIdOpt andalso is_map(Session) andalso + session_viewer(Session) andalso + maps:get(pending_connect, Session, false) =/= true andalso + thread_interested(Session, ThreadId, State) + of + true -> + {Ok, Memo1} = thread_access(Session, Ctx, Memo), + {prepend(Ok, Sid, Session, Acc), Memo1}; + false -> + {Acc, Memo} + end + end, + {[], #{}}, + maps:get(sessions, State, #{}) + ), + Pairs. + +-spec prepend(boolean(), binary(), map(), [session_pair()]) -> [session_pair()]. +prepend(true, Sid, Session, Acc) -> [{Sid, Session} | Acc]; +prepend(false, _Sid, _Session, Acc) -> Acc. + +-spec thread_interested(map(), integer(), guild_state()) -> boolean(). +thread_interested(Session, ThreadId, State) -> + maps:get(bot, Session, false) =:= true orelse + lists:member(ThreadId, maps:get(thread_member_lists, Session, [])) orelse + is_thread_member(maps:get(user_id, Session, undefined), ThreadId, State) orelse + (maps:get(thread_subscribed, Session, false) =:= true andalso + not session_passive:is_passive(maps:get(id, State, 0), Session)). + +-spec thread_view_access(map(), map(), guild_state(), access_memo()) -> + {boolean(), access_memo()}. +thread_view_access(Session, Thread, State, Memo) -> + thread_access( + Session, {Thread, guild_thread_permissions:parent_id(Thread), undefined, State}, Memo + ). + +-spec thread_access(map(), thread_ctx(), access_memo()) -> {boolean(), access_memo()}. +thread_access(#{user_id := UserId} = Session, Ctx, Memo) when is_integer(UserId) -> + case parent_viewable(Session, Ctx) of + true -> {true, Memo}; + false -> memo_thread_access(UserId, Ctx, Memo) + end; +thread_access(_Session, _Ctx, Memo) -> + {false, Memo}. + +-spec parent_viewable(map(), thread_ctx()) -> boolean(). +parent_viewable(#{viewable_channels := Viewable}, {Thread, ParentId, undefined, _State}) when + is_map(Viewable), is_integer(ParentId) +-> + is_map_key(ParentId, Viewable) andalso maps:get(<<"type">>, Thread, undefined) =/= 12; +parent_viewable(_Session, _Ctx) -> + false. + +-spec memo_thread_access(integer(), thread_ctx(), access_memo()) -> {boolean(), access_memo()}. +memo_thread_access(UserId, {Thread, _ParentId, MessageId, State}, Memo) -> + case Memo of + #{UserId := Ok} -> + {Ok, Memo}; + _ -> + Ok = resolve_thread_access(UserId, Thread, MessageId, State), + {Ok, store_access_memo(UserId, Ok, Memo)} + end. + +-spec store_access_memo(integer(), boolean(), access_memo()) -> access_memo(). +store_access_memo(UserId, Ok, Memo) when map_size(Memo) < ?MAX_ACCESS_MEMO_ENTRIES -> + Memo#{UserId => Ok}; +store_access_memo(_UserId, _Ok, Memo) -> + Memo. + +-spec resolve_thread_access(integer(), map(), binary() | undefined, guild_state()) -> boolean(). +resolve_thread_access(UserId, Thread, MessageId, State) -> + case guild_permissions:find_member_by_user_id(UserId, State) of + undefined -> + false; + Member -> + Perms = guild_thread_permissions:resolve(UserId, Thread, Member, State), + permission_bits:has(Perms, constants:view_channel_permission()) andalso + (MessageId =:= undefined orelse + guild_permissions:can_access_message_by_permissions( + Perms, MessageId, State + )) + end. + +-spec is_thread_member(term(), integer(), guild_state()) -> boolean(). +is_thread_member(UserId, ThreadId, State) when is_integer(UserId) -> + case store(State) of + undefined -> false; + Tab -> guild_thread_store:is_member(Tab, ThreadId, UserId) + end; +is_thread_member(_UserId, _ThreadId, _State) -> + false. + +-spec viewer_only_event(atom(), map(), guild_state()) -> + boolean() | {thread, integer(), map()}. +viewer_only_event(Event, Data, _State) when + Event =:= channel_create; Event =:= channel_update; Event =:= channel_delete +-> + is_gated_channel_type(maps:get(<<"type">>, Data, undefined)); +viewer_only_event(message_create, #{<<"type">> := Type} = Data, State) when + Type =:= 18; Type =:= 21 +-> + case channel_scoped_gated(Data, State) of + {thread, _ThreadId, _Thread} = Thread -> Thread; + _ -> true + end; +viewer_only_event(message_update, #{<<"__thread_only_update">> := true}, _State) -> + true; +viewer_only_event(guild_audit_log_entry_create, #{<<"__thread_scoped">> := _}, _State) -> + true; +viewer_only_event(Event, Data, State) -> + case guild_dispatch_filter:is_channel_scoped_event(Event) of + true -> channel_scoped_gated(Data, State); + false -> false + end. + +-spec channel_scoped_gated(map(), guild_state()) -> boolean() | {thread, integer(), map()}. +channel_scoped_gated(Data, State) -> + case snowflake_id:parse_optional(maps:get(<<"channel_id">>, Data, undefined)) of + ChannelId when is_integer(ChannelId) -> + case thread_or_context(ChannelId, Data, State) of + undefined -> is_gated_channel_id(ChannelId, State); + Thread -> {thread, ChannelId, Thread} + end; + _ -> + false + end. + +-spec thread_or_context(integer(), map(), guild_state()) -> map() | undefined. +thread_or_context(ChannelId, Data, State) -> + case thread(ChannelId, State) of + undefined -> guild_state_threads:thread_context(ChannelId, Data); + Thread -> Thread + end. + +-spec needs_variant(guild_state()) -> boolean(). +needs_variant(State) -> + active(State) orelse tainted(State). + +-spec mask_payload(atom(), map()) -> map(). +mask_payload(Event, Data) when + Event =:= message_create; Event =:= message_update +-> + mask_message(Data); +mask_payload(Event, Data) when + Event =:= channel_create; Event =:= channel_update +-> + mask_overwrites(strip_channel_surface(Data)); +mask_payload(channel_update_bulk, #{<<"channels">> := Channels} = Data) when + is_list(Channels) +-> + Data#{ + <<"channels">> => [ + mask_overwrites(strip_channel_surface(Channel)) + || Channel <- Channels, is_map(Channel) + ] + }; +mask_payload(Event, #{<<"role">> := Role} = Data) when + is_map(Role), (Event =:= guild_role_create orelse Event =:= guild_role_update) +-> + Data#{<<"role">> => mask_role(Role)}; +mask_payload(guild_role_update_bulk, #{<<"roles">> := Roles} = Data) when is_list(Roles) -> + Data#{<<"roles">> => [mask_role(Role) || Role <- Roles, is_map(Role)]}; +mask_payload(_Event, Data) -> + Data. + +-spec mask_message(map()) -> map(). +mask_message(Message) -> + Message1 = maps:remove(<<"thread">>, Message), + Message2 = + case maps:get(<<"flags">>, Message1, undefined) of + Flags when is_integer(Flags) -> + Message1#{<<"flags">> => Flags band bnot ?THREAD_MESSAGE_FLAGS}; + _ -> + Message1 + end, + Message3 = + case maps:get(<<"referenced_message">>, Message2, undefined) of + #{<<"type">> := 18} -> + Message2#{<<"referenced_message">> => null}; + Referenced when is_map(Referenced) -> + Message2#{<<"referenced_message">> => mask_message(Referenced)}; + _ -> + Message2 + end, + case maps:get(<<"message_snapshots">>, Message3, undefined) of + Snapshots when is_list(Snapshots) -> + Message3#{<<"message_snapshots">> => [mask_snapshot(S) || S <- Snapshots]}; + _ -> + Message3 + end. + +-spec mask_snapshot(term()) -> term(). +mask_snapshot(#{<<"message">> := Message} = Snapshot) when is_map(Message) -> + Snapshot#{<<"message">> => mask_message(Message)}; +mask_snapshot(Snapshot) when is_map(Snapshot) -> + mask_message(Snapshot); +mask_snapshot(Snapshot) -> + Snapshot. + +-spec strip_channel_surface(map()) -> map(). +strip_channel_surface(Channel) -> + maps:without(?SURFACE_KEYS, Channel). + +-spec mask_role(map()) -> map(). +mask_role(#{<<"permissions">> := Permissions} = Role) -> + Role#{<<"permissions">> => mask_bits(Permissions)}; +mask_role(Role) -> + Role. + +-spec mask_overwrites(map()) -> map(). +mask_overwrites(#{<<"permission_overwrites">> := Overwrites} = Channel) when + is_list(Overwrites) +-> + Channel#{<<"permission_overwrites">> => [mask_overwrite(O) || O <- Overwrites]}; +mask_overwrites(Channel) -> + Channel. + +-spec mask_overwrite(term()) -> term(). +mask_overwrite(Overwrite) when is_map(Overwrite) -> + maps:map( + fun + (<<"allow">>, Value) -> mask_bits(Value); + (<<"deny">>, Value) -> mask_bits(Value); + (_Key, Value) -> Value + end, + Overwrite + ); +mask_overwrite(Overwrite) -> + Overwrite. + +-spec mask_bits(term()) -> term(). +mask_bits(Value) when is_integer(Value) -> + Value band bnot ?THREAD_PERMISSION_BITS; +mask_bits(Value) when is_binary(Value) -> + case permission_bits:parse_optional(Value) of + Bits when is_integer(Bits) -> integer_to_binary(Bits band bnot ?THREAD_PERMISSION_BITS); + _ -> Value + end; +mask_bits(Value) -> + Value. + +-spec strip_internal(map()) -> map(). +strip_internal(Data) -> + maps:filter(fun(Key, _Value) -> not internal_key(Key) end, Data). + +-spec internal_key(term()) -> boolean(). +internal_key(<<"_fluxer_", _/binary>>) -> true; +internal_key(<<"__thread_", _/binary>>) -> true; +internal_key(_) -> false. + +-spec recipient_active(integer(), guild_state()) -> fun((integer()) -> boolean()). +recipient_active(ChannelId, State) -> + case is_gated_channel_id(ChannelId, State) andalso active(State) of + false -> + fun(_UserId) -> true end; + true -> + Config = channel_threads_config:config(), + fun(UserId) -> + channel_threads_config:user_active(Config, UserId) orelse + (is_bot_member(UserId, State) andalso + not channel_threads_config:user_excluded(Config, UserId)) + end + end. + +-spec is_bot_member(integer(), guild_state()) -> boolean(). +is_bot_member(UserId, State) -> + case guild_permissions:find_member_by_user_id(UserId, State) of + #{<<"user">> := #{<<"bot">> := true}} -> true; + _ -> false + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +active_state() -> + #{id => 1, data => #{thread_gate => #{active => true, version => 3}}}. + +inactive_filter_drops_thread_events_test() -> + State = #{id => 1, data => #{}}, + ?assertEqual(drop, pre_update_filter(thread_create, #{}, State)), + ?assertEqual(drop, pre_update_filter(channel_create, #{<<"type">> => 15}, State)), + ?assertEqual(drop, pre_update_filter(message_create, #{<<"type">> => 18}, State)), + ?assertEqual(drop, pre_update_filter(channel_create, #{<<"type">> => 10}, State)), + ?assertEqual(pass, pre_update_filter(channel_create, #{<<"type">> => 0}, State)), + ?assertEqual(pass, pre_update_filter(channel_create, #{<<"type">> => 5}, State)), + ?assertEqual(pass, pre_update_filter(message_create, #{<<"type">> => 0}, State)), + ?assertEqual(pass, pre_update_filter(thread_create, #{}, active_state())), + Text = #{<<"id">> => <<"1">>, <<"type">> => 0}, + Forum = #{<<"id">> => <<"2">>, <<"type">> => 15}, + ?assertEqual( + pass, pre_update_filter(channel_update_bulk, #{<<"channels">> => [Text]}, State) + ), + ?assertEqual( + {filtered, #{<<"channels">> => [Text]}}, + pre_update_filter(channel_update_bulk, #{<<"channels">> => [Text, Forum]}, State) + ), + ?assertEqual( + drop, pre_update_filter(channel_update_bulk, #{<<"channels">> => [Forum]}, State) + ). + +event_scope_is_open_when_untainted_and_inactive_test() -> + State = #{id => 1, data => #{}}, + ?assertEqual(open, event_scope(message_create, #{<<"type">> => 18}, State)), + ?assertEqual( + open, event_scope(message_delete_bulk, #{<<"channel_id">> => <<"77">>}, State) + ). + +event_scope_limits_gated_events_to_viewers_test() -> + State = active_state(), + ?assertEqual(viewers, event_scope(message_create, #{<<"type">> => 18}, State)), + ?assertEqual(viewers, event_scope(channel_create, #{<<"type">> => 16}, State)), + ?assertEqual(viewers, event_scope(channel_update, #{<<"type">> => 10}, State)), + ?assertEqual(open, event_scope(channel_update, #{<<"type">> => 5}, State)), + ?assertEqual( + viewers, event_scope(message_update, #{<<"__thread_only_update">> => true}, State) + ), + ?assertEqual(open, event_scope(message_create, #{<<"type">> => 0}, State)), + ?assertEqual( + viewers, event_scope(message_delete_bulk, #{<<"channel_id">> => <<"77">>}, State) + ), + ?assertEqual( + viewers, + event_scope( + message_delete_bulk, + #{<<"channel_id">> => <<"77">>}, + #{id => 1, data => #{thread_tainted => true}} + ) + ), + Sessions = #{ + <<"a">> => #{user_id => 1, thread_viewer => true}, + <<"b">> => #{user_id => 2, thread_viewer => false} + }, + ?assertEqual([<<"a">>], maps:keys(viewer_sessions(Sessions))). + +mask_message_clears_nested_thread_artifacts_test() -> + Message = #{ + <<"flags">> => 16#120 bor 4, + <<"thread">> => #{<<"id">> => <<"9">>}, + <<"referenced_message">> => #{<<"flags">> => 16#20, <<"thread">> => #{}}, + <<"message_snapshots">> => [#{<<"message">> => #{<<"flags">> => 16#100}}] + }, + Masked = mask_message(Message), + ?assertEqual(4, maps:get(<<"flags">>, Masked)), + ?assertNot(maps:is_key(<<"thread">>, Masked)), + ?assertEqual(#{<<"flags">> => 0}, maps:get(<<"referenced_message">>, Masked)), + ?assertEqual( + [#{<<"message">> => #{<<"flags">> => 0}}], maps:get(<<"message_snapshots">>, Masked) + ), + ?assertEqual( + null, + maps:get( + <<"referenced_message">>, + mask_message(#{<<"referenced_message">> => #{<<"type">> => 18}}) + ) + ). + +mask_role_and_overwrites_clear_thread_bits_test() -> + Bits = integer_to_binary(?THREAD_PERMISSION_BITS bor 1024), + ?assertEqual(#{<<"permissions">> => <<"1024">>}, mask_role(#{<<"permissions">> => Bits})), + ?assertEqual( + #{<<"permission_overwrites">> => [#{<<"allow">> => 1024, <<"deny">> => 0}]}, + mask_overwrites(#{ + <<"permission_overwrites">> => [ + #{<<"allow">> => ?THREAD_PERMISSION_BITS bor 1024, <<"deny">> => 1 bsl 38} + ] + }) + ). + +strip_internal_drops_fluxer_keys_test() -> + ?assertEqual( + #{<<"id">> => 1}, + strip_internal(#{ + <<"id">> => 1, <<"_fluxer_thread">> => #{}, <<"__thread_only_update">> => true + }) + ). + +thread_sessions_skip_permission_resolve_for_viewable_public_parent_test() -> + Viewer = #{ + user_id => 1, bot => true, thread_viewer => true, viewable_channels => #{10 => true} + }, + Hidden = Viewer#{user_id => 2, viewable_channels => #{}}, + State = #{ + id => 1, + data => #{}, + sessions => #{ + <<"a">> => Viewer, <<"b">> => Viewer, <<"c">> => Hidden, <<"d">> => Hidden + } + }, + Public = #{<<"id">> => <<"20">>, <<"parent_id">> => <<"10">>, <<"type">> => 11}, + meck:new(guild_thread_permissions, [passthrough, no_link]), + meck:expect(guild_thread_permissions, resolve, fun(_UserId, _Thread, _Member, _State) -> + 0 + end), + meck:new(guild_permissions, [passthrough, no_link]), + meck:expect(guild_permissions, find_member_by_user_id, fun(_UserId, _State) -> #{} end), + try + Open = thread_sessions(typing_start, #{}, 20, Public, undefined, State), + ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Open])), + ?assertEqual(1, meck:num_calls(guild_thread_permissions, resolve, '_')), + meck:reset(guild_thread_permissions), + Private = Public#{<<"type">> => 12}, + ?assertEqual([], thread_sessions(typing_start, #{}, 20, Private, undefined, State)), + ?assertEqual(2, meck:num_calls(guild_thread_permissions, resolve, '_')) + after + meck:unload(guild_permissions), + meck:unload(guild_thread_permissions) + end. + +compute_viewer_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + Config = (channel_threads_config:default_config())#{ + enabled => true, + included_users => #{<<"5">> => true}, + excluded_users => #{<<"6">> => true} + }, + persistent_term:put(Key, Config), + try + ?assert(compute_viewer(true, false, true, 5)), + ?assertNot(compute_viewer(true, false, false, 5)), + ?assertNot(compute_viewer(false, false, true, 5)), + ?assertNot(compute_viewer(true, false, true, 7)), + ?assert(compute_viewer(true, true, false, 7)), + ?assertNot(compute_viewer(true, true, false, 6)) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_load.erl b/fluxer_gateway/src/guild/guild_thread_load.erl new file mode 100644 index 000000000..c2ec0a9f3 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_load.erl @@ -0,0 +1,999 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_load). +-typing([eqwalizer]). + +-export([ + channels_collection/1, + take_extras/1, + install/2, + export_handoff/1, + handle_retry/1, + handle_load_result/3, + queue_event/3, + loading/1, + load_payload/1, + parse_gate/1, + fetch_flip_data/2, + deactivate/1, + fetch_counts/0, + init_counters/0 +]). + +-define(HANDOFF_ROW_BUDGET, 50000). +-define(HANDOFF_MAX_AGE_MS, 60000). +-define(LOAD_QUEUE_CAP, 10000). +-define(BACKOFF_MS, [30000, 120000, 600000]). +-define(FLIP_RETRIES, 5). +-define(FLIP_RETRY_DELAY_MS, 1000). +-define(MEMBER_PAGE_LIMIT, 1000). +-define(MEMBER_PAGES_PER_THREAD, 2). +-define(MEMBER_THREAD_BATCH, 100). +-define(FETCH_COUNTS_KEY, {?MODULE, fetch_counts}). +-define(EXTRA_KEYS, [thread_gate, thread_tainted, thread_load, thread_rows, thread_rows_at]). + +-type guild_state() :: map(). +-type gate() :: #{active := boolean(), version := non_neg_integer()}. +-type payload() :: #{threads := [map()], members := [map()], parent_settings := [map()]}. +-type fetch_result() :: ok | error | refetch. + +-export_type([payload/0, gate/0]). + +-spec channels_collection(map()) -> {[map()], map()}. +channels_collection(Response) -> + Channels = list_field(<<"channels">>, Response), + Gate = parse_gate(maps:get(<<"thread_gate">>, Response, undefined)), + Tainted = maps:get(<<"thread_tainted">>, Response, false) =:= true, + Base = #{thread_gate => Gate, thread_tainted => Tainted}, + case Gate of + #{active := true} -> + {merge_channels(Channels, list_field(<<"thread_only_channels">>, Response)), Base}; + _ -> + {Channels, Base} + end. + +-spec fetch_payload(integer()) -> {ok, payload()} | {error, term()}. +fetch_payload(GuildId) -> + case fetch_flip_data(GuildId, channel_threads_config:version()) of + {ok, Data} -> + count_fetch(ok), + {ok, load_payload(Data)}; + {error, Reason} -> + count_fetch(error), + {error, Reason} + end. + +-spec fetch_flip_data(integer(), non_neg_integer()) -> {ok, map()} | {error, term()}. +fetch_flip_data(GuildId, Version) -> + fetch_flip_data(GuildId, Version, ?FLIP_RETRIES). + +-spec fetch_flip_data(integer(), non_neg_integer(), non_neg_integer()) -> + {ok, map()} | {error, term()}. +fetch_flip_data(GuildId, Version, Retries) -> + Request = #{ + <<"type">> => <<"guild_thread_flip_data">>, + <<"guild_id">> => integer_to_binary(GuildId), + <<"config_version">> => Version, + <<"paged_members">> => true + }, + try rpc_client:call(Request) of + {ok, #{<<"config_version">> := ApiVersion}} when + is_integer(ApiVersion), ApiVersion < Version, Retries > 0 + -> + timer:sleep(?FLIP_RETRY_DELAY_MS), + fetch_flip_data(GuildId, Version, Retries - 1); + {ok, #{<<"threads">> := Threads} = Data} when + is_list(Threads), not is_map_key(<<"thread_members">>, Data) + -> + with_paged_members(GuildId, Threads, Data); + {ok, Data} -> + {ok, Data}; + {error, Reason} -> + {error, Reason} + catch + Class:Reason -> {error, {Class, Reason}} + end. + +-spec with_paged_members(integer(), list(), map()) -> {ok, map()} | {error, term()}. +with_paged_members(GuildId, Threads, Data) -> + ThreadIds = [ + Id + || T <- Threads, is_map(T), Id <- [maps:get(<<"id">>, T, undefined)], is_binary(Id) + ], + try fetch_thread_members(GuildId, ThreadIds, []) of + {ok, Members} -> {ok, Data#{<<"thread_members">> => Members}}; + {error, Reason} -> {error, Reason} + catch + Class:Reason -> {error, {Class, Reason}} + end. + +-spec fetch_thread_members(integer(), [binary()], [[map()]]) -> + {ok, [map()]} | {error, term()}. +fetch_thread_members(_GuildId, [], Acc) -> + {ok, lists:append(lists:reverse(Acc))}; +fetch_thread_members(GuildId, ThreadIds, Acc) -> + {Batch, Rest} = lists:split(min(?MEMBER_THREAD_BATCH, length(ThreadIds)), ThreadIds), + case + fetch_member_pages( + GuildId, Batch, undefined, Acc, ?MEMBER_PAGES_PER_THREAD * length(Batch) + ) + of + {ok, Acc1} -> fetch_thread_members(GuildId, Rest, Acc1); + {error, Reason} -> {error, Reason} + end. + +-spec fetch_member_pages( + integer(), [binary()], binary() | undefined, [[map()]], non_neg_integer() +) -> + {ok, [[map()]]} | {error, term()}. +fetch_member_pages(_GuildId, [], _After, Acc, _PagesLeft) -> + {ok, Acc}; +fetch_member_pages(_GuildId, _ThreadIds, _After, Acc, 0) -> + {ok, Acc}; +fetch_member_pages(GuildId, ThreadIds, After, Acc, PagesLeft) -> + Request0 = #{ + <<"type">> => <<"list_thread_members">>, + <<"guild_id">> => integer_to_binary(GuildId), + <<"thread_ids">> => ThreadIds, + <<"limit">> => ?MEMBER_PAGE_LIMIT + }, + Request = + case After of + undefined -> Request0; + _ -> Request0#{<<"after_user_id">> => After} + end, + case rpc_client:call(Request) of + {ok, #{<<"members">> := Members} = Page} when is_list(Members) -> + Acc1 = [[M || M <- Members, is_map(M)] | Acc], + case next_member_cursor(Page, ThreadIds) of + done -> + {ok, Acc1}; + {Remaining, Next} -> + fetch_member_pages(GuildId, Remaining, Next, Acc1, PagesLeft - 1); + invalid -> + {error, invalid_thread_members_page} + end; + {ok, _Invalid} -> + {error, invalid_thread_members_page}; + {error, Reason} -> + {error, Reason} + end. + +-spec next_member_cursor(map(), [binary()]) -> + done | invalid | {[binary()], binary() | undefined}. +next_member_cursor( + #{<<"has_more">> := true, <<"next_thread_id">> := ThreadId} = Page, ThreadIds +) when + is_binary(ThreadId) +-> + case lists:dropwhile(fun(Id) -> Id =/= ThreadId end, ThreadIds) of + [] -> + invalid; + Remaining -> + case maps:get(<<"next_after_user_id">>, Page, null) of + Next when is_binary(Next) -> {Remaining, Next}; + null -> {Remaining, undefined}; + _ -> invalid + end + end; +next_member_cursor(#{<<"has_more">> := true}, _ThreadIds) -> + invalid; +next_member_cursor(_Page, _ThreadIds) -> + done. + +-spec load_payload(map()) -> payload(). +load_payload(Response) -> + #{ + threads => [guild_data_normalize_thread(T) || T <- list_field(<<"threads">>, Response)], + members => [ + guild_data_normalize:thread_member(M) + || M <- list_field(<<"thread_members">>, Response) + ], + parent_settings => list_field(<<"thread_parent_settings">>, Response) + }. + +-spec guild_data_normalize_thread(map()) -> map(). +guild_data_normalize_thread(Thread) -> + case guild_data_normalize:thread(Thread) of + Normalized when is_map(Normalized) -> Normalized; + _ -> Thread + end. + +-spec parse_gate(term()) -> gate(). +parse_gate(#{<<"active">> := Active} = Gate) when is_boolean(Active) -> + Version = + case maps:get(<<"config_version">>, Gate, 0) of + V when is_integer(V), V >= 0 -> V; + _ -> 0 + end, + #{active => Active, version => Version}; +parse_gate(_) -> + #{active => false, version => 0}. + +-spec merge_channels([map()], [map()]) -> [map()]. +merge_channels(Channels, []) -> + Channels; +merge_channels(Channels, Forums) -> + Ids = sets:from_list([maps:get(<<"id">>, C, undefined) || C <- Channels]), + Channels ++ [F || F <- Forums, not sets:is_element(maps:get(<<"id">>, F, undefined), Ids)]. + +-spec take_extras(map()) -> {map(), map()}. +take_extras(Data) -> + Extras0 = maps:with(?EXTRA_KEYS, Data), + Data1 = maps:without(?EXTRA_KEYS, Data), + case maps:get(<<"guild">>, Data1, undefined) of + Guild when is_map(Guild) -> + Extras = maps:merge(guild_extras(Guild), Extras0), + {Extras, Data1#{ + <<"guild">> => maps:without([<<"thread_gate">>, <<"thread_tainted">>], Guild) + }}; + _ -> + {Extras0, Data1} + end. + +-spec guild_extras(map()) -> map(). +guild_extras(Guild) -> + Gate = + case maps:find(<<"thread_gate">>, Guild) of + {ok, G} -> #{thread_gate => parse_gate(G)}; + error -> #{} + end, + case maps:find(<<"thread_tainted">>, Guild) of + {ok, true} -> Gate#{thread_tainted => true}; + _ -> Gate + end. + +-spec install(map(), guild_state()) -> guild_state(). +install(Extras, State) -> + Data = maps:get(data, State), + Gate = maps:get(thread_gate, Extras, #{active => false, version => 0}), + Tainted = maps:get(thread_tainted, Extras, false) =:= true, + Data1 = put_flag(thread_tainted, Tainted, put_gate(Gate, Data)), + State1 = State#{data => Data1}, + State2 = + case Gate of + #{active := true} -> install_active(Extras, State1); + _ -> State1 + end, + case guild_thread_gate:active(State2) orelse any_viewer(State2) of + true -> + guild_thread_subscriptions:refresh_presence_users( + guild_thread_gate:recompute_session_viewers(State2) + ); + false -> + State2 + end. + +-spec any_viewer(guild_state()) -> boolean(). +any_viewer(State) -> + lists:any( + fun guild_thread_gate:session_viewer/1, + [S || S <- maps:values(maps:get(sessions, State, #{})), is_map(S)] + ). + +-spec put_gate(gate(), map()) -> map(). +put_gate(#{active := true} = Gate, Data) -> Data#{thread_gate => Gate}; +put_gate(_Gate, Data) -> maps:remove(thread_gate, Data). + +-spec put_flag(atom(), boolean(), map()) -> map(). +put_flag(Key, true, Map) -> Map#{Key => true}; +put_flag(Key, false, Map) -> maps:remove(Key, Map). + +-spec install_active(map(), guild_state()) -> guild_state(). +install_active(#{thread_rows := Rows} = Extras, State) when is_list(Rows) -> + case fresh_rows(Extras) of + true -> + put_store(guild_thread_store:import(Rows), ready, State); + false -> + count_fetch(refetch), + start_retry(put_store(guild_thread_store:new(), loading, State)) + end; +install_active(#{thread_kept := Tab}, State) -> + start_retry(put_store(Tab, loading, State)); +install_active(#{thread_load := {ok, Payload}}, State) when is_map(Payload) -> + State1 = put_store(guild_thread_store:new(), ready, State), + apply_payload(Payload, State1); +install_active(_Extras, State) -> + start_retry(put_store(guild_thread_store:new(), loading, State)). + +-spec fresh_rows(map()) -> boolean(). +fresh_rows(#{thread_rows_at := At}) when is_integer(At) -> + erlang:system_time(millisecond) - At =< ?HANDOFF_MAX_AGE_MS; +fresh_rows(_) -> + false. + +-spec put_store(ets:table(), atom(), guild_state()) -> guild_state(). +put_store(Tab, Status, State) -> + Data = maps:get(data, State), + State#{ + data => Data#{thread_store => Tab}, + thread_load => #{status => Status, queue => [], queued => 0, attempt => 0} + }. + +-spec apply_payload(payload() | map(), guild_state()) -> guild_state(). +apply_payload(Payload, State) -> + case guild_thread_gate:store(State) of + undefined -> + State; + Tab -> + ok = guild_thread_store:load( + Tab, maps:get(threads, Payload, []), maps:get(members, Payload, []) + ), + merge_parent_settings(maps:get(parent_settings, Payload, []), State) + end. + +-spec merge_parent_settings([map()], guild_state()) -> guild_state(). +merge_parent_settings([], State) -> + State; +merge_parent_settings(Settings, State) -> + Data = maps:get(data, State), + ById = maps:from_list([ + { + snowflake_id:parse_maybe(maps:get(<<"channel_id">>, S, undefined)), + maps:remove(<<"channel_id">>, S) + } + || S <- Settings, is_map(S) + ]), + Channels = [merge_parent_setting(C, ById) || C <- guild_data_index:channel_list(Data)], + State#{data => guild_data_index:put_channels(Channels, Data)}. + +-spec merge_parent_setting(map(), map()) -> map(). +merge_parent_setting(Channel, ById) -> + case + maps:get( + snowflake_id:parse_maybe(maps:get(<<"id">>, Channel, undefined)), ById, undefined + ) + of + Settings when is_map(Settings) -> maps:merge(Channel, Settings); + _ -> Channel + end. + +-spec export_handoff(guild_state()) -> map(). +export_handoff(State) -> + export_handoff_data(maps:get(data, State, #{}), loading(State)). + +-spec export_handoff_data(map(), boolean()) -> map(). +export_handoff_data(#{thread_store := Tab} = Data, false) -> + Base = maps:remove(thread_store, Data), + try guild_thread_store:row_count(Tab) =< ?HANDOFF_ROW_BUDGET of + true -> + Base#{ + thread_rows => guild_thread_store:export(Tab), + thread_rows_at => erlang:system_time(millisecond) + }; + false -> + count_fetch(refetch), + Base + catch + error:badarg -> Base + end; +export_handoff_data(Data, _Loading) -> + maps:remove(thread_store, Data). + +-spec loading(guild_state()) -> boolean(). +loading(#{thread_load := #{status := Status}}) -> Status =/= ready; +loading(_) -> false. + +-spec queue_event(atom(), map(), guild_state()) -> guild_state(). +queue_event( + Event, + EventData, + #{thread_load := #{queue := Queue, queued := Queued, ref := _} = Load} = State +) -> + case Queued >= ?LOAD_QUEUE_CAP of + true -> + count_fetch(refetch), + start_retry(State#{ + thread_load => Load#{ + queue => [], queued => 0, attempt => maps:get(attempt, Load, 0) + 1 + } + }); + false -> + State#{ + thread_load => Load#{ + queue => [{Event, EventData} | Queue], queued => Queued + 1 + } + } + end; +queue_event(_Event, _EventData, State) -> + State. + +-spec start_retry(guild_state()) -> guild_state(). +start_retry(#{thread_load := Load0} = State) -> + _ = + case maps:get(timer, Load0, undefined) of + Old when is_reference(Old) -> erlang:cancel_timer(Old); + _ -> false + end, + Load = maps:remove(ref, Load0), + Timer = erlang:send_after( + retry_delay(maps:get(attempt, Load, 0)), self(), thread_load_retry + ), + State#{thread_load => Load#{status => loading, timer => Timer}}. + +-spec retry_delay(non_neg_integer()) -> non_neg_integer(). +retry_delay(0) -> 0; +retry_delay(Attempt) -> lists:nth(min(Attempt, length(?BACKOFF_MS)), ?BACKOFF_MS). + +-spec handle_retry(guild_state()) -> guild_state(). +handle_retry(State) -> + case {maps:get(id, State, undefined), maps:get(thread_load, State, undefined)} of + {GuildId, Load} when is_integer(GuildId), is_map(Load) -> + retry_load(GuildId, Load, State); + _ -> + State + end. + +-spec retry_load(integer(), map(), guild_state()) -> guild_state(). +retry_load(GuildId, Load, State) -> + case guild_thread_gate:active(State) of + false -> + maps:remove(thread_load, State); + true -> + Self = self(), + Ref = make_ref(), + _ = spawn(fun() -> + Self ! {thread_load_result, Ref, fetch_payload(GuildId)} + end), + State#{thread_load => Load#{ref => Ref}} + end. + +-spec handle_load_result(reference(), term(), guild_state()) -> guild_state(). +handle_load_result(Ref, Result, #{thread_load := #{ref := Ref} = Load} = State) -> + case {Result, guild_thread_gate:store(State)} of + {{ok, Payload}, Tab} when Tab =/= undefined -> + Fresh = guild_thread_store:new(), + State1 = replace_store(Tab, Fresh, State), + State2 = apply_payload(Payload, State1#{ + thread_load => #{status => ready, queue => [], queued => 0, attempt => 0} + }), + State3 = replay(lists:reverse(maps:get(queue, Load, [])), State2), + guild_thread_subscriptions:resync_after_load(State3); + {{ok, _Payload}, undefined} -> + maps:remove(thread_load, State); + {_Error, _Tab} -> + start_retry(State#{ + thread_load => Load#{attempt => maps:get(attempt, Load, 0) + 1} + }) + end; +handle_load_result(_Ref, _Result, State) -> + State. + +-spec replace_store(ets:table(), ets:table(), guild_state()) -> guild_state(). +replace_store(Old, New, State) -> + Data = maps:get(data, State), + ok = guild_thread_store:retire(Old), + State1 = State#{data => Data#{thread_store => New}}, + ok = guild_maintenance:maybe_put_permission_cache(State1), + State1. + +-spec replay([{atom(), map()}], guild_state()) -> guild_state(). +replay(Events, State) -> + lists:foldl( + fun + ({Event, _EventData}, Acc) when + Event =:= message_delete; Event =:= message_delete_bulk + -> + Acc; + ({Event, EventData}, Acc) -> + guild_state_threads:after_replayed_event( + Event, EventData, guild_state_threads:apply_event(Event, EventData, Acc) + ) + end, + State, + Events + ). + +-spec deactivate(guild_state()) -> guild_state(). +deactivate(State) -> + Data = maps:get(data, State), + ok = guild_thread_store:retire(maps:get(thread_store, Data, undefined)), + _ = + case maps:get(thread_load, State, undefined) of + #{timer := Timer} when is_reference(Timer) -> erlang:cancel_timer(Timer); + _ -> false + end, + maps:remove(thread_load, State#{data => maps:remove(thread_store, Data)}). + +-spec list_field(binary(), map()) -> [map()]. +list_field(Key, Map) -> + case maps:get(Key, Map, []) of + List when is_list(List) -> [Item || Item <- List, is_map(Item)]; + _ -> [] + end. + +-spec count_fetch(ok | error | refetch) -> ok. +count_fetch(Result) -> + guild_thread_store:add_counter(?FETCH_COUNTS_KEY, fetch_index(Result), 1). + +-spec init_counters() -> ok. +init_counters() -> + guild_thread_store:ensure_counter(?FETCH_COUNTS_KEY, 3). + +-spec fetch_index(fetch_result()) -> 1..3. +fetch_index(ok) -> 1; +fetch_index(error) -> 2; +fetch_index(refetch) -> 3. + +-spec fetch_counts() -> #{fetch_result() => non_neg_integer()}. +fetch_counts() -> + case persistent_term:get(?FETCH_COUNTS_KEY, undefined) of + undefined -> + #{}; + Counters -> + maps:from_list([ + {R, counters:get(Counters, fetch_index(R))} + || R <- [ok, error, refetch] + ]) + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +channels_collection_inactive_keeps_channels_test() -> + Response = #{<<"channels">> => [#{<<"id">> => <<"1">>}]}, + ?assertEqual( + {[#{<<"id">> => <<"1">>}], #{ + thread_gate => #{active => false, version => 0}, thread_tainted => false + }}, + channels_collection(Response) + ). + +channels_collection_active_merges_forums_and_defers_threads_test() -> + Response = #{ + <<"channels">> => [#{<<"id">> => <<"1">>}], + <<"thread_only_channels">> => [#{<<"id">> => <<"2">>, <<"type">> => 15}], + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 4}, + <<"thread_tainted">> => true, + <<"threads">> => [ + #{ + <<"id">> => <<"10">>, + <<"parent_id">> => <<"1">>, + <<"type">> => 11, + <<"member_ids_preview">> => [<<"7">>] + } + ], + <<"thread_members">> => [ + #{<<"id">> => <<"10">>, <<"user_id">> => <<"7">>, <<"flags">> => 1} + ] + }, + {Channels, Extras} = channels_collection(Response), + ?assertEqual([<<"1">>, <<"2">>], [maps:get(<<"id">>, C) || C <- Channels]), + ?assertEqual( + #{thread_gate => #{active => true, version => 4}, thread_tainted => true}, Extras + ). + +load_payload_normalizes_threads_and_members_test() -> + Payload = load_payload(#{ + <<"threads">> => [ + #{ + <<"id">> => <<"10">>, + <<"parent_id">> => <<"1">>, + <<"type">> => 11, + <<"member_ids_preview">> => [<<"7">>] + } + ], + <<"thread_members">> => [ + #{<<"id">> => <<"10">>, <<"user_id">> => <<"7">>, <<"flags">> => 1} + ] + }), + #{threads := [Thread], members := [Member]} = Payload, + ?assertEqual(10, maps:get(<<"id">>, Thread)), + ?assertEqual([7], maps:get(<<"member_ids_preview">>, Thread)), + ?assertEqual(7, maps:get(<<"user_id">>, Member)). + +take_extras_strips_the_guild_map_test() -> + Data = #{ + <<"guild">> => #{<<"id">> => <<"5">>, <<"thread_gate">> => #{<<"active">> => true}}, + thread_tainted => true + }, + {Extras, Rest} = take_extras(Data), + ?assertEqual(#{<<"guild">> => #{<<"id">> => <<"5">>}}, Rest), + ?assertEqual( + #{thread_gate => #{active => true, version => 0}, thread_tainted => true}, Extras + ). + +handoff_export_respects_the_row_budget_test() -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + Data = #{thread_store => Tab, thread_gate => #{active => true}}, + Exported = export_handoff(#{ + data => Data, thread_load => #{status => ready, queue => [], queued => 0} + }), + ?assertNot(maps:is_key(thread_store, Exported)), + ?assertEqual(2, length(maps:get(thread_rows, Exported))), + guild_thread_store:destroy(Tab). + +handoff_export_while_loading_forces_a_refetch_test() -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + Gate = #{active => true, version => 1}, + Exported = export_handoff(#{ + data => #{thread_store => Tab, thread_gate => Gate}, + thread_load => #{status => loading, queue => [], queued => 0} + }), + guild_thread_store:destroy(Tab), + ?assertNot(maps:is_key(thread_store, Exported)), + ?assertNot(maps:is_key(thread_rows, Exported)), + {Extras, _} = take_extras(Exported), + Installed = install(Extras, #{id => 5, data => #{}, sessions => #{}}), + ?assert(loading(Installed)), + #{thread_load := #{timer := Timer}} = Installed, + erlang:cancel_timer(Timer), + deactivate(Installed). + +install_imports_fresh_rows_and_refetches_stale_ones_test() -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + Rows = guild_thread_store:export(Tab), + guild_thread_store:destroy(Tab), + Base = #{id => 5, data => #{<<"channels">> => []}, sessions => #{}}, + Gate = #{active => true, version => 1}, + Fresh = install( + #{ + thread_gate => Gate, + thread_rows => Rows, + thread_rows_at => erlang:system_time(millisecond) + }, + Base + ), + ?assertNotEqual(undefined, guild_thread_gate:thread(10, Fresh)), + ?assertNot(loading(Fresh)), + Stale = install(#{thread_gate => Gate, thread_rows => Rows, thread_rows_at => 0}, Base), + ?assert(loading(Stale)), + ?assertEqual(undefined, guild_thread_gate:thread(10, Stale)), + #{thread_load := #{timer := Timer}} = Stale, + erlang:cancel_timer(Timer), + deactivate(Fresh), + deactivate(Stale). + +fetch_flip_data_pages_thread_members_test() -> + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun + (#{<<"type">> := <<"guild_thread_flip_data">>, <<"paged_members">> := true}) -> + {ok, #{ + <<"config_version">> => 1, + <<"threads">> => [#{<<"id">> => <<"10">>}, #{<<"id">> => <<"11">>}] + }}; + ( + #{<<"type">> := <<"list_thread_members">>, <<"thread_ids">> := [<<"10">>, <<"11">>]} = + R + ) -> + case maps:get(<<"after_user_id">>, R, undefined) of + undefined -> + {ok, #{ + <<"members">> => [#{<<"id">> => <<"10">>, <<"user_id">> => <<"7">>}], + <<"has_more">> => true, + <<"next_thread_id">> => <<"10">>, + <<"next_after_user_id">> => <<"7">> + }}; + <<"7">> -> + {ok, #{ + <<"members">> => [#{<<"id">> => <<"10">>, <<"user_id">> => <<"8">>}], + <<"has_more">> => true, + <<"next_thread_id">> => <<"11">>, + <<"next_after_user_id">> => null + }} + end; + (#{<<"type">> := <<"list_thread_members">>, <<"thread_ids">> := [<<"11">>]} = R) -> + ?assertNot(maps:is_key(<<"after_user_id">>, R)), + {ok, #{ + <<"members">> => [#{<<"id">> => <<"11">>, <<"user_id">> => <<"9">>}], + <<"has_more">> => false + }} + end), + try + {ok, Data} = fetch_flip_data(5, 1), + ?assertEqual( + [<<"7">>, <<"8">>, <<"9">>], + [maps:get(<<"user_id">>, M) || M <- maps:get(<<"thread_members">>, Data)] + ), + ?assertEqual(4, meck:num_calls(rpc_client, call, '_')) + after + meck:unload(rpc_client) + end. + +fetch_flip_data_batches_many_threads_per_rpc_test() -> + Ids = [integer_to_binary(N) || N <- lists:seq(1, 250)], + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun + (#{<<"type">> := <<"guild_thread_flip_data">>}) -> + {ok, #{ + <<"config_version">> => 1, <<"threads">> => [#{<<"id">> => Id} || Id <- Ids] + }}; + (#{<<"type">> := <<"list_thread_members">>, <<"thread_ids">> := Batch}) -> + {ok, #{ + <<"members">> => [#{<<"id">> => Id, <<"user_id">> => <<"7">>} || Id <- Batch], + <<"has_more">> => false + }} + end), + try + {ok, Data} = fetch_flip_data(5, 1), + ?assertEqual(Ids, [maps:get(<<"id">>, M) || M <- maps:get(<<"thread_members">>, Data)]), + ?assertEqual(4, meck:num_calls(rpc_client, call, '_')) + after + meck:unload(rpc_client) + end. + +fetch_flip_data_rejects_a_cursor_outside_the_batch_test() -> + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun + (#{<<"type">> := <<"guild_thread_flip_data">>}) -> + {ok, #{<<"config_version">> => 1, <<"threads">> => [#{<<"id">> => <<"10">>}]}}; + (#{<<"type">> := <<"list_thread_members">>}) -> + {ok, #{ + <<"members">> => [], <<"has_more">> => true, <<"next_thread_id">> => <<"99">> + }} + end), + try + ?assertEqual({error, invalid_thread_members_page}, fetch_flip_data(5, 1)) + after + meck:unload(rpc_client) + end. + +loaded_forum_without_a_config_row_gets_default_surface_fields_test() -> + Forum = #{<<"id">> => <<"2">>, <<"type">> => 15, <<"name">> => <<"forum">>}, + Data = guild_data_index:put_channels([Forum], #{}), + Settings = #{ + <<"channel_id">> => <<"2">>, + <<"available_tags">> => [], + <<"default_tag_setting">> => <<"match_some">>, + <<"default_forum_layout">> => 0, + <<"flags">> => 0 + }, + State = install( + #{ + thread_gate => #{active => true, version => 1}, + thread_load => + {ok, #{threads => [], members => [], parent_settings => [Settings]}} + }, + #{id => 5, data => Data, sessions => #{}} + ), + [Loaded] = guild_data_index:channel_list(maps:get(data, State)), + ?assertEqual([], maps:get(<<"available_tags">>, Loaded)), + ?assertEqual(<<"match_some">>, maps:get(<<"default_tag_setting">>, Loaded)), + ?assertEqual(0, maps:get(<<"default_forum_layout">>, Loaded)), + deactivate(State). + +fetch_flip_data_fails_on_a_failed_member_page_test() -> + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun + (#{<<"type">> := <<"guild_thread_flip_data">>}) -> + {ok, #{<<"config_version">> => 1, <<"threads">> => [#{<<"id">> => <<"10">>}]}}; + (#{<<"type">> := <<"list_thread_members">>}) -> + {error, timeout} + end), + try + ?assertEqual({error, timeout}, fetch_flip_data(5, 1)) + after + meck:unload(rpc_client) + end. + +fetch_flip_data_fails_when_a_member_page_throws_test() -> + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun + (#{<<"type">> := <<"guild_thread_flip_data">>}) -> + {ok, #{<<"config_version">> => 1, <<"threads">> => [#{<<"id">> => <<"10">>}]}}; + (#{<<"type">> := <<"list_thread_members">>}) -> + error(circuit_open) + end), + try + ?assertEqual({error, {error, circuit_open}}, fetch_flip_data(5, 1)) + after + meck:unload(rpc_client) + end. + +fetch_flip_data_keeps_inline_members_from_an_older_api_test() -> + Inline = #{ + <<"config_version">> => 1, + <<"threads">> => [#{<<"id">> => <<"10">>}], + <<"thread_members">> => [#{<<"id">> => <<"10">>, <<"user_id">> => <<"7">>}] + }, + meck:new(rpc_client, [no_link]), + meck:expect(rpc_client, call, fun(_) -> {ok, Inline} end), + try + ?assertEqual({ok, Inline}, fetch_flip_data(5, 1)), + ?assertEqual(1, meck:num_calls(rpc_client, call, '_')) + after + meck:unload(rpc_client) + end. + +install_inactive_creates_no_store_test() -> + State = install(#{}, #{id => 5, data => #{}, sessions => #{}}), + ?assertEqual(#{}, maps:get(data, State)), + ?assertNot(maps:is_key(thread_load, State)). + +queue_overflow_triggers_refetch_test() -> + Stale = make_ref(), + State0 = #{ + id => 5, + data => #{thread_gate => #{active => true, version => 1}}, + thread_load => #{ + status => loading, + queue => [], + queued => ?LOAD_QUEUE_CAP, + attempt => 0, + ref => Stale + } + }, + State = queue_event(thread_update, #{}, State0), + #{thread_load := #{queued := 0, queue := [], attempt := 1, timer := Timer} = Load} = State, + ?assertNot(maps:is_key(ref, Load)), + ?assertEqual(State, handle_load_result(Stale, {ok, #{}}, State)), + Remaining = erlang:cancel_timer(Timer), + ?assert(is_integer(Remaining) andalso Remaining > 25000). + +deactivate_orphans_an_inflight_fetch_test() -> + Stale = make_ref(), + State = deactivate(#{ + id => 5, + data => #{}, + thread_load => #{status => loading, ref => Stale} + }), + ?assertNot(maps:is_key(thread_load, State)), + ?assertEqual(State, handle_load_result(Stale, {ok, #{}}, State)). + +events_during_a_backoff_are_not_queued_test() -> + ok = guild_thread_store:init(), + Tab = guild_thread_store:new(), + Timer = erlang:send_after(600000, self(), thread_load_retry), + State0 = #{ + id => 5, + data => #{thread_store => Tab, thread_gate => #{active => true, version => 1}}, + thread_load => #{ + status => loading, queue => [], queued => 0, attempt => 3, timer => Timer + } + }, + Message = #{<<"id">> => <<"400">>, <<"channel_id">> => <<"300">>, <<"type">> => 0}, + try + State = lists:foldl( + fun(_, Acc) -> + guild_state_threads:after_regular_event(message_create, Message, Acc) + end, + State0, + lists:seq(1, ?LOAD_QUEUE_CAP + 1) + ), + ?assertMatch( + #{thread_load := #{queued := 0, queue := [], attempt := 3, timer := Timer}}, State + ), + ?assert(is_integer(erlang:read_timer(Timer))) + after + erlang:cancel_timer(Timer), + guild_thread_store:destroy(Tab) + end. + +queued_deletes_do_not_replay_onto_the_snapshot_test() -> + ok = guild_thread_store:init(), + Loading = guild_thread_store:new(), + Ref = make_ref(), + State0 = #{ + id => 5, + disable_permission_cache_updates => true, + sessions => #{}, + data => #{thread_store => Loading, thread_gate => #{active => true, version => 1}}, + thread_load => #{status => loading, queue => [], queued => 0, attempt => 0, ref => Ref} + }, + State1 = guild_state_threads:after_regular_event( + message_delete, #{<<"id">> => <<"401">>, <<"channel_id">> => <<"300">>}, State0 + ), + State2 = guild_state_threads:after_regular_event( + message_delete_bulk, + #{<<"ids">> => [<<"402">>, <<"403">>], <<"channel_id">> => <<"300">>}, + State1 + ), + ?assertMatch(#{thread_load := #{queued := 2}}, State2), + Payload = load_payload(#{ + <<"threads">> => [ + #{ + <<"id">> => <<"300">>, + <<"parent_id">> => <<"100">>, + <<"type">> => 11, + <<"last_message_id">> => <<"400">>, + <<"message_count">> => 5, + <<"total_message_sent">> => 8, + <<"thread_metadata">> => #{<<"archived">> => false} + } + ] + }), + State3 = handle_load_result(Ref, {ok, Payload}, State2), + Fresh = guild_thread_gate:store(State3), + try + ?assertEqual( + 5, maps:get(<<"message_count">>, guild_thread_store:get_thread(Fresh, 300)) + ) + after + guild_thread_store:destroy(Fresh), + guild_thread_store:destroy(Loading) + end. + +regular_channel_messages_during_a_load_are_not_queued_test() -> + ok = guild_thread_store:init(), + Tab = guild_thread_store:new(), + Ref = make_ref(), + State0 = #{ + id => 5, + disable_permission_cache_updates => true, + sessions => #{}, + data => #{ + thread_store => Tab, + thread_gate => #{active => true, version => 1}, + <<"channels">> => [#{<<"id">> => <<"100">>, <<"type">> => 0}] + }, + thread_load => #{status => loading, queue => [], queued => 0, attempt => 1, ref => Ref} + }, + try + State1 = lists:foldl( + fun(N, Acc) -> + guild_state_threads:after_regular_event( + message_create, + #{<<"id">> => integer_to_binary(1000 + N), <<"channel_id">> => <<"100">>}, + Acc + ) + end, + State0, + lists:seq(1, ?LOAD_QUEUE_CAP + 1) + ), + ?assertMatch(#{thread_load := #{queued := 0, ref := Ref, attempt := 1}}, State1), + ?assertNot(maps:is_key(timer, maps:get(thread_load, State1))), + State2 = guild_state_threads:after_regular_event( + message_create, #{<<"id">> => <<"5000">>, <<"channel_id">> => <<"300">>}, State1 + ), + ?assertMatch(#{thread_load := #{queued := 1}}, State2) + after + guild_thread_store:retire(Tab) + end. + +store_events_during_a_load_replay_onto_the_fresh_store_test() -> + ok = guild_thread_store:init(), + Loading = guild_thread_store:new(), + Ref = make_ref(), + State0 = #{ + id => 5, + disable_permission_cache_updates => true, + sessions => #{}, + data => #{thread_store => Loading, thread_gate => #{active => true, version => 1}}, + thread_load => #{status => loading, queue => [], queued => 0, attempt => 0, ref => Ref} + }, + Message = #{<<"id">> => <<"400">>, <<"channel_id">> => <<"300">>, <<"type">> => 0}, + State1 = guild_state_threads:after_regular_event(message_create, Message, State0), + State2 = guild_state_threads:after_regular_event( + channel_delete, #{<<"id">> => <<"200">>, <<"type">> => 0}, State1 + ), + State3 = guild_state_threads:after_regular_event( + guild_member_remove, #{<<"user">> => #{<<"id">> => <<"7">>}}, State2 + ), + Thread = fun(Id, Parent) -> + #{ + <<"id">> => integer_to_binary(Id), + <<"parent_id">> => integer_to_binary(Parent), + <<"type">> => 11, + <<"message_count">> => 0, + <<"total_message_sent">> => 0, + <<"thread_metadata">> => #{<<"archived">> => false} + } + end, + Payload = load_payload(#{ + <<"threads">> => [Thread(300, 100), Thread(301, 200)], + <<"thread_members">> => [#{<<"id">> => <<"300">>, <<"user_id">> => <<"7">>}] + }), + State4 = handle_load_result(Ref, {ok, Payload}, State3), + Fresh = guild_thread_gate:store(State4), + try + ?assertNotEqual(Loading, Fresh), + ?assertNot(loading(State4)), + Loaded = guild_thread_store:get_thread(Fresh, 300), + ?assertEqual(400, snowflake_id:parse_maybe(maps:get(<<"last_message_id">>, Loaded))), + ?assertEqual(1, maps:get(<<"message_count">>, Loaded)), + ?assertEqual(undefined, guild_thread_store:get_thread(Fresh, 301)), + ?assertEqual([], guild_thread_store:member_ids(Fresh, 300)), + State5 = replay([{message_create, Message}], State4), + ?assertEqual( + 1, maps:get(<<"message_count">>, guild_thread_store:get_thread(Fresh, 300)) + ), + _ = State5 + after + guild_thread_store:destroy(Fresh), + guild_thread_store:destroy(Loading) + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_permissions.erl b/fluxer_gateway/src/guild/guild_thread_permissions.erl new file mode 100644 index 000000000..29e23b5d6 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_permissions.erl @@ -0,0 +1,209 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_permissions). +-typing([eqwalizer]). + +-export([ + resolve/4, + parent_id/1, + unknown_parent_count/0, + init_counters/0 +]). + +-define(ALL_PERMISSIONS, 16#FFFFFFFFFFFFFFFF). +-define(SEND_MESSAGES, 2048). +-define(UNKNOWN_PARENT_KEY, {?MODULE, unknown_parent}). + +-type permission() :: non_neg_integer(). +-type guild_state() :: map(). + +-spec resolve(integer(), map(), map() | undefined, guild_state()) -> permission(). +resolve(UserId, Thread, Member, State) -> + case parent_channel(Thread, State) of + undefined -> + note_unknown_parent(), + 0; + ParentId -> + ParentPerms = guild_permissions:compute_member_permissions( + UserId, ParentId, Member, State + ), + Perms = alias_permissions(ParentPerms), + case can_view_thread_from_parent(UserId, Thread, Perms, Member, State) of + true -> Perms; + false -> Perms band bnot constants:view_channel_permission() + end + end. + +-spec alias_permissions(permission()) -> permission(). +alias_permissions(?ALL_PERMISSIONS) -> + ?ALL_PERMISSIONS; +alias_permissions(Perms) -> + case permission_bits:has(Perms, constants:send_messages_in_threads_permission()) of + true -> Perms bor ?SEND_MESSAGES; + false -> Perms band bnot ?SEND_MESSAGES + end. + +-spec is_moderator(permission(), map() | undefined) -> boolean(). +is_moderator(?ALL_PERMISSIONS, _Member) -> + true; +is_moderator(Perms, Member) -> + permission_bits:has(Perms, constants:administrator_permission()) orelse + (permission_bits:has(Perms, constants:manage_threads_permission()) andalso + not timed_out(Member)). + +-spec can_view_thread_from_parent( + integer(), map(), permission(), map() | undefined, guild_state() +) -> + boolean(). +can_view_thread_from_parent(UserId, Thread, Perms, Member, State) -> + permission_bits:has(Perms, constants:view_channel_permission()) andalso + (not is_private(Thread) orelse is_moderator(Perms, Member) orelse + is_thread_member(UserId, Thread, State)). + +-spec is_private(map()) -> boolean(). +is_private(#{<<"type">> := 12}) -> true; +is_private(_) -> false. + +-spec parent_id(map()) -> integer() | undefined. +parent_id(Thread) -> + snowflake_id:parse_optional(maps:get(<<"parent_id">>, Thread, undefined)). + +-spec parent_channel(map(), guild_state()) -> integer() | undefined. +parent_channel(Thread, State) -> + case parent_id(Thread) of + ParentId when is_integer(ParentId) -> + case guild_permissions:find_channel_by_id(ParentId, State) of + undefined -> undefined; + _Channel -> ParentId + end; + _ -> + undefined + end. + +-spec is_thread_member(integer(), map(), guild_state()) -> boolean(). +is_thread_member(UserId, Thread, State) -> + case + { + guild_thread_gate:store(State), + snowflake_id:parse_optional(maps:get(<<"id">>, Thread, undefined)) + } + of + {Tab, ThreadId} when Tab =/= undefined, is_integer(ThreadId) -> + case guild_thread_store:get_thread(Tab, ThreadId) of + undefined -> listed_member(UserId, Thread); + _ -> guild_thread_store:is_member(Tab, ThreadId, UserId) + end; + _ -> + listed_member(UserId, Thread) + end. + +-spec listed_member(integer(), map()) -> boolean(). +listed_member(UserId, Thread) -> + lists:member( + UserId, snowflake_id:parse_list(maps:get(<<"_fluxer_member_ids">>, Thread, [])) + ). + +-spec timed_out(map() | undefined) -> boolean(). +timed_out(#{<<"communication_disabled_until">> := Until}) when is_binary(Until) -> + try calendar:rfc3339_to_system_time(binary_to_list(Until), [{unit, millisecond}]) of + UntilMs -> UntilMs > erlang:system_time(millisecond) + catch + _:_ -> false + end; +timed_out(_) -> + false. + +-spec note_unknown_parent() -> ok. +note_unknown_parent() -> + guild_thread_store:add_counter(?UNKNOWN_PARENT_KEY, 1, 1). + +-spec init_counters() -> ok. +init_counters() -> + guild_thread_store:ensure_counter(?UNKNOWN_PARENT_KEY, 1). + +-spec unknown_parent_count() -> non_neg_integer(). +unknown_parent_count() -> + case persistent_term:get(?UNKNOWN_PARENT_KEY, undefined) of + undefined -> 0; + Counter -> counters:get(Counter, 1) + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +alias_maps_send_messages_to_threads_bit_test() -> + InThreads = constants:send_messages_in_threads_permission(), + ?assertEqual(InThreads bor ?SEND_MESSAGES, alias_permissions(InThreads)), + ?assertEqual(1024, alias_permissions(1024 bor ?SEND_MESSAGES)), + ?assertEqual(?ALL_PERMISSIONS, alias_permissions(?ALL_PERMISSIONS)). + +moderator_rules_test() -> + Manage = constants:manage_threads_permission(), + ?assert(is_moderator(Manage, #{})), + ?assert(is_moderator(constants:administrator_permission(), #{})), + ?assertNot(is_moderator(1024, #{})), + ?assertNot( + is_moderator(Manage, #{<<"communication_disabled_until">> => <<"2999-01-01T00:00:00Z">>}) + ), + ?assert( + is_moderator(Manage, #{<<"communication_disabled_until">> => <<"2000-01-01T00:00:00Z">>}) + ). + +shared_case_table_test() -> + {ok, Contents} = file:read_file("test/thread_permission_cases.json"), + Cases = [ + Case + || #{<<"fn">> := Fn} = Case <- json:decode(Contents), + Fn =:= <<"isThreadModerator">> orelse Fn =:= <<"canViewThread">> + ], + ?assertEqual(16, length(Cases)), + lists:foreach(fun run_case/1, Cases). + +run_case( + #{<<"name">> := Name, <<"fn">> := Fn, <<"actor">> := Actor, <<"expect">> := Expect} = Case +) -> + Perms = case_perms(Actor), + Member = case_member(Actor), + Thread = case_thread(Actor, maps:get(<<"thread">>, Case, #{})), + Actual = + case Fn of + <<"isThreadModerator">> -> is_moderator(Perms, Member); + <<"canViewThread">> -> can_view_thread_from_parent(1, Thread, Perms, Member, #{}) + end, + ?assertEqual({Name, case_expect(Expect)}, {Name, Actual}). + +case_perms(#{<<"isOwner">> := true}) -> + ?ALL_PERMISSIONS; +case_perms(#{<<"perms">> := Names}) -> + case lists:member(<<"ADMINISTRATOR">>, Names) of + true -> ?ALL_PERMISSIONS; + false -> lists:foldl(fun(N, Acc) -> Acc bor case_bit(N) end, 0, Names) + end. + +case_bit(<<"VIEW_CHANNEL">>) -> constants:view_channel_permission(); +case_bit(<<"READ_MESSAGE_HISTORY">>) -> 65536; +case_bit(<<"MANAGE_THREADS">>) -> constants:manage_threads_permission(); +case_bit(<<"SEND_MESSAGES_IN_THREADS">>) -> constants:send_messages_in_threads_permission(). + +case_member(#{<<"timedOut">> := true}) -> + #{<<"communication_disabled_until">> => <<"2999-01-01T00:00:00Z">>}; +case_member(_) -> + #{}. + +case_thread(Actor, Thread) -> + MemberIds = + case maps:get(<<"isMember">>, Actor, false) of + true -> [<<"1">>]; + false -> [] + end, + #{ + <<"id">> => <<"2">>, + <<"type">> => maps:get(<<"type">>, Thread, 11), + <<"_fluxer_member_ids">> => MemberIds + }. + +case_expect(Bool) when is_boolean(Bool) -> Bool; +case_expect(null) -> true; +case_expect(<<"MISSING_ACCESS">>) -> false. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_push.erl b/fluxer_gateway/src/guild/guild_thread_push.erl new file mode 100644 index 000000000..26bd9f6a3 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_push.erl @@ -0,0 +1,308 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_push). +-typing([eqwalizer]). + +-export([ + scope/2, + push_message_data/4, + forum_starter/3, + mark_forum_thread_created/1, + mention_universe/2, + passive_threads/3 +]). + +-export_type([scope/0]). + +-define(THREAD_PUSH_KEY, <<"__thread_push">>). +-define(FORUM_THREAD_CREATED_KEY, <<"__forum_thread_created">>). +-define(MEMBER_SETTING_KEYS, [<<"flags">>, <<"muted">>, <<"mute_config">>]). + +-type user_id() :: integer(). +-type scope() :: #{thread := map(), members := #{user_id() => map()}} | undefined. + +-spec scope(integer(), map()) -> scope(). +scope(ChannelId, State) -> + case guild_thread_gate:store(State) of + undefined -> undefined; + Tab -> safe_scope(Tab, ChannelId) + end. + +-spec safe_scope(ets:table(), integer()) -> scope(). +safe_scope(Tab, ChannelId) -> + try guild_thread_store:get_thread(Tab, ChannelId) of + undefined -> + undefined; + Thread -> + #{ + thread => Thread, + members => maps:from_list(guild_thread_store:members(Tab, ChannelId)) + } + catch + error:badarg -> undefined + end. + +-spec push_message_data(map(), integer(), [user_id()], map()) -> {map(), binary()} | undefined. +push_message_data(MessageData, ChannelId, UserIds, Data) -> + case scope(ChannelId, Data) of + undefined -> + undefined; + #{thread := Thread, members := Members} -> + ParentId = snowflake_id:parse_optional( + maps:get(<<"parent_id">>, Thread, undefined) + ), + Parent = parent_channel(ParentId, Data), + {ForumThreadCreated, MessageData1} = take_forum_thread_created(MessageData), + ThreadPush = #{ + <<"parent_id">> => ParentId, + <<"parent_name">> => binary_or_undefined( + maps:get(<<"name">>, Parent, undefined) + ), + <<"category_id">> => snowflake_id:parse_optional( + maps:get(<<"parent_id">>, Parent, undefined) + ), + <<"members">> => member_settings(UserIds, Members) + }, + { + MessageData1#{ + ?THREAD_PUSH_KEY => with_forum_thread_created( + ForumThreadCreated, ThreadPush + ) + }, + thread_name(Thread) + } + end. + +-spec take_forum_thread_created(map()) -> {boolean(), map()}. +take_forum_thread_created(MessageData) -> + case maps:take(?FORUM_THREAD_CREATED_KEY, MessageData) of + {true, Rest} -> {true, Rest}; + {_Other, Rest} -> {false, Rest}; + error -> {false, MessageData} + end. + +-spec with_forum_thread_created(boolean(), map()) -> map(). +with_forum_thread_created(true, ThreadPush) -> + ThreadPush#{<<"forum_thread_created">> => true}; +with_forum_thread_created(false, ThreadPush) -> + ThreadPush. + +-spec mark_forum_thread_created(map()) -> map(). +mark_forum_thread_created(MessageData) -> + MessageData#{?FORUM_THREAD_CREATED_KEY => true}. + +-spec forum_starter(map(), integer(), map()) -> {integer(), #{user_id() => map()}} | undefined. +forum_starter(MessageData, ChannelId, Data) -> + case snowflake_id:parse_maybe(maps:get(<<"id">>, MessageData, undefined)) of + ChannelId -> forum_starter_scope(scope(ChannelId, Data), Data); + _ -> undefined + end. + +-spec forum_starter_scope(scope(), map()) -> {integer(), #{user_id() => map()}} | undefined. +forum_starter_scope(#{thread := Thread, members := Members}, Data) -> + ParentId = snowflake_id:parse_maybe(maps:get(<<"parent_id">>, Thread, undefined)), + case + guild_thread_gate:is_thread_only_type( + maps:get(<<"type">>, parent_channel(ParentId, Data), undefined) + ) + of + true when is_integer(ParentId) -> {ParentId, Members}; + _ -> undefined + end; +forum_starter_scope(undefined, _Data) -> + undefined. + +-spec parent_channel(integer() | undefined, map()) -> map(). +parent_channel(ParentId, Data) when is_integer(ParentId) -> + case maps:get(ParentId, guild_data_index:channel_index(Data), undefined) of + Parent when is_map(Parent) -> Parent; + _ -> #{} + end; +parent_channel(_ParentId, _Data) -> + #{}. + +-spec member_settings([user_id()], #{user_id() => map()}) -> #{user_id() => map()}. +member_settings(UserIds, Members) -> + maps:map( + fun(_UserId, Member) -> maps:with(?MEMBER_SETTING_KEYS, Member) end, + maps:with(UserIds, Members) + ). + +-spec thread_name(map()) -> binary(). +thread_name(Thread) -> + case binary_or_undefined(maps:get(<<"name">>, Thread, undefined)) of + undefined -> <<"unknown">>; + Name -> Name + end. + +-spec binary_or_undefined(term()) -> binary() | undefined. +binary_or_undefined(Value) when is_binary(Value), byte_size(Value) > 0 -> Value; +binary_or_undefined(_Value) -> undefined. + +-spec mention_universe(integer(), map()) -> #{user_id() => true} | undefined. +mention_universe(ChannelId, State) -> + case scope(ChannelId, State) of + undefined -> + undefined; + #{members := Members} -> + Recipient = guild_thread_gate:recipient_active(ChannelId, State), + maps:from_keys([UserId || UserId <- maps:keys(Members), Recipient(UserId)], true) + end. + +-spec passive_threads(map(), integer(), map()) -> [map()]. +passive_threads(SessionData, UserId, State) -> + case {guild_thread_gate:session_viewer(SessionData), guild_thread_gate:store(State)} of + {true, Tab} when Tab =/= undefined -> joined_threads(Tab, UserId); + _ -> [] + end. + +-spec joined_threads(ets:table(), integer()) -> [map()]. +joined_threads(Tab, UserId) -> + try + [ + Thread + || ThreadId <- guild_thread_store:user_thread_ids(Tab, UserId), + Thread <- [guild_thread_store:get_thread(Tab, ThreadId)], + is_map(Thread) + ] + catch + error:badarg -> [] + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +with_store(Fun) -> + Tab = guild_thread_store:new(), + try + guild_thread_store:load( + Tab, + [ + #{ + <<"id">> => 500, + <<"parent_id">> => 200, + <<"type">> => 11, + <<"name">> => <<"t">> + } + ], + [ + #{<<"id">> => <<"500">>, <<"user_id">> => <<"1">>, <<"flags">> => 2}, + #{ + <<"id">> => <<"500">>, + <<"user_id">> => <<"2">>, + <<"flags">> => 1, + <<"muted">> => true, + <<"join_timestamp">> => <<"x">> + } + ] + ), + Fun(#{ + thread_gate => #{active => true, version => 1}, + thread_store => Tab, + <<"channels">> => [ + #{<<"id">> => 200, <<"name">> => <<"general">>, <<"parent_id">> => 100}, + #{<<"id">> => 100, <<"name">> => <<"cat">>, <<"type">> => 4} + ] + }) + after + guild_thread_store:destroy(Tab) + end. + +scope_is_undefined_for_plain_channels_and_inactive_guilds_test() -> + ?assertEqual(undefined, scope(200, #{<<"channels">> => []})), + with_store(fun(Data) -> + ?assertEqual(undefined, scope(200, Data)), + ?assertEqual(undefined, scope(500, Data#{thread_gate => #{active => false}})), + ?assertMatch(#{members := #{1 := _, 2 := _}}, scope(500, Data)) + end). + +push_message_data_stamps_parent_and_member_settings_test() -> + with_store(fun(Data) -> + ?assertEqual(undefined, push_message_data(#{}, 200, [1], Data)), + {MessageData, Name} = push_message_data(#{<<"id">> => <<"9">>}, 500, [2, 3], Data), + ?assertEqual(<<"t">>, Name), + ?assertEqual( + #{ + <<"parent_id">> => 200, + <<"parent_name">> => <<"general">>, + <<"category_id">> => 100, + <<"members">> => #{2 => #{<<"flags">> => 1, <<"muted">> => true}} + }, + maps:get(?THREAD_PUSH_KEY, MessageData) + ) + end). + +with_forum_store(Fun) -> + Tab = guild_thread_store:new(), + try + guild_thread_store:load( + Tab, + [ + #{ + <<"id">> => 700, + <<"parent_id">> => 300, + <<"type">> => 11, + <<"name">> => <<"post">> + }, + #{ + <<"id">> => 500, + <<"parent_id">> => 200, + <<"type">> => 11, + <<"name">> => <<"t">> + } + ], + [#{<<"id">> => <<"700">>, <<"user_id">> => <<"1">>, <<"flags">> => 1}] + ), + Fun(#{ + thread_gate => #{active => true, version => 1}, + thread_store => Tab, + <<"channels">> => [ + #{<<"id">> => 300, <<"name">> => <<"ideas">>, <<"type">> => 15}, + #{<<"id">> => 200, <<"name">> => <<"general">>, <<"type">> => 0} + ] + }) + after + guild_thread_store:destroy(Tab) + end. + +forum_starter_needs_the_starter_message_of_a_forum_post_test() -> + with_forum_store(fun(Data) -> + ?assertMatch({300, #{1 := _}}, forum_starter(#{<<"id">> => <<"700">>}, 700, Data)), + ?assertEqual(undefined, forum_starter(#{<<"id">> => <<"701">>}, 700, Data)), + ?assertEqual(undefined, forum_starter(#{<<"id">> => <<"500">>}, 500, Data)), + ?assertEqual( + undefined, + forum_starter(#{<<"id">> => <<"700">>}, 700, Data#{ + thread_gate => #{active => false} + }) + ) + end). + +forum_thread_created_marker_moves_into_the_thread_push_test() -> + with_forum_store(fun(Data) -> + Marked = mark_forum_thread_created(#{<<"id">> => <<"700">>}), + {MessageData, <<"post">>} = push_message_data(Marked, 700, [2], Data), + ?assertNot(maps:is_key(<<"__forum_thread_created">>, MessageData)), + ?assertMatch( + #{ + <<"parent_id">> := 300, + <<"parent_name">> := <<"ideas">>, + <<"forum_thread_created">> := true + }, + maps:get(?THREAD_PUSH_KEY, MessageData) + ), + {Plain, _} = push_message_data(#{<<"id">> => <<"701">>}, 700, [2], Data), + ?assertNot(maps:is_key(<<"forum_thread_created">>, maps:get(?THREAD_PUSH_KEY, Plain))) + end). + +passive_threads_need_a_viewer_session_test() -> + with_store(fun(Data) -> + State = #{data => Data}, + ?assertEqual([], passive_threads(#{thread_viewer => false}, 1, State)), + ?assertEqual([], passive_threads(#{thread_viewer => true}, 3, State)), + ?assertMatch( + [#{<<"id">> := 500}], passive_threads(#{thread_viewer => true}, 1, State) + ) + end). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_store.erl b/fluxer_gateway/src/guild/guild_thread_store.erl new file mode 100644 index 000000000..1ee4cd049 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_store.erl @@ -0,0 +1,436 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_store). +-typing([eqwalizer]). + +-export([ + init/0, + ensure_counter/2, + add_counter/3, + new/0, + destroy/1, + retire/1, + load/3, + put_thread/2, + get_thread/2, + remove_thread/2, + put_member/4, + get_member/3, + remove_member/3, + is_member/3, + member_ids/2, + members/2, + user_thread_ids/2, + parent_thread_ids/2, + threads/1, + thread_count/1, + total_threads/0, + row_count/1, + remove_user/2, + export/1, + import/1 +]). + +-export_type([tab/0, thread_id/0, user_id/0]). + +-define(REGISTRY, guild_thread_store_registry). +-define(RETIRE_DELAY_MS, 60000). +-define(MAX_THREADS, 5000). + +-type tab() :: ets:table(). +-type thread_id() :: integer(). +-type user_id() :: integer(). +-type row() :: + {{t, thread_id()}, map()} + | {{m, thread_id(), user_id()}, map()} + | {{u, user_id(), thread_id()}, true} + | {{p, integer(), thread_id()}, true}. + +-spec init() -> ok. +init() -> + _ = + case ets:info(?REGISTRY) of + undefined -> + ets:new(?REGISTRY, [named_table, public, set, {write_concurrency, true}]); + _ -> + ok + end, + ok = guild_thread_permissions:init_counters(), + ok = guild_thread_dispatch:init_counters(), + ok = guild_thread_load:init_counters(), + ok = guild_thread_flip:init_counters(), + ok = guild_thread_subscriptions:init_counters(). + +-spec ensure_counter(term(), pos_integer()) -> ok. +ensure_counter(Key, Size) -> + case persistent_term:get(Key, undefined) of + undefined -> persistent_term:put(Key, counters:new(Size, [atomics])); + _Counters -> ok + end. + +-spec add_counter(term(), pos_integer(), integer()) -> ok. +add_counter(Key, Index, N) -> + case persistent_term:get(Key, undefined) of + undefined -> ok; + Counters -> counters:add(Counters, Index, N) + end. + +-spec new() -> tab(). +new() -> + Tab = ets:new(guild_thread_store, [ordered_set, protected, {read_concurrency, true}]), + ok = registry(fun() -> ets:insert(?REGISTRY, {Tab, true}) end), + Tab. + +-spec destroy(tab() | undefined) -> ok. +destroy(undefined) -> + ok; +destroy(Tab) -> + ok = registry(fun() -> ets:delete(?REGISTRY, Tab) end), + try ets:delete(Tab) of + _ -> ok + catch + error:badarg -> ok + end. + +-spec retire(tab() | undefined) -> ok. +retire(undefined) -> + ok; +retire(Tab) -> + ok = registry(fun() -> ets:delete(?REGISTRY, Tab) end), + _ = erlang:send_after(?RETIRE_DELAY_MS, self(), {thread_store_retire, Tab}), + ok. + +-spec registry(fun(() -> true)) -> ok. +registry(Fun) -> + try Fun() of + true -> ok + catch + error:badarg -> ok + end. + +-spec load(tab(), [map()], [map()]) -> ok. +load(Tab, Threads, Members) -> + lists:foreach(fun(Thread) -> insert_thread(Tab, Thread) end, Threads), + ok = trim(Tab, thread_count(Tab) - ?MAX_THREADS), + lists:foreach(fun(Member) -> put_member_row(Tab, Member) end, Members). + +-spec put_member_row(tab(), map()) -> ok. +put_member_row(Tab, Member) -> + case + { + snowflake_id:parse_optional(maps:get(<<"id">>, Member, undefined)), + snowflake_id:parse_optional(maps:get(<<"user_id">>, Member, undefined)) + } + of + {ThreadId, UserId} when is_integer(ThreadId), is_integer(UserId) -> + case ets:member(Tab, {t, ThreadId}) of + true -> put_member(Tab, ThreadId, UserId, Member); + false -> ok + end; + _ -> + ok + end. + +-spec put_thread(tab(), map()) -> ok. +put_thread(Tab, Thread) -> + case insert_thread(Tab, Thread) of + new -> enforce_cap(Tab); + _ -> ok + end. + +-spec insert_thread(tab(), map()) -> new | known | invalid. +insert_thread(Tab, Thread) -> + case snowflake_id:parse_optional(maps:get(<<"id">>, Thread, undefined)) of + ThreadId when is_integer(ThreadId) -> + Known = ets:member(Tab, {t, ThreadId}), + true = ets:insert(Tab, {{t, ThreadId}, Thread}), + case snowflake_id:parse_optional(maps:get(<<"parent_id">>, Thread, undefined)) of + ParentId when is_integer(ParentId) -> + true = ets:insert(Tab, {{p, ParentId, ThreadId}, true}); + _ -> + true + end, + case Known of + true -> known; + false -> new + end; + _ -> + invalid + end. + +-spec enforce_cap(tab()) -> ok. +enforce_cap(Tab) -> + case row_count(Tab) > ?MAX_THREADS of + true -> trim(Tab, thread_count(Tab) - ?MAX_THREADS); + false -> ok + end. + +-spec trim(tab(), integer()) -> ok. +trim(_Tab, Excess) when Excess =< 0 -> + ok; +trim(Tab, Excess) -> + case ets:select(Tab, [{{{t, '$1'}, '_'}, [], ['$1']}], Excess) of + {Oldest, _Continuation} -> + lists:foreach(fun(ThreadId) -> remove_thread(Tab, ThreadId) end, Oldest); + '$end_of_table' -> + ok + end. + +-spec get_thread(tab(), thread_id()) -> map() | undefined. +get_thread(Tab, ThreadId) -> + case ets:lookup(Tab, {t, ThreadId}) of + [{_, Thread}] when is_map(Thread) -> Thread; + _ -> undefined + end. + +-spec remove_thread(tab(), thread_id()) -> ok. +remove_thread(Tab, ThreadId) -> + case get_thread(Tab, ThreadId) of + undefined -> + ok; + Thread -> + lists:foreach( + fun(UserId) -> remove_member(Tab, ThreadId, UserId) end, + member_ids(Tab, ThreadId) + ), + case snowflake_id:parse_optional(maps:get(<<"parent_id">>, Thread, undefined)) of + ParentId when is_integer(ParentId) -> ets:delete(Tab, {p, ParentId, ThreadId}); + _ -> true + end, + true = ets:delete(Tab, {t, ThreadId}), + ok + end. + +-spec put_member(tab(), thread_id(), user_id(), map()) -> ok. +put_member(Tab, ThreadId, UserId, Member) -> + true = ets:insert(Tab, [{{m, ThreadId, UserId}, Member}, {{u, UserId, ThreadId}, true}]), + ok. + +-spec get_member(tab(), thread_id(), user_id()) -> map() | undefined. +get_member(Tab, ThreadId, UserId) -> + case ets:lookup(Tab, {m, ThreadId, UserId}) of + [{_, Member}] when is_map(Member) -> Member; + _ -> undefined + end. + +-spec remove_member(tab(), thread_id(), user_id()) -> ok. +remove_member(Tab, ThreadId, UserId) -> + true = ets:delete(Tab, {m, ThreadId, UserId}), + true = ets:delete(Tab, {u, UserId, ThreadId}), + ok. + +-spec is_member(tab(), thread_id(), user_id()) -> boolean(). +is_member(Tab, ThreadId, UserId) -> + ets:member(Tab, {m, ThreadId, UserId}). + +-spec member_ids(tab(), thread_id()) -> [user_id()]. +member_ids(Tab, ThreadId) -> + ets:select(Tab, [{{{m, ThreadId, '$1'}, '_'}, [], ['$1']}]). + +-spec members(tab(), thread_id()) -> [{user_id(), map()}]. +members(Tab, ThreadId) -> + ets:select(Tab, [{{{m, ThreadId, '$1'}, '$2'}, [], [{{'$1', '$2'}}]}]). + +-spec user_thread_ids(tab(), user_id()) -> [thread_id()]. +user_thread_ids(Tab, UserId) -> + ets:select(Tab, [{{{u, UserId, '$1'}, '_'}, [], ['$1']}]). + +-spec parent_thread_ids(tab(), integer()) -> [thread_id()]. +parent_thread_ids(Tab, ParentId) -> + ets:select(Tab, [{{{p, ParentId, '$1'}, '_'}, [], ['$1']}]). + +-spec threads(tab()) -> [map()]. +threads(Tab) -> + ets:select(Tab, [{{{t, '_'}, '$1'}, [], ['$1']}]). + +-spec thread_count(tab()) -> non_neg_integer(). +thread_count(Tab) -> + ets:select_count(Tab, [{{{t, '_'}, '_'}, [], [true]}]). + +-spec total_threads() -> non_neg_integer(). +total_threads() -> + Tabs = + try + ets:select(?REGISTRY, [{{'$1', '_'}, [], ['$1']}]) + catch + error:badarg -> [] + end, + sum_threads(Tabs, 0). + +-spec sum_threads([tab()], non_neg_integer()) -> non_neg_integer(). +sum_threads([], Total) -> + Total; +sum_threads([Tab | Rest], Total) -> + sum_threads(Rest, Total + live_thread_count(Tab)). + +-spec live_thread_count(tab()) -> non_neg_integer(). +live_thread_count(Tab) -> + try + thread_count(Tab) + catch + error:badarg -> + ok = registry(fun() -> ets:delete(?REGISTRY, Tab) end), + 0 + end. + +-spec row_count(tab()) -> non_neg_integer(). +row_count(Tab) -> + case ets:info(Tab, size) of + Size when is_integer(Size) -> Size; + _ -> 0 + end. + +-spec remove_user(tab(), user_id()) -> [thread_id()]. +remove_user(Tab, UserId) -> + ThreadIds = user_thread_ids(Tab, UserId), + lists:foreach(fun(ThreadId) -> remove_member(Tab, ThreadId, UserId) end, ThreadIds), + ThreadIds. + +-spec export(tab()) -> [row()]. +export(Tab) -> + eqwalizer:dynamic_cast(ets:tab2list(Tab)). + +-spec import([term()]) -> tab(). +import(Rows) -> + Tab = new(), + true = ets:insert(Tab, [Row || Row <- Rows, is_valid_row(Row)]), + Tab. + +-spec is_valid_row(term()) -> boolean(). +is_valid_row({{t, T}, Thread}) when is_integer(T), is_map(Thread) -> true; +is_valid_row({{m, T, U}, Member}) when is_integer(T), is_integer(U), is_map(Member) -> true; +is_valid_row({{u, U, T}, true}) when is_integer(T), is_integer(U) -> true; +is_valid_row({{p, P, T}, true}) when is_integer(T), is_integer(P) -> true; +is_valid_row(_) -> false. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +thread(Id, Parent) -> + #{<<"id">> => Id, <<"parent_id">> => Parent, <<"type">> => 11}. + +member(Thread, User) -> + #{<<"id">> => integer_to_binary(Thread), <<"user_id">> => integer_to_binary(User)}. + +store_round_trip_test() -> + Tab = new(), + try + ok = load(Tab, [thread(10, 1), thread(11, 1), thread(20, 2)], [ + member(10, 100), member(10, 101), member(20, 100), member(99, 100) + ]), + ?assertEqual(3, thread_count(Tab)), + ?assertEqual([100, 101], member_ids(Tab, 10)), + ?assertEqual([10, 20], user_thread_ids(Tab, 100)), + ?assertEqual([10, 11], parent_thread_ids(Tab, 1)), + ?assert(is_member(Tab, 20, 100)), + ?assertNot(is_member(Tab, 99, 100)), + ok = remove_thread(Tab, 10), + ?assertEqual(undefined, get_thread(Tab, 10)), + ?assertEqual([20], user_thread_ids(Tab, 100)), + ?assertEqual([], user_thread_ids(Tab, 101)), + ?assertEqual([11], parent_thread_ids(Tab, 1)), + ?assertEqual([20], remove_user(Tab, 100)), + ?assertEqual([], member_ids(Tab, 20)) + after + destroy(Tab) + end. + +export_import_test() -> + Tab = new(), + ok = load(Tab, [thread(10, 1)], [member(10, 5)]), + Rows = export(Tab), + destroy(Tab), + Tab2 = import([junk | Rows]), + try + ?assertEqual(length(Rows), row_count(Tab2)), + ?assert(is_member(Tab2, 10, 5)), + ?assertEqual([10], parent_thread_ids(Tab2, 1)) + after + destroy(Tab2) + end. + +total_threads_counts_registered_stores_only_test() -> + ok = init(), + Before = total_threads(), + Tab = new(), + ok = load(Tab, [thread(10, 1), thread(11, 1)], []), + ?assertEqual(Before + 2, total_threads()), + Owner = spawn(fun() -> + Orphan = new(), + ok = load(Orphan, [thread(12, 1)], []), + receive + stop -> ok + end + end), + wait_for_total(Before + 3), + Owner ! stop, + wait_for_total(Before + 2), + destroy(Tab), + ?assertEqual(Before, total_threads()). + +wait_for_total(Expected) -> + wait_for_total(Expected, 50). + +wait_for_total(Expected, 0) -> + ?assertEqual(Expected, total_threads()); +wait_for_total(Expected, Attempts) -> + case total_threads() of + Expected -> + ok; + _ -> + timer:sleep(10), + wait_for_total(Expected, Attempts - 1) + end. + +retire_unregisters_at_once_and_schedules_destroy_test() -> + ok = init(), + Tab = new(), + ok = load(Tab, [thread(20, 1)], []), + Before = total_threads(), + ok = retire(Tab), + ?assertEqual(Before - 1, total_threads()), + ?assertNotEqual(undefined, ets:info(Tab)), + destroy(Tab). + +cap_keeps_newest_threads_test() -> + Tab = new(), + try + ok = load(Tab, [thread(Id, 1) || Id <- lists:seq(10, ?MAX_THREADS + 9)], [ + member(10, 7), member(11, 7) + ]), + ?assertEqual(?MAX_THREADS, thread_count(Tab)), + ok = put_thread(Tab, thread(10, 1)), + ?assertEqual(?MAX_THREADS, thread_count(Tab)), + ok = put_thread(Tab, thread(?MAX_THREADS + 10, 1)), + ?assertEqual(?MAX_THREADS, thread_count(Tab)), + ?assertEqual(undefined, get_thread(Tab, 10)), + ?assertEqual([11], user_thread_ids(Tab, 7)), + ?assertNot(lists:member(10, parent_thread_ids(Tab, 1))), + ok = put_thread(Tab, thread(5, 1)), + ?assertEqual(undefined, get_thread(Tab, 5)), + ?assertEqual(?MAX_THREADS, thread_count(Tab)), + ?assertNotEqual(undefined, get_thread(Tab, ?MAX_THREADS + 10)) + after + destroy(Tab) + end. + +load_over_cap_keeps_newest_ids_test() -> + Tab = new(), + try + Ids = lists:seq(?MAX_THREADS + 10, 1, -1), + ok = load(Tab, [thread(Id, 1) || Id <- Ids], [member(5, 9), member(20, 9)]), + ?assertEqual(?MAX_THREADS, thread_count(Tab)), + ?assertEqual(undefined, get_thread(Tab, 10)), + ?assertNotEqual(undefined, get_thread(Tab, 11)), + ?assertEqual([20], user_thread_ids(Tab, 9)) + after + destroy(Tab) + end. + +destroy_is_idempotent_test() -> + Tab = new(), + ok = destroy(Tab), + ok = destroy(Tab), + ok = destroy(undefined). + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_subscriptions.erl b/fluxer_gateway/src/guild/guild_thread_subscriptions.erl new file mode 100644 index 000000000..5f40d133f --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_subscriptions.erl @@ -0,0 +1,732 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_subscriptions). +-typing([eqwalizer]). + +-export([ + update/3, + handle_tick/2, + handle_drain/1, + clear_on_passive/1, + access_gained/4, + member_lists_changed/2, + resync_after_load/1, + presence_changed/2, + handle_list_flush/1, + refresh_presence_users/1, + export_handoff/1, + restore_handoff/1, + list_sync_counts/0, + init_counters/0 +]). + +-define(MAX_MEMBER_LISTS, 10). +-define(MAX_LIST_MEMBERS, 1000). +-define(MAX_PRESENCE_USERS, 1000). +-define(SYNC_JITTER_MS, 5000). +-define(SYNC_RATE, 20). +-define(SYNC_RETRY_MS, 100). +-define(LIST_FLUSH_MS, 250). +-define(COUNTS_KEY, {?MODULE, list_sync}). + +-type guild_state() :: map(). + +-spec update(binary(), map(), guild_state()) -> guild_state(). +update(SessionId, Subscriptions, State) -> + Sessions = maps:get(sessions, State, #{}), + case maps:get(SessionId, Sessions, undefined) of + Session when is_map(Session) -> + case guild_thread_gate:session_viewer(Session) of + true -> apply_update(SessionId, Session, Subscriptions, State); + false -> State + end; + _ -> + State + end. + +-spec apply_update(binary(), map(), map(), guild_state()) -> guild_state(). +apply_update(SessionId, Session0, Subscriptions, State) -> + {Session1, SyncNeeded} = apply_threads_flag( + maps:get(threads, Subscriptions, undefined), Session0 + ), + {Session2, NewLists} = apply_member_lists( + maps:get(member_lists, Subscriptions, undefined), Session1, State + ), + Sessions = maps:get(sessions, State, #{}), + State1 = refresh_presence_users( + maps:get(thread_member_lists, Session0, []) =/= + maps:get(thread_member_lists, Session2, []), + State#{sessions => Sessions#{SessionId => Session2}} + ), + lists:foreach(fun(ThreadId) -> send_member_list(Session2, ThreadId, State1) end, NewLists), + case SyncNeeded of + true -> schedule_sync(SessionId, State1); + false -> State1 + end. + +-spec apply_threads_flag(term(), map()) -> {map(), boolean()}. +apply_threads_flag(true, Session) -> + {Session#{thread_subscribed => true}, maps:get(thread_subscribed, Session, false) =/= true}; +apply_threads_flag(false, Session) -> + {maps:remove(thread_subscribed, Session), false}; +apply_threads_flag(_Other, Session) -> + {Session, false}. + +-spec apply_member_lists(term(), map(), guild_state()) -> {map(), [integer()]}. +apply_member_lists(Ids, Session, State) when is_list(Ids) -> + Previous = maps:get(thread_member_lists, Session, []), + Viewable = lists:sublist( + [Id || Id <- lists:usort(Ids), viewable_thread(Session, Id, State)], ?MAX_MEMBER_LISTS + ), + Updated = + case Viewable of + [] -> maps:remove(thread_member_lists, Session); + _ -> Session#{thread_member_lists => Viewable} + end, + {Updated, Viewable -- Previous}; +apply_member_lists(_Other, Session, _State) -> + {Session, []}. + +-spec viewable_thread(map(), integer(), guild_state()) -> boolean(). +viewable_thread(Session, ThreadId, State) -> + case guild_thread_gate:thread(ThreadId, State) of + undefined -> false; + Thread -> session_can_view(Session, Thread, State) + end. + +-spec session_can_view(map(), map(), guild_state()) -> boolean(). +session_can_view(#{user_id := UserId}, Thread, State) when is_integer(UserId) -> + case guild_permissions:find_member_by_user_id(UserId, State) of + undefined -> false; + Member -> guild_thread_view:can_view(UserId, Thread, Member, State) + end; +session_can_view(_Session, _Thread, _State) -> + false. + +-spec schedule_sync(binary(), guild_state()) -> guild_state(). +schedule_sync(SessionId, State) -> + Pending = maps:get(thread_sync_pending, State, #{}), + case maps:is_key(SessionId, Pending) of + true -> + State; + false -> + _ = erlang:send_after( + rand:uniform(?SYNC_JITTER_MS) - 1, self(), {thread_list_sync_tick, SessionId} + ), + State#{thread_sync_pending => Pending#{SessionId => scheduled}} + end. + +-spec export_handoff(guild_state()) -> map(). +export_handoff(State) -> + maps:merge( + case State of + #{thread_sync_pending := Pending} -> #{thread_handoff_syncs => maps:keys(Pending)}; + _ -> #{} + end, + case State of + #{thread_list_dirty := Dirty} -> #{thread_handoff_lists => maps:keys(Dirty)}; + _ -> #{} + end + ). + +-spec restore_handoff(guild_state()) -> guild_state(). +restore_handoff(State0) -> + State = maps:without([thread_handoff_syncs, thread_handoff_lists], State0), + State1 = + case State0 of + #{thread_handoff_syncs := SessionIds} when is_list(SessionIds) -> + lists:foldl( + fun schedule_sync/2, State, [Sid || Sid <- SessionIds, is_binary(Sid)] + ); + _ -> + State + end, + case [Id || Id <- maps:get(thread_handoff_lists, State0, []), is_integer(Id)] of + [] -> State1; + ThreadIds -> mark_dirty(ThreadIds, State1#{thread_list_refresh => true}) + end. + +-spec handle_tick(term(), guild_state()) -> guild_state(). +handle_tick(SessionId, State) when is_binary(SessionId) -> + Pending = maps:get(thread_sync_pending, State, #{}), + case maps:get(SessionId, Pending, undefined) of + queued -> + drain(State); + _ -> + Queue = maps:get(thread_sync_queue, State, queue:new()), + drain(State#{ + thread_sync_pending => Pending#{SessionId => queued}, + thread_sync_queue => queue:in(SessionId, Queue) + }) + end; +handle_tick(_SessionId, State) -> + State. + +-spec handle_drain(guild_state()) -> guild_state(). +handle_drain(State) -> + drain(maps:remove(thread_sync_timer, State)). + +-spec drain(guild_state()) -> guild_state(). +drain(State) -> + Queue = maps:get(thread_sync_queue, State, queue:new()), + case queue:out(Queue) of + {empty, _} -> + maps:remove(thread_sync_queue, State); + {{value, SessionId}, Rest} -> + Now = erlang:monotonic_time(millisecond), + case take_token(Now, maps:get(thread_sync_bucket, State, {?SYNC_RATE, Now})) of + {ok, Bucket} -> + send_full_sync(SessionId, State), + Pending = maps:remove(SessionId, maps:get(thread_sync_pending, State, #{})), + drain( + put_pending(Pending, State#{ + thread_sync_bucket => Bucket, thread_sync_queue => Rest + }) + ); + {wait, {Tokens, _} = Bucket} -> + arm_drain(ceil((1 - Tokens) * 1000 / ?SYNC_RATE), State#{ + thread_sync_bucket => Bucket + }) + end + end. + +-spec put_pending(map(), guild_state()) -> guild_state(). +put_pending(Pending, State) when map_size(Pending) =:= 0 -> + maps:remove(thread_sync_pending, State); +put_pending(Pending, State) -> + State#{thread_sync_pending => Pending}. + +-spec arm_drain(integer(), guild_state()) -> guild_state(). +arm_drain(_DelayMs, #{thread_sync_timer := true} = State) -> + State; +arm_drain(DelayMs, State) -> + _ = erlang:send_after(max(1, DelayMs), self(), thread_list_sync_drain), + State#{thread_sync_timer => true}. + +-spec take_token(integer(), {number(), integer()}) -> + {ok, {number(), integer()}} | {wait, {number(), integer()}}. +take_token(Now, {Tokens0, Last}) -> + Tokens = min(?SYNC_RATE, Tokens0 + (Now - Last) * ?SYNC_RATE / 1000), + case Tokens >= 1 of + true -> {ok, {Tokens - 1, Now}}; + false -> {wait, {Tokens, Now}} + end. + +-spec send_full_sync(binary(), guild_state()) -> ok. +send_full_sync(SessionId, State) -> + case maps:get(SessionId, maps:get(sessions, State, #{}), undefined) of + #{thread_viewer := true, pid := Pid, user_id := UserId} = Session when + is_pid(Pid) + -> + case guild_availability:is_guild_unavailable_for_user(UserId, State) of + true -> + ok; + false -> + send_list_sync(Pid, UserId, sync_scope(Session), State), + count(full) + end; + _ -> + ok + end. + +-spec sync_scope(map()) -> all | joined. +sync_scope(#{thread_subscribed := true}) -> all; +sync_scope(#{bot := true}) -> all; +sync_scope(_Session) -> joined. + +-spec resync_after_load(guild_state()) -> guild_state(). +resync_after_load(State) -> + Tab = guild_thread_gate:store(State), + State1 = refresh_presence_users(State), + maps:fold( + fun(SessionId, Session, Acc) -> + case needs_resync(Session, Tab) of + true -> schedule_sync(SessionId, Acc); + false -> Acc + end + end, + mark_dirty(maps:keys(maps:get(thread_list_sessions, State1, #{})), State1), + maps:get(sessions, State, #{}) + ). + +-spec needs_resync(term(), ets:table() | undefined) -> boolean(). +needs_resync(_Session, undefined) -> + false; +needs_resync(#{thread_viewer := true, pending_connect := true}, _Tab) -> + false; +needs_resync(#{thread_viewer := true, user_id := UserId} = Session, Tab) -> + sync_scope(Session) =:= all orelse guild_thread_store:user_thread_ids(Tab, UserId) =/= []; +needs_resync(_Session, _Tab) -> + false. + +-spec send_list_sync(pid(), integer(), [integer()] | all | joined, guild_state()) -> ok. +send_list_sync(Pid, UserId, Parents, #{id := GuildId} = State) -> + Member = guild_permissions:find_member_by_user_id(UserId, State), + Payload = guild_thread_view:list_sync(GuildId, UserId, Parents, Member, State), + gateway_dispatch_relay:dispatch(Pid, thread_list_sync, Payload, GuildId). + +-spec clear_on_passive(map()) -> map(). +clear_on_passive(Session) -> + maps:remove(thread_subscribed, Session). + +-spec access_gained(map(), [integer()], guild_state(), integer()) -> ok. +access_gained(Session, AddedChannelIds, State, _GuildId) -> + case guild_thread_gate:session_viewer(Session) andalso guild_thread_gate:store(State) of + false -> + ok; + undefined -> + ok; + Tab -> + Parents = [ + P + || P <- AddedChannelIds, guild_thread_store:parent_thread_ids(Tab, P) =/= [] + ], + case {Parents, Session} of + {[], _} -> + ok; + {_, #{pid := Pid, user_id := UserId}} when is_pid(Pid) -> + count(access), + send_list_sync(Pid, UserId, Parents, State); + _ -> + ok + end + end. + +-spec member_lists_changed(integer(), guild_state()) -> guild_state(). +member_lists_changed(ThreadId, #{thread_list_sessions := Index} = State) when + is_map_key(ThreadId, Index) +-> + mark_dirty([ThreadId], State#{thread_list_refresh => true}); +member_lists_changed(_ThreadId, State) -> + State. + +-spec presence_changed(integer(), guild_state()) -> guild_state(). +presence_changed(UserId, #{thread_presence_users := Users} = State) -> + mark_dirty(maps:get(UserId, Users, []), State); +presence_changed(_UserId, State) -> + State. + +-spec mark_dirty([integer()], guild_state()) -> guild_state(). +mark_dirty([], State) -> + State; +mark_dirty(ThreadIds, State) -> + Dirty = maps:get(thread_list_dirty, State, #{}), + arm_list_flush(State#{ + thread_list_dirty => maps:merge(Dirty, maps:from_keys(ThreadIds, true)) + }). + +-spec arm_list_flush(guild_state()) -> guild_state(). +arm_list_flush(#{thread_list_flush := true} = State) -> + State; +arm_list_flush(State) -> + _ = erlang:send_after(?LIST_FLUSH_MS, self(), thread_member_list_flush), + State#{thread_list_flush => true}. + +-spec handle_list_flush(guild_state()) -> guild_state(). +handle_list_flush(State0) -> + case guild_thread_load:loading(State0) of + true -> maps:remove(thread_list_flush, State0); + false -> flush_member_lists(State0) + end. + +-spec flush_member_lists(guild_state()) -> guild_state(). +flush_member_lists(State0) -> + Dirty = maps:keys(maps:get(thread_list_dirty, State0, #{})), + State = refresh_presence_users( + maps:is_key(thread_list_refresh, State0) orelse stale_list_sessions(Dirty, State0), + maps:without([thread_list_dirty, thread_list_flush, thread_list_refresh], State0) + ), + ok = send_member_lists(Dirty, State), + State. + +-spec stale_list_sessions([integer()], guild_state()) -> boolean(). +stale_list_sessions(ThreadIds, State) -> + Index = maps:get(thread_list_sessions, State, #{}), + Sessions = maps:get(sessions, State, #{}), + lists:any( + fun(ThreadId) -> + lists:any( + fun(Sid) -> not lists_thread(ThreadId, maps:get(Sid, Sessions, undefined)) end, + maps:get(ThreadId, Index, []) + ) + end, + ThreadIds + ). + +-spec lists_thread(integer(), term()) -> boolean(). +lists_thread(ThreadId, #{thread_member_lists := Lists}) -> + lists:member(ThreadId, Lists); +lists_thread(_ThreadId, _Session) -> + false. + +-spec refresh_presence_users(guild_state()) -> guild_state(). +refresh_presence_users(State) -> + refresh_presence_users(true, State). + +-spec refresh_presence_users(boolean(), guild_state()) -> guild_state(). +refresh_presence_users(false, State) -> + State; +refresh_presence_users(true, State) -> + ThreadIds = lists:usort( + lists:append([ + Lists + || #{thread_member_lists := Lists} <- maps:values(maps:get(sessions, State, #{})) + ]) + ), + case {ThreadIds, guild_thread_gate:store(State)} of + {[_ | _], Tab} when Tab =/= undefined -> + State#{ + thread_presence_users => presence_users(ThreadIds, Tab), + thread_list_sessions => list_sessions(maps:get(sessions, State, #{})) + }; + _ -> + maps:without([thread_presence_users, thread_list_sessions], State) + end. + +-spec list_sessions(map()) -> #{integer() => [binary()]}. +list_sessions(Sessions) -> + maps:fold( + fun + (SessionId, #{thread_member_lists := Lists}, Acc0) -> + lists:foldl( + fun(ThreadId, Acc) -> + Acc#{ThreadId => [SessionId | maps:get(ThreadId, Acc, [])]} + end, + Acc0, + Lists + ); + (_SessionId, _Session, Acc) -> + Acc + end, + #{}, + Sessions + ). + +-spec presence_users([integer()], ets:table()) -> #{integer() => [integer()]}. +presence_users(ThreadIds, Tab) -> + lists:foldl( + fun(ThreadId, Acc0) -> + lists:foldl( + fun(UserId, Acc) -> + case maps:find(UserId, Acc) of + {ok, Threads} -> + Acc#{UserId => [ThreadId | Threads]}; + error when map_size(Acc) < ?MAX_PRESENCE_USERS -> + Acc#{UserId => [ThreadId]}; + error -> + Acc + end + end, + Acc0, + lists:sublist(guild_thread_store:member_ids(Tab, ThreadId), ?MAX_LIST_MEMBERS) + ) + end, + #{}, + ThreadIds + ). + +-spec send_member_lists([integer()], guild_state()) -> ok. +send_member_lists([], _State) -> + ok; +send_member_lists(ThreadIds, #{id := GuildId} = State) -> + Index = maps:get(thread_list_sessions, State, #{}), + Sessions = maps:get(sessions, State, #{}), + Ctx = guild_member_list_connected:presence_context(State), + lists:foreach( + fun(ThreadId) -> + case list_pids(ThreadId, maps:get(ThreadId, Index, []), Sessions, State) of + [] -> + ok; + Pids -> + gateway_dispatch_relay:dispatch_many( + Pids, + thread_member_list_update, + member_list_payload(GuildId, ThreadId, Ctx, State), + GuildId + ) + end + end, + ThreadIds + ); +send_member_lists(_ThreadIds, _State) -> + ok. + +-spec list_pids(integer(), [binary()], map(), guild_state()) -> [pid()]. +list_pids(ThreadId, SessionIds, Sessions, State) -> + case guild_thread_gate:thread(ThreadId, State) of + undefined -> + []; + Thread -> + {Pids, _Memo} = lists:foldl( + fun(Sid, {Acc, Memo}) -> + list_pid( + ThreadId, Thread, maps:get(Sid, Sessions, undefined), State, Acc, Memo + ) + end, + {[], #{}}, + SessionIds + ), + Pids + end. + +-spec list_pid(integer(), map(), term(), guild_state(), [pid()], map()) -> {[pid()], map()}. +list_pid( + ThreadId, + Thread, + #{pid := Pid, thread_viewer := true, thread_member_lists := Lists} = Session, + State, + Acc, + Memo +) when is_pid(Pid) -> + case lists:member(ThreadId, Lists) of + true -> + case guild_thread_gate:thread_view_access(Session, Thread, State, Memo) of + {true, Memo1} -> {[Pid | Acc], Memo1}; + {false, Memo1} -> {Acc, Memo1} + end; + false -> + {Acc, Memo} + end; +list_pid(_ThreadId, _Thread, _Session, _State, Acc, Memo) -> + {Acc, Memo}. + +-spec send_member_list(map(), integer(), guild_state()) -> ok. +send_member_list(#{pid := Pid, thread_viewer := true}, ThreadId, #{id := GuildId} = State) when + is_pid(Pid) +-> + Ctx = guild_member_list_connected:presence_context(State), + gateway_dispatch_relay:dispatch( + Pid, + thread_member_list_update, + member_list_payload(GuildId, ThreadId, Ctx, State), + GuildId + ); +send_member_list(_Session, _ThreadId, _State) -> + ok. + +-spec member_list_payload(integer(), integer(), map(), guild_state()) -> map(). +member_list_payload(GuildId, ThreadId, Ctx, State) -> + Members = + case guild_thread_gate:store(State) of + undefined -> []; + Tab -> lists:sublist(guild_thread_store:members(Tab, ThreadId), ?MAX_LIST_MEMBERS) + end, + #{ + <<"guild_id">> => integer_to_binary(GuildId), + <<"thread_id">> => integer_to_binary(ThreadId), + <<"members">> => [list_member(UserId, M, Ctx, State) || {UserId, M} <- Members] + }. + +-spec list_member(integer(), map(), map(), guild_state()) -> map(). +list_member(UserId, ThreadMember, Ctx, State) -> + #{<<"presence">> := Presence} = guild_member_list_connected:add_presence_to_member( + #{}, UserId, Ctx + ), + GuildMember = + case guild_data_members:find_member_by_user_id(UserId, State) of + undefined -> null; + M -> M + end, + #{ + <<"user_id">> => integer_to_binary(UserId), + <<"join_timestamp">> => maps:get(<<"join_timestamp">>, ThreadMember, null), + <<"flags">> => maps:get(<<"flags">>, ThreadMember, 0), + <<"member">> => GuildMember, + <<"presence">> => Presence + }. + +-spec count(full | access) -> ok. +count(Reason) -> + guild_thread_store:add_counter(?COUNTS_KEY, reason_index(Reason), 1). + +-spec init_counters() -> ok. +init_counters() -> + guild_thread_store:ensure_counter(?COUNTS_KEY, 2). + +-spec reason_index(full | access) -> 1 | 2. +reason_index(full) -> 1; +reason_index(access) -> 2. + +-spec list_sync_counts() -> #{full | access => non_neg_integer()}. +list_sync_counts() -> + case persistent_term:get(?COUNTS_KEY, undefined) of + undefined -> #{}; + Counters -> #{R => counters:get(Counters, reason_index(R)) || R <- [full, access]} + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +token_bucket_allows_twenty_per_second_test() -> + Now = 1000, + Bucket = lists:foldl( + fun(_, B) -> + {ok, Next} = take_token(Now, B), + Next + end, + {?SYNC_RATE, Now}, + lists:seq(1, ?SYNC_RATE) + ), + ?assertMatch({wait, _}, take_token(Now, Bucket)), + ?assertMatch({ok, _}, take_token(Now + 50, Bucket)). + +queued_syncs_share_one_drain_timer_test_() -> + {spawn, fun queued_syncs_share_one_drain_timer/0}. + +queued_syncs_share_one_drain_timer() -> + Ids = [integer_to_binary(N) || N <- lists:seq(1, 1000)], + Now = erlang:monotonic_time(millisecond), + State0 = #{id => 1, sessions => #{}, thread_sync_bucket => {0, Now + 60000}}, + State1 = lists:foldl(fun schedule_sync/2, State0, Ids ++ Ids), + ?assertEqual(1000, map_size(maps:get(thread_sync_pending, State1))), + State2 = lists:foldl(fun handle_tick/2, State1, Ids ++ Ids), + ?assertEqual(1000, queue:len(maps:get(thread_sync_queue, State2))), + ?assertEqual(true, maps:get(thread_sync_timer, State2)), + State3 = handle_drain(State2#{ + thread_sync_bucket => {?SYNC_RATE, erlang:monotonic_time(millisecond)} + }), + Left = queue:len(maps:get(thread_sync_queue, State3)), + ?assert(Left =< 1000 - ?SYNC_RATE andalso Left >= 1000 - ?SYNC_RATE - 2), + ?assertEqual(true, maps:get(thread_sync_timer, State3)), + ?assertEqual(1, count_drains(300, 0)). + +presence_changes_coalesce_into_one_flush_test_() -> + {spawn, fun presence_changes_coalesce_into_one_flush/0}. + +presence_changes_coalesce_into_one_flush() -> + Me = self(), + Viewer = #{ + pid => Me, user_id => 1, thread_viewer => true, viewable_channels => #{5 => true} + }, + Sessions = #{ + <<"a">> => Viewer#{thread_member_lists => [10]}, + <<"b">> => Viewer#{user_id => 2, thread_member_lists => [10, 11]}, + <<"c">> => #{pid => Me, thread_viewer => false}, + <<"d">> => Viewer#{thread_member_lists => [12]}, + <<"e">> => Viewer#{ + user_id => 3, viewable_channels => #{}, thread_member_lists => [10, 11] + } + }, + State0 = #{ + id => 1, + data => store_data([10, 11, 12]), + sessions => Sessions, + connected_user_ids => sets:new(), + thread_presence_users => #{7 => [10], 8 => [10, 11]}, + thread_list_sessions => list_sessions(Sessions) + }, + ?assertEqual(State0, presence_changed(9, State0)), + State1 = lists:foldl(fun presence_changed/2, State0, [7, 8, 7, 8, 9]), + ?assertEqual(#{10 => true, 11 => true}, maps:get(thread_list_dirty, State1)), + ?assertEqual([], list_updates()), + receive + thread_member_list_flush -> ok + after 1000 -> error(no_flush) + end, + receive + thread_member_list_flush -> error(second_flush) + after 300 -> ok + end, + State2 = handle_list_flush(State1), + ?assertNot(maps:is_key(thread_list_dirty, State2)), + ?assertNot(maps:is_key(thread_list_flush, State2)), + ?assertEqual([<<"10">>, <<"10">>, <<"11">>], lists:sort(list_updates())), + ?assertEqual(State2, member_lists_changed(99, State2)), + ?assertMatch( + #{ + thread_list_dirty := #{12 := true}, + thread_list_refresh := true, + thread_list_flush := true + }, + member_lists_changed(12, State2) + ). + +flush_drops_presence_users_of_departed_list_sessions_test_() -> + {spawn, fun flush_drops_presence_users_of_departed_list_sessions/0}. + +flush_drops_presence_users_of_departed_list_sessions() -> + Viewer = #{ + pid => self(), + user_id => 1, + thread_viewer => true, + viewable_channels => #{5 => true}, + thread_member_lists => [10] + }, + State0 = #{ + id => 1, + data => store_data([10]), + sessions => #{<<"b">> => maps:without([thread_member_lists], Viewer)}, + connected_user_ids => sets:new(), + thread_presence_users => #{7 => [10]}, + thread_list_sessions => #{10 => [<<"a">>, <<"b">>]} + }, + State1 = presence_changed(7, State0), + receive + thread_member_list_flush -> ok + after 1000 -> error(no_flush) + end, + State2 = handle_list_flush(State1), + ?assertEqual([], list_updates()), + ?assertNot(maps:is_key(thread_presence_users, State2)), + ?assertNot(maps:is_key(thread_list_sessions, State2)), + ?assertEqual(State2, presence_changed(7, State2)). + +flush_waits_for_the_thread_load_test_() -> + {spawn, fun flush_waits_for_the_thread_load/0}. + +flush_waits_for_the_thread_load() -> + Sessions = #{ + <<"a">> => #{ + pid => self(), + user_id => 1, + thread_viewer => true, + viewable_channels => #{5 => true}, + thread_member_lists => [10] + } + }, + State0 = #{ + id => 1, + data => store_data([10]), + sessions => Sessions, + connected_user_ids => sets:new(), + thread_list_dirty => #{10 => true}, + thread_list_flush => true, + thread_load => #{status => loading, queue => [], queued => 0} + }, + State1 = handle_list_flush(State0), + ?assertEqual([], list_updates()), + ?assertEqual(#{10 => true}, maps:get(thread_list_dirty, State1)), + ?assertNot(maps:is_key(thread_list_flush, State1)), + Loaded = State1#{thread_load => #{status => ready, queue => [], queued => 0}}, + State2 = resync_after_load(maps:remove(thread_list_dirty, Loaded)), + ?assertMatch(#{thread_list_dirty := #{10 := true}, thread_list_flush := true}, State2), + State3 = handle_list_flush(State2), + ?assertEqual([<<"10">>], list_updates()), + ?assertNot(maps:is_key(thread_list_dirty, State3)). + +store_data(ThreadIds) -> + Tab = guild_thread_store:new(), + lists:foreach( + fun(Id) -> + guild_thread_store:put_thread(Tab, #{ + <<"id">> => integer_to_binary(Id), <<"parent_id">> => <<"5">>, <<"type">> => 11 + }) + end, + ThreadIds + ), + #{thread_store => Tab, thread_gate => #{active => true}}. + +list_updates() -> + receive + {'$gen_cast', {dispatch, thread_member_list_update, #{<<"thread_id">> := Id}}} -> + [Id | list_updates()] + after 0 -> [] + end. + +count_drains(Timeout, Count) -> + receive + thread_list_sync_drain -> count_drains(Timeout, Count + 1) + after Timeout -> Count + end. + +-endif. diff --git a/fluxer_gateway/src/guild/guild_thread_view.erl b/fluxer_gateway/src/guild/guild_thread_view.erl new file mode 100644 index 000000000..9d8482601 --- /dev/null +++ b/fluxer_gateway/src/guild/guild_thread_view.erl @@ -0,0 +1,187 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_view). +-typing([eqwalizer]). + +-export([ + guild_threads/4, + list_sync/5, + can_view/4, + thread_payload/1, + hide_for_non_viewer/4 +]). + +-type guild_state() :: map(). + +-spec guild_threads(integer(), map(), map() | undefined, guild_state()) -> [map()]. +guild_threads(UserId, Opts, Member, State) -> + case guild_thread_gate:store(State) of + undefined -> + []; + Tab -> + case maps:get(bot, Opts, false) of + true -> bot_threads(UserId, Member, Tab, State); + false -> joined_threads(UserId, Member, Tab, State) + end + end. + +-spec bot_threads(integer(), map() | undefined, ets:table(), guild_state()) -> [map()]. +bot_threads(UserId, Member, Tab, State) -> + [ + with_own_member(UserId, Thread, Tab) + || Thread <- guild_thread_store:threads(Tab), can_view(UserId, Thread, Member, State) + ]. + +-spec joined_threads(integer(), map() | undefined, ets:table(), guild_state()) -> [map()]. +joined_threads(UserId, Member, Tab, State) -> + lists:filtermap( + fun(ThreadId) -> + case guild_thread_store:get_thread(Tab, ThreadId) of + undefined -> + false; + Thread -> + case can_view(UserId, Thread, Member, State) of + true -> {true, with_own_member(UserId, Thread, Tab)}; + false -> false + end + end + end, + guild_thread_store:user_thread_ids(Tab, UserId) + ). + +-spec with_own_member(integer(), map(), ets:table()) -> map(). +with_own_member(UserId, Thread, Tab) -> + Payload = thread_payload(Thread), + case guild_thread_store:get_member(Tab, thread_id(Thread), UserId) of + undefined -> Payload; + ThreadMember -> Payload#{<<"member">> => own_member_payload(ThreadMember)} + end. + +-spec own_member_payload(map()) -> map(). +own_member_payload(ThreadMember) -> + maps:without([<<"id">>, <<"user_id">>, <<"member">>, <<"presence">>], ThreadMember). + +-spec thread_payload(map()) -> map(). +thread_payload(Thread) -> + maps:without([<<"member_ids_preview">>, <<"member">>], Thread). + +-spec thread_id(map()) -> integer(). +thread_id(Thread) -> + snowflake_id:parse(maps:get(<<"id">>, Thread)). + +-spec can_view(integer(), map(), map() | undefined, guild_state()) -> boolean(). +can_view(UserId, Thread, Member, State) -> + Perms = guild_thread_permissions:resolve(UserId, Thread, Member, State), + permission_bits:has(Perms, constants:view_channel_permission()). + +-spec viewable_thread_ids( + integer(), [integer()] | all | joined, map() | undefined, guild_state() +) -> + [{integer(), map()}]. +viewable_thread_ids(UserId, Parents, Member, State) -> + case guild_thread_gate:store(State) of + undefined -> + []; + Tab -> + [ + {thread_id(Thread), Thread} + || Thread <- candidate_threads(Parents, UserId, Tab), + can_view(UserId, Thread, Member, State) + ] + end. + +-spec candidate_threads([integer()] | all | joined, integer(), ets:table()) -> [map()]. +candidate_threads(all, _UserId, Tab) -> + guild_thread_store:threads(Tab); +candidate_threads(joined, UserId, Tab) -> + stored_threads(guild_thread_store:user_thread_ids(Tab, UserId), Tab); +candidate_threads(Parents, _UserId, Tab) -> + stored_threads( + lists:append([guild_thread_store:parent_thread_ids(Tab, P) || P <- Parents]), Tab + ). + +-spec stored_threads([integer()], ets:table()) -> [map()]. +stored_threads(ThreadIds, Tab) -> + lists:filtermap( + fun(ThreadId) -> + case guild_thread_store:get_thread(Tab, ThreadId) of + undefined -> false; + Thread -> {true, Thread} + end + end, + ThreadIds + ). + +-spec list_sync( + integer(), integer(), [integer()] | all | joined, map() | undefined, guild_state() +) -> + map(). +list_sync(GuildId, UserId, Parents, Member, State) -> + Viewable = viewable_thread_ids(UserId, Parents, Member, State), + Members = + case guild_thread_gate:store(State) of + undefined -> + []; + Tab -> + [ + M + || {ThreadId, _Thread} <- Viewable, + M <- [guild_thread_store:get_member(Tab, ThreadId, UserId)], + M =/= undefined + ] + end, + Base = #{ + <<"guild_id">> => integer_to_binary(GuildId), + <<"threads">> => [thread_payload(Thread) || {_Id, Thread} <- Viewable], + <<"members">> => Members + }, + case Parents of + all -> Base; + joined -> Base; + _ -> Base#{<<"channel_ids">> => [integer_to_binary(P) || P <- Parents]} + end. + +-spec hide_for_non_viewer(integer(), map() | undefined, guild_state(), [map()]) -> [map()]. +hide_for_non_viewer(UserId, Member, State, Channels) -> + Kept = [C || C <- Channels, not guild_thread_gate:is_thread_only_type(channel_type(C))], + Visible = + case length(Kept) =:= length(Channels) of + true -> Kept; + false -> drop_forum_only_categories(UserId, Member, State, Kept) + end, + [ + guild_thread_gate:mask_overwrites(guild_thread_gate:strip_channel_surface(C)) + || C <- Visible + ]. + +-spec drop_forum_only_categories(integer(), map() | undefined, guild_state(), [map()]) -> + [map()]. +drop_forum_only_categories(UserId, Member, State, Channels) -> + Parents = sets:from_list([ + P + || C <- Channels, + channel_type(C) =/= 4, + P <- [snowflake_id:parse_maybe(maps:get(<<"parent_id">>, C, undefined))], + is_integer(P) + ]), + lists:filter( + fun(C) -> + case + {channel_type(C), snowflake_id:parse_maybe(maps:get(<<"id">>, C, undefined))} + of + {4, Id} when is_integer(Id) -> + sets:is_element(Id, Parents) orelse + guild_permissions:can_view_channel_by_permissions( + UserId, Id, Member, State + ) orelse + guild_virtual_channel_access:has_virtual_access(UserId, Id, State); + _ -> + true + end + end, + Channels + ). + +-spec channel_type(map()) -> integer() | undefined. +channel_type(Channel) -> + guild_data_normalize_schema:int(maps:get(<<"type">>, Channel, undefined)). diff --git a/fluxer_gateway/src/guild/guild_unified_subscriptions.erl b/fluxer_gateway/src/guild/guild_unified_subscriptions.erl index 5823cd69b..24b933bee 100644 --- a/fluxer_gateway/src/guild/guild_unified_subscriptions.erl +++ b/fluxer_gateway/src/guild/guild_unified_subscriptions.erl @@ -17,6 +17,7 @@ -define(MAX_RANGE_SPAN, 99). -define(MAX_RANGES_PER_CHANNEL, 10). -define(MAX_MEMBER_SUBSCRIPTION_IDS, 1000). +-define(MAX_THREAD_MEMBER_LISTS, 10). -spec handle_subscriptions(term(), pid(), session_state()) -> ok. handle_subscriptions(Data, SocketPid, SessionState) when is_map(Data) -> @@ -108,8 +109,53 @@ process_guild_sub_options(GuildId, GuildPid, GuildSubData, SessionId, SocketPid, process_member_list_channels(GuildSubData, GuildId, GuildPid, SessionId, SocketPid), process_member_subscriptions(GuildSubData, GuildPid, SessionId), process_typing_flag(GuildSubData, GuildPid, SessionId), + process_thread_subscriptions(GuildSubData, GuildPid, SessionId, SessionState), ok. +-spec process_thread_subscriptions(map(), pid(), session_id(), session_state()) -> ok. +process_thread_subscriptions(GuildSubData, GuildPid, SessionId, SessionState) when + is_binary(SessionId) +-> + Capable = + maps:get(bot, SessionState, false) =:= true orelse + maps:get(thread_channels_capable, SessionState, false) =:= true, + Threads = + case maps:get(<<"threads">>, GuildSubData, undefined) of + Flag when is_boolean(Flag) -> Flag; + _ -> undefined + end, + Lists = + case maps:get(<<"thread_member_lists">>, GuildSubData, undefined) of + Ids when is_list(Ids) -> parse_thread_ids(Ids); + _ -> undefined + end, + case {Capable, Threads, Lists} of + {false, _, _} -> + ok; + {true, undefined, undefined} -> + ok; + {true, _, _} -> + _ = shard_utils:safe_cast( + GuildPid, + {thread_subscriptions, SessionId, #{threads => Threads, member_lists => Lists}} + ), + ok + end; +process_thread_subscriptions(_GuildSubData, _GuildPid, _SessionId, _SessionState) -> + ok. + +-spec parse_thread_ids(list()) -> [integer()]. +parse_thread_ids(Ids) -> + lists:sublist( + [ + Id + || Raw <- Ids, + Id <- [snowflake_id:parse_maybe(Raw)], + is_integer(Id) + ], + ?MAX_THREAD_MEMBER_LISTS + ). + -spec process_active_flag(map(), pid(), session_id(), boolean()) -> boolean(). process_active_flag(GuildSubData, GuildPid, SessionId, WasActive) -> case maps:get(<<"active">>, GuildSubData, undefined) of @@ -325,4 +371,20 @@ parse_member_ids_caps_at_max_test() -> Capped = lists:sublist(AllParsed, ?MAX_MEMBER_SUBSCRIPTION_IDS), ?assertEqual(?MAX_MEMBER_SUBSCRIPTION_IDS, length(Capped)). +thread_keys_are_forwarded_only_for_capable_sessions_test() -> + Sub = #{<<"threads">> => true, <<"thread_member_lists">> => [<<"5">>, <<"x">>]}, + ok = process_thread_subscriptions(Sub, self(), <<"s">>, #{}), + ok = process_thread_subscriptions(Sub, self(), <<"s">>, #{thread_channels_capable => true}), + receive + {'$gen_cast', {thread_subscriptions, <<"s">>, Subs}} -> + ?assertEqual(#{threads => true, member_lists => [5]}, Subs) + after 1000 -> ?assert(false) + end, + ok = process_thread_subscriptions(#{<<"active">> => true}, self(), <<"s">>, #{bot => true}), + receive + {'$gen_cast', {thread_subscriptions, _, _}} = Unexpected -> + ?assertEqual(none, Unexpected) + after 100 -> ok + end. + -endif. diff --git a/fluxer_gateway/src/guild/guild_visibility_overwrites.erl b/fluxer_gateway/src/guild/guild_visibility_overwrites.erl index f2d89dec5..1d04d0d30 100644 --- a/fluxer_gateway/src/guild/guild_visibility_overwrites.erl +++ b/fluxer_gateway/src/guild/guild_visibility_overwrites.erl @@ -142,12 +142,31 @@ dispatch_session_changes( UserId = maps:get(user_id, SessionData), Pid = maps:get(pid, SessionData), ConnectedSet = maps:get(UserId, ConnectedVoiceByUser, sets:new()), - {StateWithCache, Removed, Added} = + {StateWithCache, Removed0, Added0} = compute_channel_diffs(SessionId, SessionData, UserId, OldState, AccState, ConnectedSet), + Removed = gate_channel_set(SessionData, Removed0, OldState), + Added = gate_channel_set(SessionData, Added0, StateWithCache), dispatch_removed_channels(Removed, Pid, OldState, GuildId), - dispatch_added_channels(Added, Pid, SessionId, SessionData, StateWithCache, GuildId), + dispatch_added_channels(Added, SessionId, SessionData, StateWithCache, GuildId), + ok = guild_thread_subscriptions:access_gained( + SessionData, sets:to_list(Added), StateWithCache, GuildId + ), StateWithCache. +-spec gate_channel_set(map(), sets:set(channel_id()), guild_state()) -> sets:set(channel_id()). +gate_channel_set(SessionData, ChannelIds, State) -> + case guild_thread_gate:needs_variant(State) of + false -> + ChannelIds; + true -> + sets:filter( + fun(ChannelId) -> + guild_thread_gate:channel_visible(SessionData, ChannelId, State) + end, + ChannelIds + ) + end. + -spec compute_channel_diffs( binary(), map(), user_id(), guild_state(), guild_state(), sets:set(channel_id()) ) -> @@ -180,13 +199,13 @@ dispatch_removed_channels(Removed, Pid, OldState, GuildId) -> ). -spec dispatch_added_channels( - sets:set(channel_id()), pid(), binary(), map(), guild_state(), integer() + sets:set(channel_id()), binary(), map(), guild_state(), integer() ) -> ok. -dispatch_added_channels(Added, Pid, SessionId, SessionData, StateWithCache, GuildId) -> +dispatch_added_channels(Added, SessionId, SessionData, StateWithCache, GuildId) -> lists:foreach( fun(ChannelId) -> guild_visibility_roles:dispatch_channel_create( - ChannelId, Pid, StateWithCache, GuildId + ChannelId, SessionData, StateWithCache, GuildId ), guild_visibility_roles:send_member_list_sync( SessionId, SessionData, ChannelId, GuildId, StateWithCache @@ -401,23 +420,36 @@ apply_visibility(ChannelId, OldVisible, NewVisible, {ViewableMapAcc, StateAcc}, integer(), map() ) -> {map(), guild_state()}. -handle_visibility_transition(true, false, ChId, Pid, _Sid, _SD, Old, New, GId, VMap) when +handle_visibility_transition(true, false, ChId, Pid, _Sid, SD, Old, New, GId, VMap) when is_pid(Pid) -> - guild_visibility_roles:dispatch_channel_delete(ChId, Pid, Old, GId), + case guild_thread_gate:channel_visible(SD, ChId, Old) of + true -> guild_visibility_roles:dispatch_channel_delete(ChId, Pid, Old, GId); + false -> ok + end, {VMap, New}; handle_visibility_transition(false, true, ChId, Pid, Sid, SD, _Old, New, GId, VMap) when is_pid(Pid) -> - guild_visibility_roles:dispatch_channel_create(ChId, Pid, New, GId), - guild_visibility_roles:send_member_list_sync(Sid, SD, ChId, GId, New), - FinalMap = guild_visibility_roles:maybe_ensure_parent_category_visible( - ChId, VMap, New, Pid, GId - ), - {FinalMap, New}; + case guild_thread_gate:channel_visible(SD, ChId, New) of + true -> gated_channel_create(ChId, Sid, SD, New, GId, VMap); + false -> {VMap, New} + end; handle_visibility_transition(_, _, _ChId, _Pid, _Sid, _SD, _Old, New, _GId, VMap) -> {VMap, New}. +-spec gated_channel_create( + channel_id(), binary(), map(), guild_state(), integer(), map() +) -> {map(), guild_state()}. +gated_channel_create(ChId, Sid, SD, New, GId, VMap) -> + guild_visibility_roles:dispatch_channel_create(ChId, SD, New, GId), + ok = guild_thread_subscriptions:access_gained(SD, [ChId], New, GId), + guild_visibility_roles:send_member_list_sync(Sid, SD, ChId, GId, New), + FinalMap = guild_visibility_roles:maybe_ensure_parent_category_visible( + ChId, VMap, New, SD, GId + ), + {FinalMap, New}. + -spec guild_id(guild_state()) -> integer() | undefined. guild_id(State) -> snowflake_id:parse_optional(maps:get(id, State, undefined)). diff --git a/fluxer_gateway/src/guild/guild_visibility_roles.erl b/fluxer_gateway/src/guild/guild_visibility_roles.erl index 290db2965..8a3db2039 100644 --- a/fluxer_gateway/src/guild/guild_visibility_roles.erl +++ b/fluxer_gateway/src/guild/guild_visibility_roles.erl @@ -18,9 +18,9 @@ -type channel_id() :: integer(). -spec maybe_ensure_parent_category_visible( - channel_id(), map(), guild_state(), pid() | term(), integer() + channel_id(), map(), guild_state(), map(), integer() ) -> map(). -maybe_ensure_parent_category_visible(ChannelId, ViewableMap, State, Pid, GuildId) -> +maybe_ensure_parent_category_visible(ChannelId, ViewableMap, State, SessionData, GuildId) -> case {snowflake_id(GuildId), guild_permissions:find_channel_by_id(ChannelId, State)} of {undefined, _} -> ViewableMap; @@ -28,24 +28,28 @@ maybe_ensure_parent_category_visible(ChannelId, ViewableMap, State, Pid, GuildId ViewableMap; {ResolvedGuildId, Channel} -> ensure_parent_category_visible( - channel_parent_id(Channel), ViewableMap, State, Pid, ResolvedGuildId + channel_parent_id(Channel), ViewableMap, State, SessionData, ResolvedGuildId ) end. -spec ensure_parent_category_visible( - channel_id() | undefined, map(), guild_state(), pid() | term(), integer() + channel_id() | undefined, map(), guild_state(), map(), integer() ) -> map(). -ensure_parent_category_visible(undefined, ViewableMap, _State, _Pid, _GuildId) -> +ensure_parent_category_visible(undefined, ViewableMap, _State, _SessionData, _GuildId) -> ViewableMap; -ensure_parent_category_visible(ParentId, ViewableMap, State, Pid, GuildId) when is_pid(Pid) -> +ensure_parent_category_visible( + ParentId, ViewableMap, State, #{pid := Pid} = SessionData, GuildId +) when + is_pid(Pid) +-> case maps:is_key(ParentId, ViewableMap) of true -> ViewableMap; false -> - dispatch_channel_create(ParentId, Pid, State, GuildId), + dispatch_channel_create(ParentId, SessionData, State, GuildId), ViewableMap#{ParentId => true} end; -ensure_parent_category_visible(_ParentId, ViewableMap, _State, _Pid, _GuildId) -> +ensure_parent_category_visible(_ParentId, ViewableMap, _State, _SessionData, _GuildId) -> ViewableMap. -spec dispatch_channel_delete(channel_id(), pid(), guild_state(), integer()) -> ok. @@ -61,14 +65,16 @@ dispatch_channel_delete(ChannelId, SessionPid, OldState, GuildId) -> ok end. --spec dispatch_channel_create(channel_id(), pid(), guild_state(), integer()) -> ok. -dispatch_channel_create(ChannelId, SessionPid, NewState, GuildId) -> - case {is_pid(SessionPid), snowflake_id(ChannelId), snowflake_id(GuildId)} of - {true, ResolvedChannelId, ResolvedGuildId} when - is_integer(ResolvedChannelId), is_integer(ResolvedGuildId) +-spec dispatch_channel_create(channel_id(), map(), guild_state(), integer()) -> ok. +dispatch_channel_create(ChannelId, SessionData, NewState, GuildId) -> + case + {maps:get(pid, SessionData, undefined), snowflake_id(ChannelId), snowflake_id(GuildId)} + of + {SessionPid, ResolvedChannelId, ResolvedGuildId} when + is_pid(SessionPid), is_integer(ResolvedChannelId), is_integer(ResolvedGuildId) -> maybe_dispatch_channel_create( - ResolvedChannelId, SessionPid, NewState, ResolvedGuildId + ResolvedChannelId, SessionData, NewState, ResolvedGuildId ); _ -> ok @@ -158,18 +164,31 @@ maybe_dispatch_channel_delete(ChannelId, SessionPid, OldState, GuildId) -> gateway_dispatch_relay:dispatch(SessionPid, channel_delete, ChannelDelete, GuildId) end. --spec maybe_dispatch_channel_create(channel_id(), pid(), guild_state(), integer()) -> ok. -maybe_dispatch_channel_create(ChannelId, SessionPid, NewState, GuildId) -> +-spec maybe_dispatch_channel_create(channel_id(), map(), guild_state(), integer()) -> ok. +maybe_dispatch_channel_create(ChannelId, #{pid := SessionPid} = SessionData, NewState, GuildId) -> case guild_permissions:find_channel_by_id(ChannelId, NewState) of undefined -> ok; Channel -> - ChannelWithGuild = Channel#{<<"guild_id">> => integer_to_binary(GuildId)}, + SessionChannel = session_channel(Channel, SessionData, NewState), + ChannelWithGuild = SessionChannel#{<<"guild_id">> => integer_to_binary(GuildId)}, gateway_dispatch_relay:dispatch( SessionPid, channel_create, ChannelWithGuild, GuildId ) end. +-spec session_channel(map(), map(), guild_state()) -> map(). +session_channel(Channel, SessionData, State) -> + case + guild_thread_gate:needs_variant(State) andalso + not guild_thread_gate:session_viewer(SessionData) + of + true -> + guild_thread_gate:mask_overwrites(guild_thread_gate:strip_channel_surface(Channel)); + false -> + Channel + end. + -spec channel_parent_id(map()) -> channel_id() | undefined. channel_parent_id(Channel) -> snowflake_id(maps:get(<<"parent_id">>, Channel, undefined)). diff --git a/fluxer_gateway/src/presence/presence.erl b/fluxer_gateway/src/presence/presence.erl index f44c1fc72..4b11ca1b8 100644 --- a/fluxer_gateway/src/presence/presence.erl +++ b/fluxer_gateway/src/presence/presence.erl @@ -183,6 +183,9 @@ handle_info({initial_presences, Presences}, State) when is_list(Presences) -> {noreply, State}; handle_info({'DOWN', Ref, process, _Pid, Reason}, State) when is_reference(Ref) -> presence_connect:handle_process_down(Ref, Reason, State); +handle_info({thread_user_flip, Version} = Flip, State) when is_integer(Version) -> + lists:foreach(fun(Pid) -> Pid ! Flip end, presence_connect:collect_session_pids(State)), + {noreply, State}; handle_info(_, State) -> {noreply, State}. @@ -315,6 +318,14 @@ presence_rejoin_notifies_all_sessions_test() -> ?assert(received_rejoin_ack(one)), ?assert(received_rejoin_ack(two)). +thread_user_flip_is_forwarded_to_every_session_test() -> + State = test_state(#{<<"s1">> => test_session_entry(self())}), + ?assertEqual({noreply, State}, handle_info({thread_user_flip, 4}, State)), + receive + {thread_user_flip, 4} -> ok + after 1000 -> ?assert(false) + end. + presence_rejoin_with_no_sessions_is_noop_test() -> State = test_state(#{}), ?assertEqual({noreply, State}, handle_cast(presence_rejoin, State)). diff --git a/fluxer_gateway/src/push/push.erl b/fluxer_gateway/src/push/push.erl index 36ce3cf7e..c3fe397d0 100644 --- a/fluxer_gateway/src/push/push.erl +++ b/fluxer_gateway/src/push/push.erl @@ -487,7 +487,7 @@ do_handle_message_create_context(Context) -> }), publish_eligible_users( EligibleUsers, - MessageData, + push_eligibility_checks:strip_thread_eligibility(MessageData), MarkdownContext, GuildId, ChannelId, diff --git a/fluxer_gateway/src/push/push_eligibility_checks.erl b/fluxer_gateway/src/push/push_eligibility_checks.erl index 30128b8b4..4142ff463 100644 --- a/fluxer_gateway/src/push/push_eligibility_checks.erl +++ b/fluxer_gateway/src/push/push_eligibility_checks.erl @@ -17,6 +17,7 @@ -export([large_guild_threshold/0]). -export([has_large_guild_override/1]). -export([get_guild_large_metadata/1]). +-export([strip_thread_eligibility/1]). -define(LARGE_GUILD_THRESHOLD, 2500). -define(LARGE_GUILD_OVERRIDE_FEATURE, <<"LARGE_GUILD_OVERRIDE">>). @@ -29,6 +30,11 @@ -define(CHANNEL_TYPE_GROUP_DM, 3). -define(LARGE_METADATA_MAILBOX_SHED_THRESHOLD, 100). -define(LARGE_METADATA_CALL_TIMEOUT_MS, 200). +-define(THREAD_PUSH_KEY, <<"__thread_push">>). +-define(THREAD_MEMBER_ALL_MESSAGES, 16#2). +-define(THREAD_MEMBER_ONLY_MENTIONS, 16#4). +-define(THREAD_MEMBER_NO_MESSAGES, 16#8). +-define(NEW_FORUM_THREADS_OFF, 16#2000). -spec check_muted_and_notifications( integer(), integer(), map(), integer(), map(), map(), integer(), map(), map() | undefined @@ -43,6 +49,50 @@ check_muted_and_notifications( _GuildId, ConnectedUsers, LargeGuildMetadata +) -> + case maps:get(?THREAD_PUSH_KEY, MessageData, undefined) of + #{<<"parent_id">> := ParentId, <<"forum_thread_created">> := true} = Thread when + is_integer(ParentId) + -> + check_forum_thread_created( + Thread, GuildDefaultNotifications, Settings, LargeGuildMetadata + ); + #{<<"parent_id">> := ParentId} = Thread when is_integer(ParentId) -> + check_thread_muted_and_notifications( + UserId, + Thread, + MessageData, + GuildDefaultNotifications, + UserRolesMap, + Settings, + ConnectedUsers, + LargeGuildMetadata + ); + _ -> + check_channel_muted_and_notifications( + UserId, + ChannelId, + MessageData, + GuildDefaultNotifications, + UserRolesMap, + Settings, + ConnectedUsers, + LargeGuildMetadata + ) + end. + +-spec check_channel_muted_and_notifications( + integer(), integer(), map(), integer(), map(), map(), map(), map() | undefined +) -> boolean(). +check_channel_muted_and_notifications( + UserId, + ChannelId, + MessageData, + GuildDefaultNotifications, + UserRolesMap, + Settings, + ConnectedUsers, + LargeGuildMetadata ) -> ChannelOverrides = map_setting(channel_overrides, Settings), ChannelOverride = channel_override(ChannelId, ChannelOverrides, #{}), @@ -59,6 +109,134 @@ check_muted_and_notifications( ) end. +-spec check_thread_muted_and_notifications( + integer(), map(), map(), integer(), map(), map(), map(), map() | undefined +) -> boolean(). +check_thread_muted_and_notifications( + UserId, + Thread, + MessageData, + GuildDefaultNotifications, + UserRolesMap, + Settings, + ConnectedUsers, + LargeGuildMetadata +) -> + #{<<"parent_id">> := ParentId} = Thread, + ChannelOverrides = map_setting(channel_overrides, Settings), + ThreadMember = thread_member(UserId, Thread), + Muted = + is_mute_active(Settings) orelse + is_override_muted(maps:get(<<"category_id">>, Thread, undefined), ChannelOverrides) orelse + is_override_muted(ParentId, ChannelOverrides) orelse + is_mute_active(ThreadMember), + case Muted of + true -> + false; + false -> + Level = thread_notification_level( + ThreadMember, + ParentId, + Settings, + GuildDefaultNotifications, + LargeGuildMetadata + ), + push_eligibility:should_allow_notification( + Level, + thread_mention_scope(ThreadMember, MessageData), + UserId, + Settings, + UserRolesMap, + ConnectedUsers + ) + end. + +-spec check_forum_thread_created(map(), integer(), map(), map() | undefined) -> boolean(). +check_forum_thread_created(Thread, GuildDefaultNotifications, Settings, LargeGuildMetadata) -> + #{<<"parent_id">> := ForumId} = Thread, + ChannelOverrides = map_setting(channel_overrides, Settings), + ForumOverride = channel_override(ForumId, ChannelOverrides, #{}), + Suppressed = + is_mute_active(Settings) orelse + is_override_muted(maps:get(<<"category_id">>, Thread, undefined), ChannelOverrides) orelse + is_mute_active(ForumOverride) orelse + new_forum_threads_flag(ForumOverride, ?NEW_FORUM_THREADS_OFF), + not Suppressed andalso + inherited_thread_level(ForumId, Settings, GuildDefaultNotifications, LargeGuildMetadata) =:= + ?MESSAGE_NOTIFICATIONS_ALL. + +-spec new_forum_threads_flag(term(), integer()) -> boolean(). +new_forum_threads_flag(Override, Flag) -> + case push_eligibility:get_setting(flags, Override, 0) of + Flags when is_integer(Flags) -> Flags band Flag =/= 0; + _ -> false + end. + +-spec thread_member(integer(), map()) -> map() | undefined. +thread_member(UserId, Thread) -> + case maps:get(<<"members">>, Thread, #{}) of + #{UserId := Member} when is_map(Member) -> Member; + _ -> undefined + end. + +-spec is_override_muted(term(), map()) -> boolean(). +is_override_muted(ChannelId, ChannelOverrides) when is_integer(ChannelId) -> + is_mute_active(channel_override(ChannelId, ChannelOverrides, #{})); +is_override_muted(_ChannelId, _ChannelOverrides) -> + false. + +-spec thread_notification_level( + map() | undefined, integer(), map(), integer(), map() | undefined +) -> integer(). +thread_notification_level(undefined, ParentId, Settings, GuildDefault, LargeGuildMetadata) -> + Inherited = inherited_thread_level(ParentId, Settings, GuildDefault, LargeGuildMetadata), + max(Inherited, ?MESSAGE_NOTIFICATIONS_ONLY_MENTIONS); +thread_notification_level(ThreadMember, ParentId, Settings, GuildDefault, LargeGuildMetadata) -> + case explicit_thread_level(push_eligibility:get_setting(flags, ThreadMember, 0)) of + undefined -> + inherited_thread_level(ParentId, Settings, GuildDefault, LargeGuildMetadata); + Level -> + Level + end. + +-spec inherited_thread_level(integer(), map(), integer(), map() | undefined) -> integer(). +inherited_thread_level(ParentId, Settings, GuildDefault, LargeGuildMetadata) -> + override_for_large_guild_metadata( + LargeGuildMetadata, resolve_message_notifications(ParentId, Settings, GuildDefault) + ). + +-spec explicit_thread_level(term()) -> integer() | undefined. +explicit_thread_level(Flags) when + is_integer(Flags), Flags band ?THREAD_MEMBER_NO_MESSAGES =/= 0 +-> + ?MESSAGE_NOTIFICATIONS_NO_MESSAGES; +explicit_thread_level(Flags) when + is_integer(Flags), Flags band ?THREAD_MEMBER_ONLY_MENTIONS =/= 0 +-> + ?MESSAGE_NOTIFICATIONS_ONLY_MENTIONS; +explicit_thread_level(Flags) when + is_integer(Flags), Flags band ?THREAD_MEMBER_ALL_MESSAGES =/= 0 +-> + ?MESSAGE_NOTIFICATIONS_ALL; +explicit_thread_level(_Flags) -> + undefined. + +-spec thread_mention_scope(map() | undefined, map()) -> map(). +thread_mention_scope(undefined, MessageData) -> + MessageData#{<<"mention_everyone">> => false, <<"mention_here">> => false}; +thread_mention_scope(_ThreadMember, MessageData) -> + MessageData. + +-spec strip_thread_eligibility(map()) -> map(). +strip_thread_eligibility(#{?THREAD_PUSH_KEY := Thread} = MessageData) when is_map(Thread) -> + MessageData#{ + ?THREAD_PUSH_KEY => maps:with( + [<<"parent_id">>, <<"parent_name">>, <<"forum_thread_created">>], Thread + ) + }; +strip_thread_eligibility(MessageData) -> + MessageData. + -spec is_mute_active(term()) -> boolean(). is_mute_active(Config) -> boolean_setting(muted, Config, false) andalso @@ -322,6 +500,175 @@ check_with_settings(Settings) -> undefined ). +thread_message(Members) -> + thread_message(Members, #{}). + +thread_message(Members, Extra) -> + maps:merge( + #{ + <<"channel_type">> => 11, + <<"__thread_push">> => #{ + <<"parent_id">> => 200, + <<"parent_name">> => <<"general">>, + <<"category_id">> => 50, + <<"members">> => Members + } + }, + Extra + ). + +check_thread(MessageData, Settings, Metadata) -> + check_muted_and_notifications(100, 500, MessageData, 0, #{}, Settings, 1, #{}, Metadata). + +thread_member_inherits_the_parent_level_test() -> + Member = #{100 => #{<<"flags">> => 1}}, + ?assert(check_thread(thread_message(Member), #{}, undefined)), + ParentMentions = #{channel_overrides => #{<<"200">> => #{message_notifications => 1}}}, + ?assertNot(check_thread(thread_message(Member), ParentMentions, undefined)), + ThreadOverride = #{channel_overrides => #{<<"500">> => #{message_notifications => 1}}}, + ?assert(check_thread(thread_message(Member), ThreadOverride, undefined)). + +explicit_thread_level_beats_the_parent_level_test() -> + ParentMentions = #{channel_overrides => #{<<"200">> => #{message_notifications => 1}}}, + ?assert( + check_thread(thread_message(#{100 => #{<<"flags">> => 2}}), ParentMentions, undefined) + ), + ?assertNot(check_thread(thread_message(#{100 => #{<<"flags">> => 8}}), #{}, undefined)), + Mentioned = thread_message( + #{100 => #{<<"flags">> => 4}}, #{<<"mentions">> => [#{<<"id">> => <<"100">>}]} + ), + ?assert(check_thread(Mentioned, #{}, undefined)), + ?assertNot(check_thread(thread_message(#{100 => #{<<"flags">> => 4}}), #{}, undefined)). + +large_guild_clamp_applies_to_inherited_thread_levels_only_test() -> + Large = #{member_count => 3000, features => []}, + ?assertNot(check_thread(thread_message(#{100 => #{<<"flags">> => 0}}), #{}, Large)), + ?assert(check_thread(thread_message(#{100 => #{<<"flags">> => 2}}), #{}, Large)). + +thread_mutes_apply_at_every_layer_test() -> + Member = #{100 => #{<<"flags">> => 2}}, + Muted = #{muted => true}, + ?assertNot(check_thread(thread_message(Member), Muted, undefined)), + ?assertNot( + check_thread( + thread_message(Member), #{channel_overrides => #{<<"50">> => Muted}}, undefined + ) + ), + ?assertNot( + check_thread( + thread_message(Member), #{channel_overrides => #{<<"200">> => Muted}}, undefined + ) + ), + ?assertNot( + check_thread( + thread_message(#{100 => #{<<"flags">> => 2, <<"muted">> => true}}), #{}, undefined + ) + ), + Expired = #{<<"end_time">> => rfc3339_in_ms(-60000)}, + ?assert( + check_thread( + thread_message(#{ + 100 => #{<<"flags">> => 2, <<"muted">> => true, <<"mute_config">> => Expired} + }), + #{}, + undefined + ) + ). + +thread_non_members_need_a_direct_or_role_mention_test() -> + ?assertNot(check_thread(thread_message(#{}), #{}, undefined)), + Everyone = thread_message(#{}, #{<<"mention_everyone">> => true}), + ?assertNot(check_thread(Everyone, #{}, undefined)), + Direct = thread_message(#{}, #{<<"mentions">> => [#{<<"id">> => <<"100">>}]}), + ?assert(check_thread(Direct, #{}, undefined)), + ?assert( + check_thread( + thread_message(#{100 => #{<<"flags">> => 0}}, #{<<"mention_everyone">> => true}), + #{}, + undefined + ) + ). + +forum_message() -> + thread_message(#{}, #{ + <<"__thread_push">> => #{ + <<"parent_id">> => 200, + <<"parent_name">> => <<"ideas">>, + <<"category_id">> => 50, + <<"members">> => #{}, + <<"forum_thread_created">> => true + } + }). + +forum_thread_created_needs_all_messages_on_the_forum_test() -> + ?assert(check_thread(forum_message(), #{}, undefined)), + Mentions = #{channel_overrides => #{<<"200">> => #{message_notifications => 1}}}, + ?assertNot(check_thread(forum_message(), Mentions, undefined)), + ?assertNot(check_thread(forum_message(), #{message_notifications => 1}, undefined)), + ?assertNot( + check_thread(forum_message(), #{}, #{member_count => 3000, features => []}) + ), + ExplicitAll = #{ + message_notifications => 1, + channel_overrides => #{<<"200">> => #{message_notifications => 0}} + }, + ?assert(check_thread(forum_message(), ExplicitAll, undefined)). + +forum_thread_created_honours_mutes_and_new_forum_threads_off_test() -> + Muted = #{muted => true}, + ?assertNot(check_thread(forum_message(), Muted, undefined)), + ?assertNot( + check_thread(forum_message(), #{channel_overrides => #{<<"50">> => Muted}}, undefined) + ), + ?assertNot( + check_thread(forum_message(), #{channel_overrides => #{<<"200">> => Muted}}, undefined) + ), + Off = #{channel_overrides => #{<<"200">> => #{<<"flags">> => 16#2000}}}, + ?assertNot(check_thread(forum_message(), Off, undefined)), + On = #{channel_overrides => #{<<"200">> => #{<<"flags">> => 16#4000}}}, + ?assert(check_thread(forum_message(), On, undefined)). + +forum_thread_created_new_forum_threads_on_does_not_opt_in_below_all_test() -> + OnMentions = #{ + channel_overrides => #{ + <<"200">> => #{<<"flags">> => 16#4000, message_notifications => 1} + } + }, + ?assertNot(check_thread(forum_message(), OnMentions, undefined)), + ?assertNot( + check_thread( + forum_message(), + #{channel_overrides => #{<<"200">> => #{<<"flags">> => 16#4000}}}, + #{member_count => 3000, features => []} + ) + ), + MutedOn = #{ + channel_overrides => #{ + <<"200">> => #{<<"flags">> => 16#4000, muted => true} + } + }, + ?assertNot(check_thread(forum_message(), MutedOn, undefined)), + BothFlags = #{channel_overrides => #{<<"200">> => #{<<"flags">> => 16#6000}}}, + ?assertNot(check_thread(forum_message(), BothFlags, undefined)). + +strip_thread_eligibility_keeps_delivery_fields_test() -> + ?assertEqual( + #{ + <<"parent_id">> => 200, + <<"parent_name">> => <<"ideas">>, + <<"forum_thread_created">> => true + }, + maps:get(<<"__thread_push">>, strip_thread_eligibility(forum_message())) + ), + ?assertEqual( + #{ + <<"channel_type">> => 11, + <<"__thread_push">> => #{<<"parent_id">> => 200, <<"parent_name">> => <<"general">>} + }, + strip_thread_eligibility(thread_message(#{100 => #{}})) + ), + ?assertEqual(#{<<"id">> => 1}, strip_thread_eligibility(#{<<"id">> => 1})). + is_user_in_mentions_test() -> Mentions = [#{<<"id">> => <<"123">>}, #{<<"id">> => <<"456">>}], ?assertEqual(true, is_user_in_mentions(123, Mentions)), diff --git a/fluxer_gateway/src/push/push_job_publisher.erl b/fluxer_gateway/src/push/push_job_publisher.erl index 5114d3989..9c2dba143 100644 --- a/fluxer_gateway/src/push/push_job_publisher.erl +++ b/fluxer_gateway/src/push/push_job_publisher.erl @@ -40,6 +40,40 @@ publish_message( MessageId, GuildName, ChannelName +) -> + Job = message_job( + MessageData, + MarkdownContext, + GuildId, + ChannelId, + MessageId, + GuildName, + ChannelName + ), + publish_recipients(Job, UserIds, #{ + kind => message, + user_ids => UserIds, + channel_id => ChannelId, + message_id => MessageId + }). + +-spec message_job( + map(), + map(), + integer(), + integer(), + integer(), + binary() | undefined, + binary() | undefined +) -> map(). +message_job( + MessageData, + MarkdownContext, + GuildId, + ChannelId, + MessageId, + GuildName, + ChannelName ) -> Job = #{ <<"v">> => ?JOB_VERSION, @@ -57,12 +91,7 @@ publish_message( ChannelName ) }, - publish_recipients(Job, UserIds, #{ - kind => message, - user_ids => UserIds, - channel_id => ChannelId, - message_id => MessageId - }). + maps:merge(Job, thread_job_fields(MessageData, ChannelName)). -spec publish_recipients(map(), [integer()], meta()) -> ok | {error, term()}. publish_recipients(Job, UserIds, Meta) -> @@ -219,6 +248,19 @@ notification_fields( <<"image_url">> => nullable(push_notification_format:extract_image_url(MessageData)) }. +-spec thread_job_fields(map(), binary() | undefined) -> map(). +thread_job_fields(MessageData, ChannelName) -> + case push_notification:thread_fields(MessageData, ChannelName) of + Fields when map_size(Fields) =:= 0 -> Fields; + Fields -> Fields#{<<"kind">> => thread_job_kind(MessageData)} + end. + +-spec thread_job_kind(map()) -> binary(). +thread_job_kind(#{<<"__thread_push">> := #{<<"forum_thread_created">> := true}}) -> + <<"forum_thread_created">>; +thread_job_kind(_MessageData) -> + <<"thread">>. + -spec nullable(binary() | undefined) -> binary() | null. nullable(undefined) -> null; @@ -344,6 +386,89 @@ caller_fields_falls_back_to_the_default_avatar_test() -> Fields ). +thread_job_fields_are_absent_for_channel_messages_test() -> + ?assertEqual(#{}, thread_job_fields(#{}, <<"general">>)). + +thread_job_fields_mark_thread_jobs_test() -> + MessageData = #{ + <<"__thread_push">> => #{<<"parent_id">> => 200, <<"parent_name">> => <<"general">>} + }, + ?assertEqual( + #{ + <<"kind">> => <<"thread">>, + <<"channel_name">> => <<"ideas">>, + <<"parent_id">> => <<"200">>, + <<"parent_name">> => <<"general">> + }, + thread_job_fields(MessageData, <<"ideas">>) + ). + +thread_job_fields_mark_forum_thread_created_jobs_test() -> + MessageData = #{ + <<"__thread_push">> => #{ + <<"parent_id">> => 300, + <<"parent_name">> => <<"ideas">>, + <<"forum_thread_created">> => true + } + }, + ?assertMatch( + #{<<"kind">> := <<"forum_thread_created">>, <<"parent_id">> := <<"300">>}, + thread_job_fields(MessageData, <<"post">>) + ). + +encoded_message_job(MessageData, ChannelName) -> + ok = meck:new(push_notification_format, [passthrough, no_link]), + ok = meck:new(push_utils, [passthrough, no_link]), + try + ok = meck:expect( + push_notification_format, resolve_author_avatar_url, fun(_) -> <<"icon">> end + ), + ok = meck:expect(push_utils, construct_static_asset_url, fun(_) -> <<"badge">> end), + Job = message_job(MessageData, #{}, 5, 6, 7, <<"Guild">>, ChannelName), + json:decode(iolist_to_binary(json:encode(Job))) + after + meck:unload(push_utils), + meck:unload(push_notification_format) + end. + +message_job_puts_thread_fields_at_the_top_level_test() -> + MessageData = #{ + <<"author">> => #{<<"username">> => <<"ada">>}, + <<"content">> => <<"hi">>, + <<"__thread_push">> => #{<<"parent_id">> => 200, <<"parent_name">> => <<"general">>} + }, + Decoded = encoded_message_job(MessageData, <<"ideas">>), + ?assertMatch( + #{ + <<"kind">> := <<"thread">>, + <<"parent_id">> := <<"200">>, + <<"parent_name">> := <<"general">>, + <<"channel_name">> := <<"ideas">> + }, + Decoded + ), + Notification = maps:get(<<"notification">>, Decoded), + ?assertEqual( + [], + [ + K + || K <- [<<"kind">>, <<"parent_id">>, <<"parent_name">>, <<"channel_name">>], + maps:is_key(K, Notification) + ] + ). + +message_job_has_no_thread_fields_for_channel_messages_test() -> + MessageData = #{<<"author">> => #{<<"username">> => <<"ada">>}, <<"content">> => <<"hi">>}, + Decoded = encoded_message_job(MessageData, <<"general">>), + ?assertEqual( + [], + [ + K + || K <- [<<"kind">>, <<"parent_id">>, <<"parent_name">>, <<"channel_name">>], + maps:is_key(K, Decoded) + ] + ). + caller_fields_caps_the_caller_name_test() -> Name = binary:copy(<<"a">>, ?MAX_CALLER_NAME_BYTES + 32), Fields = with_endpoint_env(fun() -> diff --git a/fluxer_gateway/src/push/push_notification.erl b/fluxer_gateway/src/push/push_notification.erl index ee00e9eb5..d4aab9d29 100644 --- a/fluxer_gateway/src/push/push_notification.erl +++ b/fluxer_gateway/src/push/push_notification.erl @@ -3,7 +3,9 @@ -module(push_notification). -typing([eqwalizer]). --export([build_notification_title/5]). +-export([build_notification_title/5, thread_fields/2]). + +-define(THREAD_PUSH_KEY, <<"__thread_push">>). -spec build_notification_title( binary(), map(), integer(), binary() | undefined, binary() | undefined @@ -11,10 +13,48 @@ build_notification_title(AuthorUsername, MessageData, GuildId, GuildName, ChannelName) -> ChannelType = maps:get(<<"channel_type">>, MessageData, 1), case GuildId of - 0 -> format_dm_title(AuthorUsername, ChannelType); - _ -> format_guild_title(AuthorUsername, GuildName, ChannelName) + 0 -> + format_dm_title(AuthorUsername, ChannelType); + _ -> + format_guild_title( + AuthorUsername, GuildName, channel_label(MessageData, ChannelName) + ) end. +-spec channel_label(map(), binary() | undefined) -> binary() | undefined. +channel_label(MessageData, ChannelName) when is_binary(ChannelName) -> + case thread_parent_name(MessageData) of + undefined -> ChannelName; + ParentName -> iolist_to_binary([ChannelName, <<", #">>, ParentName]) + end; +channel_label(_MessageData, ChannelName) -> + ChannelName. + +-spec thread_parent_name(map()) -> binary() | undefined. +thread_parent_name(#{?THREAD_PUSH_KEY := #{<<"parent_name">> := Name}}) when + is_binary(Name), byte_size(Name) > 0 +-> + Name; +thread_parent_name(_MessageData) -> + undefined. + +-spec thread_fields(map(), binary() | undefined) -> map(). +thread_fields( + #{?THREAD_PUSH_KEY := #{<<"parent_id">> := ParentId}} = MessageData, ChannelName +) when + is_integer(ParentId) +-> + maps:filter( + fun(_Key, Value) -> is_binary(Value) end, + #{ + <<"channel_name">> => ChannelName, + <<"parent_id">> => integer_to_binary(ParentId), + <<"parent_name">> => thread_parent_name(MessageData) + } + ); +thread_fields(_MessageData, _ChannelName) -> + #{}. + -spec format_dm_title(binary(), term()) -> binary(). format_dm_title(AuthorUsername, 3) -> iolist_to_binary([AuthorUsername, <<" (Group DM)">>]); @@ -51,6 +91,15 @@ build_notification_title_guild_test() -> build_notification_title(<<"Alice">>, #{}, 123, <<"My Server">>, <<"general">>) ). +build_notification_title_thread_test() -> + MessageData = #{ + <<"__thread_push">> => #{<<"parent_id">> => 200, <<"parent_name">> => <<"general">>} + }, + ?assertEqual( + <<"Alice (#ideas, #general, My Server)">>, + build_notification_title(<<"Alice">>, MessageData, 123, <<"My Server">>, <<"ideas">>) + ). + build_url_dm_test() -> ?assertEqual(<<"/channels/@me/456/789">>, push_notification_format:build_url(0, 456, 789)). diff --git a/fluxer_gateway/src/session/session.erl b/fluxer_gateway/src/session/session.erl index 6cb35bd99..833614c3e 100644 --- a/fluxer_gateway/src/session/session.erl +++ b/fluxer_gateway/src/session/session.erl @@ -71,6 +71,7 @@ voice_queue => queue:queue(map()), voice_queue_timer => reference() | undefined, debounce_reactions => boolean(), + thread_channels_capable => boolean(), reaction_buffer => reaction_buffer(), reaction_buffer_timer => reference() | undefined, _ => _ @@ -244,6 +245,8 @@ handle_info({call_reconnect, ChannelId, Attempt}, State) when session_connection:handle_call_reconnect(ChannelId, Attempt, State); handle_info({gateway_timing_update, Timings}, State) -> {noreply, gateway_timings:merge_state(Timings, State)}; +handle_info({thread_user_flip, _Version}, State) -> + {noreply, notify_thread_user_flip(State)}; handle_info({dm_partner_mutual, GuildId, PartnerIds}, State) when is_integer(GuildId), is_list(PartnerIds) -> @@ -251,6 +254,31 @@ handle_info({dm_partner_mutual, GuildId, PartnerIds}, State) when handle_info(Msg, State) -> handle_info_lifecycle(Msg, State). +-spec notify_thread_user_flip(session_state()) -> session_state(). +notify_thread_user_flip(#{id := SessionId} = State) when is_binary(SessionId) -> + case + maps:get(bot, State, false) =:= true orelse + maps:get(thread_channels_capable, State, false) =:= true + of + true -> cast_thread_user_flip(SessionId, State); + false -> State + end; +notify_thread_user_flip(State) -> + State. + +-spec cast_thread_user_flip(binary(), session_state()) -> session_state(). +cast_thread_user_flip(SessionId, State) -> + maps:foreach( + fun + (_GuildId, {GuildPid, _Ref}) when is_pid(GuildPid) -> + gen_server:cast(GuildPid, {thread_user_flip, SessionId}); + (_GuildId, _Ref) -> + ok + end, + maps:get(guilds, State, #{}) + ), + State. + -spec handle_info_guild_connect_result(tuple(), session_state()) -> {noreply, session_state()} | {stop, normal, session_state()}. handle_info_guild_connect_result( @@ -428,6 +456,7 @@ test_base_extra() -> voice_queue => queue:new(), voice_queue_timer => undefined, debounce_reactions => false, + thread_channels_capable => false, reaction_buffer => [], reaction_buffer_timer => undefined }. @@ -588,4 +617,21 @@ received_presence_connect() -> after 200 -> false end. +thread_user_flip_skips_sessions_that_cannot_become_viewers_test() -> + Base = #{id => <<"s">>, guilds => #{1 => {self(), make_ref()}, 2 => undefined}}, + _ = notify_thread_user_flip(Base), + _ = notify_thread_user_flip(Base#{bot => false, thread_channels_capable => false}), + ?assertNot(received_thread_user_flip()), + _ = notify_thread_user_flip(Base#{thread_channels_capable => true}), + ?assert(received_thread_user_flip()), + _ = notify_thread_user_flip(Base#{bot => true}), + ?assert(received_thread_user_flip()), + ?assertNot(received_thread_user_flip()). + +received_thread_user_flip() -> + receive + {'$gen_cast', {thread_user_flip, <<"s">>}} -> true + after 100 -> false + end. + -endif. diff --git a/fluxer_gateway/src/session/session_connection_guild.erl b/fluxer_gateway/src/session/session_connection_guild.erl index bcd2462c1..df3da5754 100644 --- a/fluxer_gateway/src/session/session_connection_guild.erl +++ b/fluxer_gateway/src/session/session_connection_guild.erl @@ -103,7 +103,8 @@ spawn_guild_connect_worker(GuildId, Attempt, SessionId, UserId, Bot, IsStaff, In bot => Bot, is_staff => IsStaff, initial_guild_id => maps:get(initial_guild_id, State, undefined), - user_data => maps:get(user_data, State, #{}) + user_data => maps:get(user_data, State, #{}), + thread_channels_capable => maps:get(thread_channels_capable, State, false) }, {WorkerPid, WorkerRef} = spawn_monitor(fun() -> do_guild_connect_with_release(Ctx) diff --git a/fluxer_gateway/src/session/session_connection_guild_resolve.erl b/fluxer_gateway/src/session/session_connection_guild_resolve.erl index 66c299409..7e1616ec9 100644 --- a/fluxer_gateway/src/session/session_connection_guild_resolve.erl +++ b/fluxer_gateway/src/session/session_connection_guild_resolve.erl @@ -23,7 +23,8 @@ bot := boolean(), is_staff := boolean(), initial_guild_id := guild_id() | undefined, - user_data := map() + user_data := map(), + thread_channels_capable => boolean() }. -spec do_local_guild_connect(connect_ctx()) -> @@ -203,7 +204,8 @@ do_start_connect_async(GuildPid, Ctx) -> bot => Bot, is_staff => IsStaff, initial_guild_id => InitialGuildId, - active_guilds => ActiveGuilds + active_guilds => ActiveGuilds, + thread_channels_capable => maps:get(thread_channels_capable, Ctx, false) }, CastMsg = {session_connect_async, #{ @@ -250,3 +252,35 @@ notify_remote_timing(SessionPid, Role, OwnerNode, FunctionName, StartedAt) when ok; notify_remote_timing(_, _, _, _, _) -> ok. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +connect_request_carries_thread_channels_capability_test() -> + ?assertEqual(true, sent_capability(#{thread_channels_capable => true})), + ?assertEqual(false, sent_capability(#{thread_channels_capable => false})), + ?assertEqual(false, sent_capability(#{})). + +sent_capability(Extra) -> + Ctx = maps:merge( + #{ + session_pid => self(), + guild_id => 1, + attempt => 0, + session_id => <<"s">>, + user_id => 2, + bot => false, + is_staff => false, + initial_guild_id => undefined, + user_data => #{} + }, + Extra + ), + pending = do_start_connect_async(self(), Ctx), + receive + {'$gen_cast', {session_connect_async, #{request := Request}}} -> + maps:get(thread_channels_capable, Request) + after 1000 -> erlang:error(connect_cast_not_sent) + end. + +-endif. diff --git a/fluxer_gateway/src/session/session_dispatch.erl b/fluxer_gateway/src/session/session_dispatch.erl index 437bcc3bb..a2fc70e57 100644 --- a/fluxer_gateway/src/session/session_dispatch.erl +++ b/fluxer_gateway/src/session/session_dispatch.erl @@ -23,12 +23,21 @@ {noreply, session_state()}. handle_dispatch(Event, {pre_encoded, _} = Data, State) -> case - should_skip_for_shard(Event, Data, State) orelse should_ignore_event(Event, Data, State) + should_skip_for_shard(Event, Data, State) orelse should_ignore_event(Event, Data, State) orelse + thread_backstop(Event, #{}, State) =:= drop of true -> {noreply, State}; false -> do_handle_dispatch_pre_encoded(Event, Data, State) end; handle_dispatch(Event, Data, State) -> + case thread_backstop(Event, Data, State) of + drop -> {noreply, State}; + {ok, Data1} -> handle_plain_dispatch(Event, Data1, State) + end. + +-spec handle_plain_dispatch(event(), map() | list(), session_state()) -> + {noreply, session_state()}. +handle_plain_dispatch(Event, Data, State) -> case should_skip_for_shard(Event, Data, State) orelse should_ignore_event(Event, Data, State) of @@ -36,6 +45,69 @@ handle_dispatch(Event, Data, State) -> false -> route_bot_guild_event(Event, Data, State) end. +-spec thread_backstop(event(), map() | list(), session_state()) -> drop | {ok, map() | list()}. +thread_backstop(Event, Data, State) -> + Capable = + maps:get(bot, State, false) =:= true orelse + maps:get(thread_channels_capable, State, false) =:= true, + case {Capable, is_thread_event(Event), Data} of + {false, true, _} -> + drop; + {_, _, #{<<"__thread_scoped">> := GuildId}} -> + thread_scoped(Capable, GuildId, Data, State); + {_, _, #{<<"__thread_unscoped">> := GuildId}} -> + thread_unscoped(thread_viewer(Capable, GuildId, State), Data); + _ -> + {ok, Data} + end. + +-spec thread_scoped(boolean(), term(), map(), session_state()) -> drop | {ok, map()}. +thread_scoped(false, _GuildId, _Data, _State) -> + drop; +thread_scoped(true, GuildId, Data, State) -> + case thread_viewer(true, GuildId, State) of + true -> {ok, maps:remove(<<"__thread_scoped">>, Data)}; + false -> drop + end. + +-spec thread_unscoped(boolean(), map()) -> drop | {ok, map()}. +thread_unscoped(true, _Data) -> + drop; +thread_unscoped(false, Data) -> + {ok, maps:remove(<<"__thread_unscoped">>, Data)}. + +-spec thread_viewer(boolean(), term(), session_state()) -> boolean(). +thread_viewer(false, _GuildId, _State) -> + false; +thread_viewer(true, GuildId, State) -> + UserId = maps:get(user_id, State, undefined), + case {channel_threads_config:loaded(), snowflake_id:parse_maybe(GuildId)} of + {false, _} -> + true; + {true, G} when is_integer(G), is_integer(UserId) -> + guild_thread_gate:compute_viewer( + channel_threads_config:guild_active(G), + maps:get(bot, State, false) =:= true, + true, + UserId + ); + _ -> + false + end. + +-spec is_thread_event(event()) -> boolean(). +is_thread_event(thread_create) -> true; +is_thread_event(thread_update) -> true; +is_thread_event(thread_delete) -> true; +is_thread_event(thread_list_sync) -> true; +is_thread_event(thread_member_update) -> true; +is_thread_event(thread_members_update) -> true; +is_thread_event(thread_member_list_update) -> true; +is_thread_event(forum_unreads) -> true; +is_thread_event(<<"THREAD_", _/binary>>) -> true; +is_thread_event(<<"FORUM_UNREADS">>) -> true; +is_thread_event(_) -> false. + -spec route_bot_guild_event(event(), map() | list(), session_state()) -> {noreply, session_state()}. route_bot_guild_event(Event, Data, State) when is_map(Data) -> @@ -310,6 +382,8 @@ should_skip_replay_buffer(Event) -> <<"GUILD_MEMBERS_CHUNK">> -> true; <<"GUILD_MEMBER_LIST_UPDATE">> -> true; <<"GUILD_SYNC">> -> true; + <<"THREAD_LIST_SYNC">> -> true; + <<"THREAD_MEMBER_LIST_UPDATE">> -> true; _Other -> false end. @@ -401,6 +475,7 @@ base_state(Opts) -> presence_pid => undefined, ignored_events => #{}, debounce_reactions => false, + thread_channels_capable => false, reaction_buffer => [], reaction_buffer_timer => undefined }, @@ -641,4 +716,112 @@ assert_no_dispatch() -> ok end. +thread_backstop_drops_thread_events_for_incapable_sessions_test() -> + State = base_state(#{}), + ?assertEqual(drop, thread_backstop(thread_create, #{}, State)), + ?assertEqual(drop, thread_backstop(<<"THREAD_LIST_SYNC">>, #{}, State)), + ?assertEqual( + drop, thread_backstop(message_ack, #{<<"__thread_scoped">> => <<"5">>}, State) + ), + ?assertEqual({ok, #{}}, thread_backstop(message_create, #{}, State)), + ?assertEqual({ok, #{}}, thread_backstop(thread_create, #{}, State#{bot => true})), + ?assertEqual( + {ok, #{}}, thread_backstop(thread_create, #{}, State#{thread_channels_capable => true}) + ). + +thread_backstop_passes_scoped_events_before_the_config_loads_test() -> + with_default_config(false, fun() -> + Data = #{<<"__thread_scoped">> => <<"5">>, <<"id">> => <<"9">>}, + Unscoped = #{<<"__thread_unscoped">> => <<"5">>, <<"id">> => <<"9">>}, + Capable = base_state(#{thread_channels_capable => true}), + ?assertEqual({ok, #{<<"id">> => <<"9">>}}, thread_backstop(message_ack, Data, Capable)), + ?assertEqual(drop, thread_backstop(user_guild_settings_update, Unscoped, Capable)), + ?assertEqual(drop, thread_backstop(message_ack, Data, base_state(#{}))), + ?assertEqual( + {ok, #{<<"id">> => <<"9">>}}, + thread_backstop(user_guild_settings_update, Unscoped, base_state(#{})) + ) + end). + +thread_backstop_fails_closed_once_a_default_config_is_pulled_test() -> + with_default_config(true, fun() -> + Data = #{<<"__thread_scoped">> => <<"5">>, <<"id">> => <<"9">>}, + Unscoped = #{<<"__thread_unscoped">> => <<"5">>, <<"id">> => <<"9">>}, + Capable = base_state(#{thread_channels_capable => true, user_id => 7}), + Bot = base_state(#{thread_channels_capable => true, bot => true, user_id => 7}), + ?assertEqual(drop, thread_backstop(message_ack, Data, Capable)), + ?assertEqual(drop, thread_backstop(message_ack, Data, Bot)), + ?assertEqual( + {ok, #{<<"id">> => <<"9">>}}, + thread_backstop(user_guild_settings_update, Unscoped, Capable) + ) + end). + +with_default_config(Pulled, Fun) -> + Key = channel_threads_config, + PulledKey = {channel_threads_config, pulled}, + Previous = persistent_term:get(Key, undefined), + PreviousPulled = persistent_term:get(PulledKey, undefined), + persistent_term:put(Key, channel_threads_config:default_config()), + _ = persistent_term:erase(PulledKey), + Pulled andalso persistent_term:put(PulledKey, true), + try + Fun() + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end, + case PreviousPulled of + undefined -> persistent_term:erase(PulledKey); + _ -> persistent_term:put(PulledKey, PreviousPulled) + end + end. + +thread_backstop_strips_the_scope_key_for_viewers_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, + enabled_guilds => #{<<"5">> => true}, + included_users => #{<<"1">> => true} + }), + try + Data = #{<<"__thread_scoped">> => <<"5">>, <<"id">> => <<"9">>}, + Capable = base_state(#{thread_channels_capable => true}), + ?assertEqual({ok, #{<<"id">> => <<"9">>}}, thread_backstop(message_ack, Data, Capable)), + ?assertEqual( + drop, + thread_backstop(message_ack, Data#{<<"__thread_scoped">> => <<"6">>}, Capable) + ), + ?assertEqual(drop, thread_backstop(message_ack, Data, Capable#{user_id => 2})), + Unscoped = #{<<"__thread_unscoped">> => <<"5">>, <<"id">> => <<"9">>}, + ?assertEqual(drop, thread_backstop(user_guild_settings_update, Unscoped, Capable)), + ?assertEqual( + {ok, #{<<"id">> => <<"9">>}}, + thread_backstop(user_guild_settings_update, Unscoped, Capable#{user_id => 2}) + ), + ?assertEqual( + {ok, #{<<"id">> => <<"9">>}}, + thread_backstop( + user_guild_settings_update, + Unscoped#{<<"__thread_unscoped">> => <<"6">>}, + Capable + ) + ), + ?assertEqual( + {ok, #{<<"id">> => <<"9">>}}, + thread_backstop(user_guild_settings_update, Unscoped, base_state(#{})) + ) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +thread_subscription_events_skip_the_replay_buffer_test() -> + ?assert(should_skip_replay_buffer(thread_list_sync)), + ?assert(should_skip_replay_buffer(thread_member_list_update)). + -endif. diff --git a/fluxer_gateway/src/session/session_guilds.erl b/fluxer_gateway/src/session/session_guilds.erl index 2952798c0..8dde52c1b 100644 --- a/fluxer_gateway/src/session/session_guilds.erl +++ b/fluxer_gateway/src/session/session_guilds.erl @@ -417,6 +417,7 @@ leave_dispatch_state() -> presence_pid => undefined, ignored_events => #{}, debounce_reactions => false, + thread_channels_capable => false, reaction_buffer => [], reaction_buffer_timer => undefined }. diff --git a/fluxer_gateway/src/session/session_init.erl b/fluxer_gateway/src/session/session_init.erl index cc9137ae3..cf6f80005 100644 --- a/fluxer_gateway/src/session/session_init.erl +++ b/fluxer_gateway/src/session/session_init.erl @@ -267,6 +267,7 @@ extract_extra_fields(D, Ready) -> guild_connect_workers => #{}, guild_connect_timers => #{}, debounce_reactions => maps:get(debounce_reactions, D, false), + thread_channels_capable => maps:get(thread_channels_capable, D, false), reaction_buffer => [], reaction_buffer_timer => undefined }. diff --git a/fluxer_gateway/src/session/session_lifecycle.erl b/fluxer_gateway/src/session/session_lifecycle.erl index 0ef9de6f4..cd769d2ce 100644 --- a/fluxer_gateway/src/session/session_lifecycle.erl +++ b/fluxer_gateway/src/session/session_lifecycle.erl @@ -603,6 +603,7 @@ serialize_state(State) -> bot => maps:get(bot, State, false), shard => maps:get(shard, State, undefined), e2ee_capable => maps:get(e2ee_capable, State, false), + thread_channels_capable => maps:get(thread_channels_capable, State, false), guilds => maps:get(guilds, State, #{}), active_guilds => maps:get(active_guilds, State, sets:new()), collected_guild_states => maps:get(collected_guild_states, State), @@ -640,7 +641,8 @@ serialize_transfer_identity(State) -> ignored_events => maps:keys(maps:get(ignored_events, State, #{})), initial_guild_id => maps:get(initial_guild_id, State, undefined), active_guilds => maps:get(active_guilds, State, sets:new()), - debounce_reactions => maps:get(debounce_reactions, State, false) + debounce_reactions => maps:get(debounce_reactions, State, false), + thread_channels_capable => maps:get(thread_channels_capable, State, false) }. -spec serialize_transfer_runtime(session_state()) -> map(). diff --git a/fluxer_gateway/src/session/session_manager_shard_drain.erl b/fluxer_gateway/src/session/session_manager_shard_drain.erl index 6f1fc9cb7..3984b6b15 100644 --- a/fluxer_gateway/src/session/session_manager_shard_drain.erl +++ b/fluxer_gateway/src/session/session_manager_shard_drain.erl @@ -14,6 +14,7 @@ validate_identify_sharding/3, check_identify_rate_limit/1, should_debounce_reactions/1, + thread_channels_capable/1, extract_e2ee_capable/1, fetch_rpc_data/2 ]). @@ -21,6 +22,8 @@ -export_type([handoff_result/0, identify_data/0, session_id/0, shard_identify_data/0, state/0]). -define(IDENTIFY_FLAG_DEBOUNCE_MESSAGE_REACTIONS, 16#2). +-define(IDENTIFY_FLAG_CHANNEL_THREADS, 16#4). +-define(READ_STATE_FLAG_IS_THREAD, 16#2). -define(SESSION_RPC_RETRY_CONFIG, {1, 1000, 10000, 500}). -type session_id() :: binary(). @@ -188,13 +191,19 @@ build_session_data(Data, IdentifyData, Version, SocketPid, SessionId, UserDataMa mobile => extract_mobile(Presence, Properties), socket_pid => SocketPid, guilds => filter_guild_ids_for_identify(Data, IdentifyData), - ready => build_ready_data_for_session(Data), + ready => filter_thread_read_states( + build_ready_data_for_session(Data), + thread_channels_capable(IdentifyData), + map_utils:get_safe(UserDataMap, <<"bot">>, false) =:= true, + UserId + ), bot => map_utils:get_safe(UserDataMap, <<"bot">>, false), e2ee_capable => extract_e2ee_capable(Properties), ignored_events => term_detach:detach(IgnoredEvents), initial_guild_id => map_utils:get_safe(IdentifyData, initial_guild_id, undefined), shard => Shard, - debounce_reactions => should_debounce_reactions(IdentifyData) + debounce_reactions => should_debounce_reactions(IdentifyData), + thread_channels_capable => thread_channels_capable(IdentifyData) }, BaseFields. @@ -285,6 +294,58 @@ build_ready_data_for_session(Data) -> _ -> #{} end. +-spec filter_thread_read_states(map(), boolean(), boolean(), integer()) -> map(). +filter_thread_read_states(#{<<"read_states">> := Rows} = Ready, Capable, Bot, UserId) when + is_list(Rows) +-> + case lists:any(fun has_read_state_flags/1, Rows) of + false -> + Ready; + true -> + Viewer = + case channel_threads_config:loaded() of + false -> + fun(_Row) -> Capable orelse Bot end; + true -> + fun(Row) -> + guild_thread_gate:compute_viewer( + read_state_guild_active(Row), Bot, Capable, UserId + ) + end + end, + Ready#{ + <<"read_states">> => lists:filtermap( + fun(Row) -> filter_read_state(Row, Capable orelse Bot, Viewer) end, Rows + ) + } + end; +filter_thread_read_states(Ready, _Capable, _Bot, _UserId) -> + Ready. + +-spec has_read_state_flags(term()) -> boolean(). +has_read_state_flags(#{<<"flags">> := _}) -> true; +has_read_state_flags(_) -> false. + +-spec read_state_guild_active(map()) -> boolean(). +read_state_guild_active(Row) -> + case snowflake_id:parse_maybe(maps:get(<<"guild_id">>, Row, undefined)) of + GuildId when is_integer(GuildId) -> channel_threads_config:guild_active(GuildId); + _ -> true + end. + +-spec filter_read_state(term(), boolean(), fun((map()) -> boolean())) -> + boolean() | {true, term()}. +filter_read_state(#{<<"flags">> := Flags} = Row, Capable, Viewer) when is_integer(Flags) -> + case Flags band ?READ_STATE_FLAG_IS_THREAD =/= 0 andalso not Viewer(Row) of + true -> false; + false when Capable -> true; + false -> {true, maps:remove(<<"flags">>, Row)} + end; +filter_read_state(Row, false, _Viewer) when is_map(Row) -> + {true, maps:remove(<<"flags">>, Row)}; +filter_read_state(_Row, _Capable, _Viewer) -> + true. + -spec check_identify_rate_limit(list()) -> {ok, list()} | {error, rate_limited}. check_identify_rate_limit(Attempts) -> case fluxer_gateway_env:get(identify_rate_limit_enabled) of @@ -310,6 +371,15 @@ should_debounce_reactions(IdentifyData) -> false end. +-spec thread_channels_capable(map()) -> boolean(). +thread_channels_capable(IdentifyData) -> + case map_utils:get_safe(IdentifyData, flags, 0) of + Flags when is_integer(Flags), Flags >= 0 -> + bitset:has(Flags, ?IDENTIFY_FLAG_CHANNEL_THREADS); + _ -> + false + end. + -spec extract_e2ee_capable(term()) -> boolean(). extract_e2ee_capable(Properties) when is_map(Properties) -> is_truthy(maps:get(<<"e2ee_capable">>, Properties, false)); @@ -349,9 +419,18 @@ do_fetch_rpc_data(Request, PeerIP) -> <<"ip">> => PeerIP }, RpcWithCoords = session_manager_shard_lookup:add_coordinates(RpcRequest, Lat, Lon), - RpcResult = api_rpc_client:call_with_retry(RpcWithCoords, ?SESSION_RPC_RETRY_CONFIG), + RpcWithCapability = add_thread_channels_capable( + RpcWithCoords, thread_channels_capable(IdentifyData) + ), + RpcResult = api_rpc_client:call_with_retry(RpcWithCapability, ?SESSION_RPC_RETRY_CONFIG), classify_rpc_result(RpcResult, PeerIP). +-spec add_thread_channels_capable(map(), boolean()) -> map(). +add_thread_channels_capable(RpcRequest, true) -> + RpcRequest#{<<"thread_channels_capable">> => true}; +add_thread_channels_capable(RpcRequest, false) -> + RpcRequest. + -spec classify_rpc_result(term(), term()) -> {ok, map()} | {error, term()}. classify_rpc_result({ok, Data}, _PeerIP) when is_map(Data) -> {ok, Data}; classify_rpc_result({error, {rpc_error, 401, _}}, _PeerIP) -> @@ -370,3 +449,87 @@ classify_rpc_result({error, {retries_exhausted, Reason}}, PeerIP) -> {error, {retries_exhausted, Reason}}; classify_rpc_result({error, Reason}, _PeerIP) -> {error, {network_error, Reason}}. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +filter_thread_read_states_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{config_version => 1}), + try + Rows = [ + #{<<"id">> => <<"1">>}, + #{<<"id">> => <<"2">>, <<"flags">> => 1}, + #{<<"id">> => <<"3">>, <<"flags">> => 3} + ], + Ready = #{<<"read_states">> => Rows}, + ?assertEqual( + #{<<"read_states">> => [#{<<"id">> => <<"1">>}, #{<<"id">> => <<"2">>}]}, + filter_thread_read_states(Ready, false, false, 7) + ), + ?assertEqual( + #{ + <<"read_states">> => [ + #{<<"id">> => <<"1">>}, #{<<"id">> => <<"2">>, <<"flags">> => 1} + ] + }, + filter_thread_read_states(Ready, true, false, 7) + ), + Plain = #{<<"read_states">> => [#{<<"id">> => <<"1">>}]}, + ?assertEqual(Plain, filter_thread_read_states(Plain, false, false, 7)) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +filter_thread_read_states_keeps_thread_rows_before_the_config_loads_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, channel_threads_config:default_config()), + try + Thread = #{<<"id">> => <<"1">>, <<"guild_id">> => <<"5">>, <<"flags">> => 3}, + Ready = #{<<"read_states">> => [Thread]}, + ?assertEqual(Ready, filter_thread_read_states(Ready, true, false, 7)), + ?assertEqual(Ready, filter_thread_read_states(Ready, false, true, 7)), + ?assertEqual( + #{<<"read_states">> => []}, filter_thread_read_states(Ready, false, false, 7) + ) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +filter_thread_read_states_checks_the_row_guild_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, + enabled_guilds => #{<<"5">> => true}, + included_users => #{<<"7">> => true} + }), + try + Active = #{<<"id">> => <<"1">>, <<"guild_id">> => <<"5">>, <<"flags">> => 3}, + Inactive = #{<<"id">> => <<"2">>, <<"guild_id">> => <<"6">>, <<"flags">> => 3}, + Ready = #{<<"read_states">> => [Active, Inactive]}, + ?assertEqual( + #{<<"read_states">> => [Active]}, filter_thread_read_states(Ready, true, false, 7) + ), + ?assertEqual( + #{<<"read_states">> => []}, filter_thread_read_states(Ready, false, false, 7) + ), + ?assertEqual( + #{<<"read_states">> => [Active]}, filter_thread_read_states(Ready, false, true, 8) + ) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +-endif. diff --git a/fluxer_gateway/src/session/session_monitor.erl b/fluxer_gateway/src/session/session_monitor.erl index 6e6104553..207be29a5 100644 --- a/fluxer_gateway/src/session/session_monitor.erl +++ b/fluxer_gateway/src/session/session_monitor.erl @@ -293,6 +293,7 @@ build_test_voice_state() -> voice_queue => queue:new(), voice_queue_timer => undefined, debounce_reactions => false, + thread_channels_capable => false, reaction_buffer => [], reaction_buffer_timer => undefined }. diff --git a/fluxer_gateway/src/session/session_state_transfer.erl b/fluxer_gateway/src/session/session_state_transfer.erl index 3e789ef10..20bda1465 100644 --- a/fluxer_gateway/src/session/session_state_transfer.erl +++ b/fluxer_gateway/src/session/session_state_transfer.erl @@ -196,6 +196,7 @@ build_round_trip_state(SessionId) -> ignored_events => [<<"TYPING_START">>], initial_guild_id => 100, debounce_reactions => true, + thread_channels_capable => true, channels => #{1 => #{}}, relationships => #{10 => 1}, seq => 15, @@ -219,7 +220,8 @@ assert_round_trip_fields(SessionId, RS) -> ?assert(sets:is_element(100, maps:get(active_guilds, RS))), ?assert(sets:is_element(300, maps:get(active_guilds, RS))), ?assertEqual(true, maps:get(e2ee_capable, RS)), - ?assertEqual(true, maps:get(debounce_reactions, RS)). + ?assertEqual(true, maps:get(debounce_reactions, RS)), + ?assertEqual(true, maps:get(thread_channels_capable, RS)). push_and_pop(SessionId, SessionState, Store0) -> {reply, ok, Store1} = handle_call( diff --git a/fluxer_gateway/src/utils/constants.erl b/fluxer_gateway/src/utils/constants.erl index a310d1d19..74f9a0ceb 100644 --- a/fluxer_gateway/src/utils/constants.erl +++ b/fluxer_gateway/src/utils/constants.erl @@ -26,6 +26,10 @@ kick_members_permission/0, ban_members_permission/0, view_channel_members_permission/0, + manage_threads_permission/0, + create_public_threads_permission/0, + create_private_threads_permission/0, + send_messages_in_threads_permission/0, voice_channel_camera_user_limit/0 ]). @@ -46,6 +50,7 @@ gateway_opcode(12) -> gateway_error; gateway_opcode(14) -> lazy_request; gateway_opcode(15) -> request_guild_counts; gateway_opcode(16) -> request_channel_member_counts; +gateway_opcode(28) -> request_forum_unreads; gateway_opcode(_) -> unknown. -spec opcode_to_num(atom()) -> integer(). @@ -64,7 +69,8 @@ opcode_to_num(heartbeat_ack) -> 11; opcode_to_num(gateway_error) -> 12; opcode_to_num(lazy_request) -> 14; opcode_to_num(request_guild_counts) -> 15; -opcode_to_num(request_channel_member_counts) -> 16. +opcode_to_num(request_channel_member_counts) -> 16; +opcode_to_num(request_forum_unreads) -> 28. -spec close_code_to_num(atom()) -> integer(). close_code_to_num(unknown_error) -> 4000; @@ -152,6 +158,18 @@ ban_members_permission() -> 4. -spec view_channel_members_permission() -> pos_integer(). view_channel_members_permission() -> 18014398509481984. +-spec manage_threads_permission() -> pos_integer(). +manage_threads_permission() -> 17179869184. + +-spec create_public_threads_permission() -> pos_integer(). +create_public_threads_permission() -> 34359738368. + +-spec create_private_threads_permission() -> pos_integer(). +create_private_threads_permission() -> 68719476736. + +-spec send_messages_in_threads_permission() -> pos_integer(). +send_messages_in_threads_permission() -> 274877906944. + -spec voice_channel_camera_user_limit() -> pos_integer(). voice_channel_camera_user_limit() -> 25. @@ -201,4 +219,14 @@ constants_values_test() -> ?assertEqual(128, view_audit_log_permission()), ?assertEqual(8, administrator_permission()). +thread_permission_bits_test() -> + ?assertEqual(1 bsl 34, manage_threads_permission()), + ?assertEqual(1 bsl 35, create_public_threads_permission()), + ?assertEqual(1 bsl 36, create_private_threads_permission()), + ?assertEqual(1 bsl 38, send_messages_in_threads_permission()). + +dispatch_event_atom_normalizes_thread_events_test() -> + ?assertEqual(thread_member_update, dispatch_event_atom(<<"THREAD_MEMBER_UPDATE">>)), + ?assertEqual(<<"THREAD_MEMBERS_UPDATE">>, dispatch_event_atom(thread_members_update)). + -endif. diff --git a/fluxer_gateway/src/utils/event_atoms.erl b/fluxer_gateway/src/utils/event_atoms.erl index bf8beb169..2a8cffafd 100644 --- a/fluxer_gateway/src/utils/event_atoms.erl +++ b/fluxer_gateway/src/utils/event_atoms.erl @@ -38,6 +38,7 @@ known_event_map() -> channel_event_map(), guild_event_map(), message_event_map(), + thread_event_map(), user_voice_event_map() ] ). @@ -115,6 +116,19 @@ message_event_map() -> <<"MESSAGE_UPDATE">> => message_update }. +-spec thread_event_map() -> #{binary() => atom()}. +thread_event_map() -> + #{ + <<"FORUM_UNREADS">> => forum_unreads, + <<"THREAD_CREATE">> => thread_create, + <<"THREAD_DELETE">> => thread_delete, + <<"THREAD_LIST_SYNC">> => thread_list_sync, + <<"THREAD_MEMBER_LIST_UPDATE">> => thread_member_list_update, + <<"THREAD_MEMBER_UPDATE">> => thread_member_update, + <<"THREAD_MEMBERS_UPDATE">> => thread_members_update, + <<"THREAD_UPDATE">> => thread_update + }. + -spec user_voice_event_map() -> #{binary() => atom()}. user_voice_event_map() -> #{ @@ -145,6 +159,16 @@ normalize_known_private_event_test() -> ?assertEqual(user_guild_settings_update, normalize(<<"USER_GUILD_SETTINGS_UPDATE">>)), ?assertEqual(user_note_update, normalize(<<"USER_NOTE_UPDATE">>)). +normalize_thread_events_test() -> + ?assertEqual(thread_create, normalize(<<"THREAD_CREATE">>)), + ?assertEqual(thread_update, normalize(<<"THREAD_UPDATE">>)), + ?assertEqual(thread_delete, normalize(<<"THREAD_DELETE">>)), + ?assertEqual(thread_list_sync, normalize(<<"THREAD_LIST_SYNC">>)), + ?assertEqual(thread_member_update, normalize(<<"THREAD_MEMBER_UPDATE">>)), + ?assertEqual(thread_members_update, normalize(<<"THREAD_MEMBERS_UPDATE">>)), + ?assertEqual(thread_member_list_update, normalize(<<"THREAD_MEMBER_LIST_UPDATE">>)), + ?assertEqual(forum_unreads, normalize(<<"FORUM_UNREADS">>)). + normalize_binary_unknown_test() -> Result = normalize(<<"UNKNOWN_EVENT_XYZ_12345">>), ?assertEqual(<<"UNKNOWN_EVENT_XYZ_12345">>, Result). diff --git a/fluxer_gateway/src/utils/experiment_bucket.erl b/fluxer_gateway/src/utils/experiment_bucket.erl new file mode 100644 index 000000000..412d4ee8b --- /dev/null +++ b/fluxer_gateway/src/utils/experiment_bucket.erl @@ -0,0 +1,24 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(experiment_bucket). +-typing([eqwalizer]). + +-export([bucket/2, resolution/0]). + +-define(RESOLUTION, 10000). +-define(FNV_OFFSET_BASIS_32, 16#811c9dc5). +-define(FNV_PRIME_32, 16#01000193). + +-spec resolution() -> pos_integer(). +resolution() -> + ?RESOLUTION. + +-spec bucket(binary(), binary()) -> non_neg_integer(). +bucket(Id, Salt) -> + hash(<>, ?FNV_OFFSET_BASIS_32) rem ?RESOLUTION. + +-spec hash(binary(), non_neg_integer()) -> non_neg_integer(). +hash(<<>>, Hash) -> + Hash; +hash(<>, Hash) -> + hash(Rest, ((Hash bxor Byte) * ?FNV_PRIME_32) band 16#ffffffff). diff --git a/fluxer_gateway/test/channel_threads_config_tests.erl b/fluxer_gateway/test/channel_threads_config_tests.erl new file mode 100644 index 000000000..b6db1ede1 --- /dev/null +++ b/fluxer_gateway/test/channel_threads_config_tests.erl @@ -0,0 +1,331 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(channel_threads_config_tests). +-typing([eqwalizer]). +-include_lib("eunit/include/eunit.hrl"). + +-define(KEY, channel_threads_config). +-define(PULLED_KEY, {channel_threads_config, pulled}). + +config_fails_closed_when_nothing_is_stored_test() -> + with_stored(undefined, fun() -> + ?assertEqual(channel_threads_config:default_config(), channel_threads_config:config()), + ?assertEqual(0, channel_threads_config:version()), + ?assertNot(channel_threads_config:ever_enabled()), + ?assertNot(channel_threads_config:guild_active(1)), + ?assertNot(channel_threads_config:user_active(1)), + ?assertNot(channel_threads_config:user_excluded(1)) + end). + +validate_accepts_a_full_wire_config_test() -> + {ok, Config} = channel_threads_config:validate_config(full_wire()), + ?assertMatch( + #{ + enabled := true, + config_version := 7, + ever_enabled := true, + guild_basis_points := 250, + guild_salt := <<"guild salt">>, + user_basis_points := 10000, + user_salt := <<"user-salt">> + }, + Config + ), + ?assertEqual(#{<<"10">> => true, <<"11">> => true}, maps:get(enabled_guilds, Config)), + ?assertEqual(#{<<"12">> => true}, maps:get(disabled_guilds, Config)), + ?assertEqual(#{<<"20">> => true}, maps:get(included_users, Config)), + ?assertEqual(#{<<"21">> => true}, maps:get(excluded_users, Config)). + +validate_fills_defaults_and_ignores_unknown_keys_test() -> + {ok, Config} = channel_threads_config:validate_config(#{ + <<"enabled">> => true, <<"future_field">> => 1 + }), + Default = channel_threads_config:default_config(), + ?assertEqual(Default#{enabled := true}, Config), + ?assertEqual(<<"channel-threads-guild-v1">>, maps:get(guild_salt, Config)), + ?assertEqual(<<"channel-threads-user-v1">>, maps:get(user_salt, Config)). + +validate_rejects_every_malformed_field_test() -> + Invalid = [ + {<<"enabled">>, <<"true">>}, + {<<"ever_enabled">>, 1}, + {<<"config_version">>, -1}, + {<<"config_version">>, 1.5}, + {<<"guild_basis_points">>, 10001}, + {<<"user_basis_points">>, -1}, + {<<"guild_salt">>, <<>>}, + {<<"user_salt">>, binary:copy(<<"a">>, 65)}, + {<<"guild_salt">>, <<"caf", 16#c3, 16#a9>>}, + {<<"user_salt">>, <<"tab\there">>}, + {<<"enabled_guild_ids">>, <<"1">>}, + {<<"disabled_guild_ids">>, [1]}, + {<<"included_user_ids">>, [<<"12a">>]}, + {<<"excluded_user_ids">>, [<<>>]}, + {<<"included_user_ids">>, [binary:copy(<<"1">>, 21)]}, + {<<"enabled_guild_ids">>, [integer_to_binary(N) || N <- lists:seq(1, 1001)]} + ], + lists:foreach( + fun({Key, Value} = Case) -> + ?assertMatch( + {Case, {error, {invalid_field, Key, _}}}, + {Case, channel_threads_config:validate_config(#{Key => Value})} + ) + end, + Invalid + ). + +validate_rejection_never_carries_the_ids_test() -> + Ids = [integer_to_binary(N) || N <- lists:seq(1, 1001)], + ?assertEqual( + {error, {invalid_field, <<"excluded_user_ids">>, 1001}}, + channel_threads_config:validate_config(#{<<"excluded_user_ids">> => Ids}) + ), + ?assertEqual( + {error, {invalid_field, <<"enabled_guild_ids">>, 2}}, + channel_threads_config:validate_config(#{<<"enabled_guild_ids">> => [<<"1">>, <<"x">>]}) + ), + ?assertEqual( + {error, {invalid_field, <<"guild_salt">>, undefined}}, + channel_threads_config:validate_config(#{<<"guild_salt">> => <<>>}) + ). + +validate_accepts_the_boundaries_test() -> + {ok, _} = channel_threads_config:validate_config(#{ + <<"guild_basis_points">> => 10000, + <<"user_basis_points">> => 0, + <<"guild_salt">> => binary:copy(<<"~">>, 64), + <<"included_user_ids">> => [<<"18446744073709551615">>], + <<"disabled_guild_ids">> => [integer_to_binary(N) || N <- lists:seq(1, 1000)] + }). + +guild_precedence_is_excluded_then_included_then_bucket_test() -> + Config = config(#{ + <<"guild_basis_points">> => 10000, + <<"enabled_guild_ids">> => [<<"5">>], + <<"disabled_guild_ids">> => [<<"5">>, <<"6">>] + }), + ?assertNot(channel_threads_config:guild_active(Config, 5)), + ?assertNot(channel_threads_config:guild_active(Config, 6)), + ?assert(channel_threads_config:guild_active(Config, 7)), + Included = config(#{<<"enabled_guild_ids">> => [<<"8">>]}), + ?assert(channel_threads_config:guild_active(Included, 8)), + ?assertNot(channel_threads_config:guild_active(Included, 9)). + +guild_bucket_threshold_matches_the_shared_vectors_test() -> + ?assertNot( + channel_threads_config:guild_active(config(#{<<"guild_basis_points">> => 6741}), 1) + ), + ?assert( + channel_threads_config:guild_active(config(#{<<"guild_basis_points">> => 6742}), 1) + ), + ?assertNot( + channel_threads_config:guild_active(config(#{<<"guild_basis_points">> => 9122}), 0) + ), + ?assert( + channel_threads_config:guild_active(config(#{<<"guild_basis_points">> => 9123}), 0) + ). + +user_bucket_threshold_matches_the_shared_vectors_test() -> + ?assertNot( + channel_threads_config:user_active(config(#{<<"user_basis_points">> => 3403}), 1) + ), + ?assert(channel_threads_config:user_active(config(#{<<"user_basis_points">> => 3404}), 1)). + +user_precedence_and_exclusion_test() -> + Config = config(#{ + <<"user_basis_points">> => 10000, + <<"included_user_ids">> => [<<"1">>], + <<"excluded_user_ids">> => [<<"1">>, <<"2">>] + }), + ?assertNot(channel_threads_config:user_active(Config, 1)), + ?assertNot(channel_threads_config:user_active(Config, 2)), + ?assert(channel_threads_config:user_active(Config, 3)), + ?assert(channel_threads_config:user_excluded(Config, 2)), + ?assertNot(channel_threads_config:user_excluded(Config, 3)). + +disabled_config_is_inactive_for_everyone_test() -> + {ok, Config} = channel_threads_config:validate_config( + (full_wire())#{<<"enabled">> => false} + ), + ?assertNot(channel_threads_config:guild_active(Config, 10)), + ?assertNot(channel_threads_config:user_active(Config, 20)), + ?assert(channel_threads_config:user_excluded(Config, 21)). + +zero_basis_points_never_buckets_in_test() -> + Config = config(#{}), + ?assertEqual([], [ + Id + || Id <- lists:seq(1, 2000), channel_threads_config:guild_active(Config, Id) + ]), + ?assertEqual([], [ + Id + || Id <- lists:seq(1, 2000), channel_threads_config:user_active(Config, Id) + ]). + +store_keeps_the_version_monotonic_test() -> + with_stored(undefined, fun() -> + V5 = config(#{<<"config_version">> => 5}), + V4 = config(#{<<"config_version">> => 4, <<"enabled">> => false}), + Default = channel_threads_config:default_config(), + ?assertEqual(updated, channel_threads_config:store_validated_config(Default, V5, nats)), + ?assertEqual(5, channel_threads_config:version()), + ?assertEqual(stale, channel_threads_config:store_validated_config(V5, V4, api)), + ?assertEqual(5, channel_threads_config:version()), + ?assert(maps:get(enabled, channel_threads_config:config())), + ?assertEqual(unchanged, channel_threads_config:store_validated_config(V5, V5, api)) + end). + +store_keeps_ever_enabled_sticky_test() -> + with_stored(undefined, fun() -> + Enabled = config(#{<<"config_version">> => 1, <<"ever_enabled">> => true}), + Killed = config(#{ + <<"config_version">> => 2, <<"enabled">> => false, <<"ever_enabled">> => false + }), + Default = channel_threads_config:default_config(), + updated = channel_threads_config:store_validated_config(Default, Enabled, api), + updated = channel_threads_config:store_validated_config(Enabled, Killed, api), + ?assert(channel_threads_config:ever_enabled()), + ?assertNot(maps:get(enabled, channel_threads_config:config())), + ?assertEqual( + unchanged, + channel_threads_config:store_validated_config( + channel_threads_config:config(), Killed, api + ) + ) + end). + +nats_payload_is_validated_before_it_is_stored_test() -> + with_stored(undefined, fun() -> + Payload = iolist_to_binary( + json:encode(#{ + <<"type">> => <<"channel_threads_config">>, + <<"config">> => #{<<"enabled">> => true, <<"config_version">> => 3} + }) + ), + ?assertEqual(updated, channel_threads_config:apply_nats_payload(Payload)), + ?assertEqual(3, channel_threads_config:version()), + Bad = iolist_to_binary( + json:encode(#{ + <<"config">> => #{<<"enabled">> => <<"yes">>, <<"config_version">> => 9} + }) + ), + ?assertEqual(rejected, channel_threads_config:apply_nats_payload(Bad)), + ?assertEqual(rejected, channel_threads_config:apply_nats_payload(<<"not json">>)), + ?assertEqual(rejected, channel_threads_config:apply_nats_payload(<<"[1]">>)), + ?assertEqual(3, channel_threads_config:version()) + end). + +a_default_pull_counts_as_loaded_test() -> + with_stored(undefined, fun() -> + ?assertNot(channel_threads_config:loaded()), + Default = iolist_to_binary( + json:encode(#{<<"config">> => #{<<"enabled">> => false, <<"config_version">> => 0}}) + ), + ?assertEqual(unchanged, channel_threads_config:apply_nats_payload(Default)), + ?assertEqual(channel_threads_config:default_config(), channel_threads_config:config()), + ?assert(channel_threads_config:loaded()), + ?assertNot(channel_threads_config:enabled()) + end). + +a_rejected_pull_is_not_loaded_test() -> + with_stored(undefined, fun() -> + ?assertEqual(rejected, channel_threads_config:apply_nats_payload(<<"not json">>)), + ?assertNot(channel_threads_config:loaded()) + end). + +fields_changed_are_split_by_dimension_test() -> + Base = config(#{}), + GuildOnly = config(#{<<"enabled_guild_ids">> => [<<"1">>]}), + UserOnly = config(#{<<"user_basis_points">> => 5}), + VersionOnly = config(#{<<"config_version">> => 9}), + Disabled = config(#{<<"enabled">> => false}), + ?assert(channel_threads_config:guild_fields_changed(Base, GuildOnly)), + ?assertNot(channel_threads_config:user_fields_changed(Base, GuildOnly)), + ?assert(channel_threads_config:user_fields_changed(Base, UserOnly)), + ?assertNot(channel_threads_config:guild_fields_changed(Base, UserOnly)), + ?assertNot(channel_threads_config:guild_fields_changed(Base, VersionOnly)), + ?assertNot(channel_threads_config:user_fields_changed(Base, VersionOnly)), + ?assert(channel_threads_config:guild_fields_changed(Base, Disabled)), + ?assert(channel_threads_config:user_fields_changed(Base, Disabled)). + +identify_os_label_buckets_client_strings_test() -> + Cases = [ + {<<"Windows">>, <<"windows">>}, + {<<"Mac OS X">>, <<"macos">>}, + {<<"macOS">>, <<"macos">>}, + {<<"Darwin">>, <<"macos">>}, + {<<"Linux">>, <<"linux">>}, + {<<"Android">>, <<"android">>}, + {<<"iOS">>, <<"ios">>}, + {<<"iPadOS">>, <<"ios">>}, + {<<"FreeBSD">>, <<"linux">>}, + {<<"TempleOS">>, <<"other">>}, + {<<>>, <<"other">>}, + {undefined, <<"other">>} + ], + lists:foreach( + fun({Input, Expected}) -> + ?assertEqual( + {Input, Expected}, {Input, channel_threads_config:identify_os_label(Input)} + ) + end, + Cases + ). + +note_identify_counts_by_capability_and_os_test() -> + Key = {channel_threads_config, identify_counts}, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, counters:new(12, [write_concurrency])), + try + ok = channel_threads_config:note_identify(true, #{<<"os">> => <<"Windows">>}), + ok = channel_threads_config:note_identify(true, #{<<"os">> => <<"Windows">>}), + ok = channel_threads_config:note_identify(false, #{<<"os">> => <<"Android">>}), + ok = channel_threads_config:note_identify(false, #{}), + Counts = channel_threads_config:identify_counts(), + ?assertEqual(12, map_size(Counts)), + ?assertEqual(2, maps:get({true, <<"windows">>}, Counts)), + ?assertEqual(1, maps:get({false, <<"android">>}, Counts)), + ?assertEqual(1, maps:get({false, <<"other">>}, Counts)), + ?assertEqual(0, maps:get({true, <<"android">>}, Counts)) + after + restore(Key, Previous) + end. + +config(Overrides) -> + {ok, Config} = channel_threads_config:validate_config( + maps:merge(#{<<"enabled">> => true}, Overrides) + ), + Config. + +full_wire() -> + #{ + <<"enabled">> => true, + <<"config_version">> => 7, + <<"ever_enabled">> => true, + <<"guild_basis_points">> => 250, + <<"guild_salt">> => <<"guild salt">>, + <<"enabled_guild_ids">> => [<<"10">>, <<"11">>], + <<"disabled_guild_ids">> => [<<"12">>], + <<"user_basis_points">> => 10000, + <<"user_salt">> => <<"user-salt">>, + <<"included_user_ids">> => [<<"20">>], + <<"excluded_user_ids">> => [<<"21">>] + }. + +with_stored(Value, Fun) -> + Previous = persistent_term:get(?KEY, undefined), + PreviousPulled = persistent_term:get(?PULLED_KEY, undefined), + restore(?KEY, Value), + restore(?PULLED_KEY, undefined), + try + Fun() + after + restore(?KEY, Previous), + restore(?PULLED_KEY, PreviousPulled) + end. + +restore(Key, undefined) -> + _ = persistent_term:erase(Key), + ok; +restore(Key, Value) -> + persistent_term:put(Key, Value). diff --git a/fluxer_gateway/test/channel_threads_flip_tests.erl b/fluxer_gateway/test/channel_threads_flip_tests.erl new file mode 100644 index 000000000..eb7fc2bc4 --- /dev/null +++ b/fluxer_gateway/test/channel_threads_flip_tests.erl @@ -0,0 +1,296 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(channel_threads_flip_tests). +-typing([eqwalizer]). +-include_lib("eunit/include/eunit.hrl"). + +guild_fan_out_touches_only_guilds_whose_activity_changed_test() -> + Previous = config(#{<<"enabled_guild_ids">> => [<<"1">>, <<"2">>]}), + Current = config(#{<<"enabled_guild_ids">> => [<<"2">>, <<"3">>]}), + Candidates = [{guild, Id, self()} || Id <- [1, 2, 3, 4]] ++ [{user, 1, self()}], + ?assertEqual( + [{guild, 1, self()}, {guild, 3, self()}], + channel_threads_flip:affected_targets(Previous, Current, Candidates) + ). + +user_fan_out_touches_only_users_whose_activity_or_exclusion_changed_test() -> + Previous = config(#{<<"included_user_ids">> => [<<"1">>]}), + Current = config(#{ + <<"included_user_ids">> => [<<"1">>, <<"2">>], <<"excluded_user_ids">> => [<<"3">>] + }), + Candidates = [{user, Id, self()} || Id <- [1, 2, 3, 4]] ++ [{guild, 2, self()}], + ?assertEqual( + [{user, 2, self()}, {user, 3, self()}], + channel_threads_flip:affected_targets(Previous, Current, Candidates) + ). + +user_only_change_also_recomputes_active_guilds_test() -> + Previous = config(#{<<"enabled_guild_ids">> => [<<"1">>]}), + Current = Previous#{excluded_users := #{<<"7">> => true}, config_version := 2}, + Candidates = [{guild, 1, self()}, {guild, 2, self()}, {user, 7, self()}], + ?assertEqual( + [{viewers, 1, self()}, {user, 7, self()}], + channel_threads_flip:affected_targets(Previous, Current, Candidates) + ). + +guild_flip_is_not_queued_behind_a_viewer_sweep_test() -> + Sweep = [{viewers, Id, self()} || Id <- lists:seq(1, 100)], + Pending0 = channel_threads_flip:schedule(Sweep, 0, empty(), fun() -> 0 end), + Pending1 = channel_threads_flip:schedule([{guild, 500, self()}], 10, Pending0, fun() -> + 0 + end), + {Due, _Pending2} = channel_threads_flip:take_due(Pending1, 10, limits(10, 1000)), + ?assert(lists:member({guild, 500, self()}, Due)), + ?assertEqual(10, length([T || {viewers, _, _} = T <- Due])). + +first_load_flips_guilds_but_not_users_test() -> + Previous = channel_threads_config:default_config(), + Current = config(#{ + <<"config_version">> => 3, + <<"enabled_guild_ids">> => [<<"1">>], + <<"user_basis_points">> => 10000 + }), + Candidates = [{guild, 1, self()}, {guild, 2, self()}, {user, 7, self()}], + ?assertEqual( + [{guild, 1, self()}], + channel_threads_flip:affected_targets(Previous, Current, Candidates) + ). + +version_only_change_touches_nothing_test() -> + Previous = config(#{<<"guild_basis_points">> => 10000, <<"user_basis_points">> => 10000}), + Current = Previous#{config_version := 42}, + Candidates = + [{guild, Id, self()} || Id <- lists:seq(1, 50)] ++ + [{user, Id, self()} || Id <- lists:seq(1, 50)], + ?assertEqual([], channel_threads_flip:affected_targets(Previous, Current, Candidates)). + +changes_while_disabled_touch_nothing_test() -> + Previous = config(#{<<"enabled">> => false}), + Current = config(#{ + <<"enabled">> => false, + <<"enabled_guild_ids">> => [<<"1">>], + <<"included_user_ids">> => [<<"1">>], + <<"excluded_user_ids">> => [<<"2">>] + }), + Candidates = [{guild, 1, self()}, {user, 1, self()}, {user, 2, self()}], + ?assertEqual([], channel_threads_flip:affected_targets(Previous, Current, Candidates)). + +disabled_config_change_skips_registry_scan_test() -> + process_registry:init(), + Entries = [{{guild, 9200001}, self()}, {{presence, 9200001}, self()}], + ets:insert(process_registry_table, Entries), + try + Previous = config(#{<<"enabled">> => false}), + Current = config(#{ + <<"enabled">> => false, + <<"enabled_guild_ids">> => [<<"9200001">>], + <<"included_user_ids">> => [<<"9200001">>] + }), + State0 = #{pending => empty(), timer => undefined, healed => #{}}, + ?assertEqual( + {noreply, State0}, + channel_threads_flip:handle_cast({config_changed, Previous, Current}, State0) + ) + after + [ets:delete(process_registry_table, Key) || {Key, _} <- Entries] + end. + +global_kill_flips_every_active_guild_test() -> + Previous = config(#{<<"guild_basis_points">> => 10000}), + Current = Previous#{enabled := false, config_version := 2}, + Candidates = [{guild, Id, self()} || Id <- lists:seq(1, 20)], + ?assertEqual( + Candidates, channel_threads_flip:affected_targets(Previous, Current, Candidates) + ). + +schedule_applies_jitter_and_dedupes_targets_test() -> + Jitters = counter_jitter([500, 100, 900]), + Pending0 = channel_threads_flip:schedule( + [{guild, 1, self()}, {guild, 2, self()}, {guild, 1, self()}, {user, 1, self()}], + 1000, + empty(), + Jitters + ), + ?assertEqual(#{{guild, 1} => 1500, {guild, 2} => 1100, {user, 1} => 1900}, keys(Pending0)), + {Due, Pending1} = channel_threads_flip:take_due(Pending0, 1600, limits(10, 10)), + ?assertEqual([{guild, 2, self()}, {guild, 1, self()}], Due), + ?assertEqual(#{{user, 1} => 1900}, keys(Pending1)). + +rescheduling_a_restarted_process_keeps_its_slot_and_takes_the_new_pid_test() -> + Stale = spawn(fun() -> ok end), + Pending0 = channel_threads_flip:schedule([{guild, 1, Stale}], 0, empty(), fun() -> 500 end), + Pending1 = channel_threads_flip:schedule([{guild, 1, self()}], 100, Pending0, fun() -> + 900 + end), + ?assertEqual(#{{guild, 1} => 500}, keys(Pending1)), + ?assertEqual( + {[{guild, 1, self()}], empty()}, + channel_threads_flip:take_due(Pending1, 500, limits(10, 10)) + ). + +take_due_respects_the_batch_size_test() -> + Pending0 = channel_threads_flip:schedule( + [{guild, Id, self()} || Id <- lists:seq(1, 25)], 0, empty(), fun() -> 0 end + ), + {First, Pending1} = channel_threads_flip:take_due(Pending0, 0, limits(10, 1000)), + {Second, Pending2} = channel_threads_flip:take_due(Pending1, 0, limits(10, 1000)), + {Third, Pending3} = channel_threads_flip:take_due(Pending2, 0, limits(10, 1000)), + ?assertEqual([10, 10, 5], [length(First), length(Second), length(Third)]), + ?assertEqual(#{}, keys(Pending3)), + ?assertEqual( + lists:seq(1, 25), lists:sort([Id || {guild, Id, _} <- First ++ Second ++ Third]) + ). + +user_flips_drain_in_large_batches_behind_throttled_guilds_test() -> + Targets = + [{guild, Id, self()} || Id <- lists:seq(1, 50)] ++ + [{user, Id, self()} || Id <- lists:seq(1, 2500)], + Pending0 = channel_threads_flip:schedule(Targets, 0, empty(), fun() -> 0 end), + {First, Pending1} = channel_threads_flip:take_due(Pending0, 0, limits(10, 1000)), + ?assertEqual({10, 1000}, kind_counts(First)), + {Second, Pending2} = channel_threads_flip:take_due(Pending1, 0, limits(10, 1000)), + {Third, Pending3} = channel_threads_flip:take_due(Pending2, 0, limits(10, 1000)), + ?assertEqual([{10, 1000}, {10, 500}], [kind_counts(Second), kind_counts(Third)]), + ?assertEqual(20, map_size(keys(Pending3))), + ?assert(lists:all(fun({Kind, _Id}) -> Kind =:= guild end, maps:keys(keys(Pending3)))). + +self_heal_flips_immediately_once_per_cooldown_test() -> + Pending0 = channel_threads_flip:schedule( + [{guild, 7, self()}, {guild, 8, self()}], 0, empty(), fun() -> 50000 end + ), + State0 = #{pending => Pending0, timer => undefined, healed => #{}}, + {Healed1, Pending1} = channel_threads_flip:heal(7, self(), 1000, State0), + ?assertEqual(ok, receive_flip()), + ?assertEqual(#{{guild, 8} => 50000}, keys(Pending1)), + State1 = State0#{healed := Healed1, pending := Pending1}, + {Healed2, Pending2} = channel_threads_flip:heal(7, self(), 20000, State1), + ?assertEqual(none, receive_flip()), + ?assertEqual(Pending1, Pending2), + {_Healed3, _Pending3} = channel_threads_flip:heal( + 7, self(), 40000, State1#{healed := Healed2} + ), + ?assertEqual(ok, receive_flip()). + +config_change_flips_only_local_registry_targets_test() -> + process_registry:init(), + Parent = self(), + [LocalGuild, OtherGuild, LocalUser, ExcludedUser, Session] = + [spawn(fun() -> forward(Parent) end) || _ <- lists:seq(1, 5)], + Remote = remote_pid(), + Entries = [ + {{guild, 9100001}, LocalGuild}, + {{guild, 9100002}, Remote}, + {{guild, 9100003}, OtherGuild}, + {{guild, 9100004}, loading}, + {{guild, <<"9100005">>}, OtherGuild}, + {{presence, 9100001}, LocalUser}, + {{presence, 9100002}, ExcludedUser}, + {{presence, 9100003}, Remote}, + {{session, 9100001}, Session} + ], + ets:insert(process_registry_table, Entries), + try + Previous = config(#{}), + Current = config(#{ + <<"enabled_guild_ids">> => [<<"9100001">>, <<"9100002">>, <<"9100004">>], + <<"included_user_ids">> => [<<"9100001">>, <<"9100003">>], + <<"excluded_user_ids">> => [<<"9100002">>] + }), + State0 = #{pending => empty(), timer => undefined, healed => #{}}, + {noreply, #{pending := Scheduled, timer := Timer}} = + channel_threads_flip:handle_cast({config_changed, Previous, Current}, State0), + _ = erlang:cancel_timer(Timer), + Expected = [ + {guild, 9100001, LocalGuild}, + {user, 9100001, LocalUser}, + {user, 9100002, ExcludedUser} + ], + ?assertEqual(lists:sort(Expected), lists:sort(ours(Scheduled))), + Due = channel_threads_flip:schedule( + Expected, erlang:monotonic_time(millisecond), empty(), fun() -> 0 end + ), + {noreply, #{pending := Rest}} = + channel_threads_flip:handle_info(flip_tick, State0#{pending := Due}), + ?assertEqual(#{}, keys(Rest)), + Version = channel_threads_config:version(), + ?assertEqual( + lists:sort([ + {LocalGuild, {thread_gate_flip, Version}}, + {LocalUser, {thread_user_flip, Version}}, + {ExcludedUser, {thread_user_flip, Version}} + ]), + lists:sort(collect(3)) + ), + ?assertEqual([], collect(1)) + after + [ets:delete(process_registry_table, Key) || {Key, _} <- Entries], + [exit(Pid, kill) || Pid <- [LocalGuild, OtherGuild, LocalUser, ExcludedUser, Session]] + end. + +ours(#{due := Due}) -> + [ + {Kind, Id, Pid} + || KindDue <- maps:values(Due), + {_At, {Kind, Id}, Pid} <- gb_sets:to_list(KindDue), + Id div 100 =:= 91000 + ]. + +limits(Guilds, Users) -> + #{guild => Guilds, viewers => Guilds, user => Users}. + +kind_counts(Targets) -> + {length([T || {guild, _, _} = T <- Targets]), length([T || {user, _, _} = T <- Targets])}. + +forward(Parent) -> + receive + Msg -> + Parent ! {self(), Msg}, + forward(Parent) + end. + +collect(0) -> + []; +collect(N) -> + receive + {Pid, {Tag, _} = Msg} when + is_pid(Pid), (Tag =:= thread_gate_flip orelse Tag =:= thread_user_flip) + -> + [{Pid, Msg} | collect(N - 1)] + after 200 -> [] + end. + +remote_pid() -> + Node = <<"remote@nowhere">>, + binary_to_term( + <<131, 88, 119, (byte_size(Node)), Node/binary, 1:32, 0:32, 1:32>> + ). + +receive_flip() -> + receive + {thread_gate_flip, Version} when is_integer(Version) -> ok + after 0 -> none + end. + +counter_jitter(Values) -> + Ref = make_ref(), + put(Ref, Values), + fun() -> + [Next | Rest] = get(Ref), + put(Ref, Rest), + Next + end. + +empty() -> + #{ + due => #{guild => gb_sets:new(), viewers => gb_sets:new(), user => gb_sets:new()}, + keys => #{} + }. + +keys(#{keys := Keys}) -> + maps:map(fun(_Key, {At, _Pid}) -> At end, Keys). + +config(Overrides) -> + {ok, Config} = channel_threads_config:validate_config( + maps:merge(#{<<"enabled">> => true}, Overrides) + ), + Config. diff --git a/fluxer_gateway/test/experiment_bucket_tests.erl b/fluxer_gateway/test/experiment_bucket_tests.erl new file mode 100644 index 000000000..570dc36f8 --- /dev/null +++ b/fluxer_gateway/test/experiment_bucket_tests.erl @@ -0,0 +1,41 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(experiment_bucket_tests). +-typing([eqwalizer]). +-include_lib("eunit/include/eunit.hrl"). + +-define(VECTORS_PATH, "../packages/schema/src/domains/experiment/ExperimentBucketVectors.json"). + +bucket_matches_shared_vectors_test() -> + Vectors = read_vectors(), + ?assertMatch([_ | _], Vectors), + lists:foreach(fun run_vector/1, Vectors). + +bucket_stays_below_resolution_test() -> + Resolution = experiment_bucket:resolution(), + lists:foreach( + fun(N) -> + Bucket = experiment_bucket:bucket(integer_to_binary(N), <<"range-check">>), + ?assert(Bucket >= 0 andalso Bucket < Resolution) + end, + lists:seq(0, 500) + ). + +read_vectors() -> + case file:read_file(?VECTORS_PATH) of + {ok, Contents} -> decode_vectors(Contents); + {error, Reason} -> erlang:error({bucket_vectors_unreadable, ?VECTORS_PATH, Reason}) + end. + +decode_vectors(Contents) -> + case json:decode(Contents) of + [_ | _] = Vectors -> Vectors; + _ -> erlang:error({bucket_vectors_empty, ?VECTORS_PATH}) + end. + +run_vector([Salt, Id, Expected] = Vector) when + is_binary(Salt), is_binary(Id), is_integer(Expected) +-> + ?assertEqual({Vector, Expected}, {Vector, experiment_bucket:bucket(Id, Salt)}); +run_vector(Vector) -> + erlang:error({bucket_vector_malformed, Vector}). diff --git a/fluxer_gateway/test/gateway_handler_tests.erl b/fluxer_gateway/test/gateway_handler_tests.erl index 1e3e7cbd5..8f3cada30 100644 --- a/fluxer_gateway/test/gateway_handler_tests.erl +++ b/fluxer_gateway/test/gateway_handler_tests.erl @@ -518,6 +518,125 @@ websocket_handle_rejects_compressed_frame_past_max_payload_test() -> ?assertEqual(constants:close_code_to_num(decode_error), CloseCode), ?assertEqual(<<"Payload too large">>, Reason). +forum_unreads_gate_off_closes_like_an_unknown_opcode_test() -> + Previous = persistent_term:get(channel_threads_config, undefined), + persistent_term:put( + channel_threads_config, (channel_threads_config:default_config())#{config_version => 1} + ), + try + State = (new_json_state())#{session_pid => self()}, + Unknown = gateway_handler_dispatch:handle_opcode( + not_an_opcode, #{<<"d">> => #{}}, State + ), + ?assertEqual({[{close, 4001, <<"Unknown opcode">>}], State}, Unknown), + ?assertEqual( + Unknown, + gateway_handler_dispatch:handle_opcode( + request_forum_unreads, + #{<<"d">> => #{<<"guild_id">> => <<"1">>, <<"channel_id">> => <<"2">>}}, + State + ) + ), + ?assertEqual( + Unknown, gateway_handler_dispatch:handle_request_forum_unreads(#{}, self(), State) + ), + ?assertEqual( + Unknown, gateway_handler:websocket_info(forum_unreads_unknown_opcode, State) + ) + after + case Previous of + undefined -> persistent_term:erase(channel_threads_config); + _ -> persistent_term:put(channel_threads_config, Previous) + end + end. + +forum_unreads_closes_like_an_unknown_opcode_once_a_default_config_is_pulled_test() -> + Key = channel_threads_config, + PulledKey = {channel_threads_config, pulled}, + Previous = persistent_term:get(Key, undefined), + PreviousPulled = persistent_term:get(PulledKey, undefined), + persistent_term:put(Key, channel_threads_config:default_config()), + persistent_term:put(PulledKey, true), + try + Bot = fake_session(#{user_id => <<"6">>, bot => true, thread_channels_capable => true}), + State = (new_json_state())#{session_pid => Bot}, + Unknown = {[{close, 4001, <<"Unknown opcode">>}], State}, + ?assertEqual( + Unknown, + gateway_handler_dispatch:handle_opcode( + request_forum_unreads, + #{<<"d">> => #{<<"guild_id">> => <<"1">>, <<"channel_id">> => <<"2">>}}, + State + ) + ), + ?assertEqual( + Unknown, gateway_handler_dispatch:handle_request_forum_unreads(#{}, Bot, State) + ), + ?assertNot(received_unknown_opcode()) + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end, + case PreviousPulled of + undefined -> persistent_term:erase(PulledKey); + _ -> persistent_term:put(PulledKey, PreviousPulled) + end + end. + +forum_unreads_gate_on_closes_non_viewers_on_every_path_test() -> + with_threads_enabled(fun() -> + NonViewer = fake_session(#{user_id => <<"5">>, thread_channels_capable => false}), + State = (new_json_state())#{session_pid => NonViewer}, + {ok, _} = gateway_handler_dispatch:handle_request_forum_unreads(#{}, NonViewer, State), + ?assert(received_unknown_opcode()), + Full = State#{request_worker_max => 1, request_workers => #{make_ref() => #{}}}, + {ok, Full} = gateway_handler_dispatch:handle_request_forum_unreads( + #{}, NonViewer, Full + ), + ?assert(received_unknown_opcode()), + Dead = spawn(fun() -> ok end), + DeadState = State#{session_pid => Dead}, + {ok, _} = gateway_handler_dispatch:handle_request_forum_unreads(#{}, Dead, DeadState), + ?assert(received_unknown_opcode()), + Viewer = fake_session(#{user_id => <<"6">>, bot => true}), + ViewerFull = Full#{session_pid => Viewer}, + {ok, ViewerFull} = gateway_handler_dispatch:handle_request_forum_unreads( + #{}, Viewer, ViewerFull + ), + ?assertNot(received_unknown_opcode()) + end). + +with_threads_enabled(Fun) -> + Previous = persistent_term:get(channel_threads_config, undefined), + persistent_term:put( + channel_threads_config, (channel_threads_config:default_config())#{enabled => true} + ), + try + Fun() + after + case Previous of + undefined -> persistent_term:erase(channel_threads_config); + _ -> persistent_term:put(channel_threads_config, Previous) + end + end. + +fake_session(SessionState) -> + spawn(fun Loop() -> + receive + {'$gen_call', From, {get_state}} -> + gen_server:reply(From, SessionState), + Loop() + after 2000 -> ok + end + end). + +received_unknown_opcode() -> + receive + forum_unreads_unknown_opcode -> true + after 300 -> false + end. + new_json_state() -> (gateway_handler:new_state())#{ version => 1, encoding => json, compress_ctx => gateway_compress:new_context(none) diff --git a/fluxer_gateway/test/gateway_rpc_guild_channels_tests.erl b/fluxer_gateway/test/gateway_rpc_guild_channels_tests.erl index 20a4c492f..61bd7f82c 100644 --- a/fluxer_gateway/test/gateway_rpc_guild_channels_tests.erl +++ b/fluxer_gateway/test/gateway_rpc_guild_channels_tests.erl @@ -25,3 +25,31 @@ resolve_channel_mentions_skips_invalid_channel_ids_test() -> #{<<"channels">> => []}, gateway_rpc_guild_channels:handle(<<"guild.resolve_channel_mentions">>, Params) ). + +resolve_channel_mentions_from_snapshot_skips_forum_and_media_test() -> + GuildId = 7301, + ok = guild_permission_cache:put_data(GuildId, #{ + <<"guild">> => #{<<"owner_id">> => <<"1">>}, + <<"roles">> => [#{<<"id">> => <<"7301">>, <<"permissions">> => <<"1024">>}], + <<"members">> => [], + <<"channels">> => [ + #{<<"id">> => <<"10">>, <<"name">> => <<"text">>, <<"type">> => 0}, + #{<<"id">> => <<"11">>, <<"name">> => <<"forum">>, <<"type">> => 15}, + #{<<"id">> => <<"12">>, <<"name">> => <<"media">>, <<"type">> => 16} + ] + }), + try + Params = #{ + <<"guild_id">> => <<"7301">>, <<"channel_ids">> => [<<"10">>, <<"11">>, <<"12">>] + }, + ?assertEqual( + #{ + <<"channels">> => [ + #{<<"id">> => <<"10">>, <<"name">> => <<"text">>, <<"type">> => 0} + ] + }, + gateway_rpc_guild_channels:handle(<<"guild.resolve_channel_mentions">>, Params) + ) + after + guild_permission_cache:delete(GuildId) + end. diff --git a/fluxer_gateway/test/gateway_rpc_guild_lifecycle_tests.erl b/fluxer_gateway/test/gateway_rpc_guild_lifecycle_tests.erl new file mode 100644 index 000000000..529ba430a --- /dev/null +++ b/fluxer_gateway/test/gateway_rpc_guild_lifecycle_tests.erl @@ -0,0 +1,115 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(gateway_rpc_guild_lifecycle_tests). + +-include_lib("eunit/include/eunit.hrl"). + +with_guild_pid(Fun) -> + meck:new(gateway_rpc_guild_infra, [passthrough, no_link]), + meck:expect(gateway_rpc_guild_infra, with_guild_unchecked, fun(_GuildId, GuildFun) -> + GuildFun(self()) + end), + try + Fun() + after + meck:unload(gateway_rpc_guild_infra) + end. + +dispatch_many_casts_every_event_in_order_test() -> + with_guild_pid(fun() -> + Params = #{ + <<"guild_id">> => <<"1">>, + <<"events">> => [ + #{<<"event">> => <<"THREAD_UPDATE">>, <<"data">> => #{<<"id">> => <<"5">>}}, + #{ + <<"event">> => <<"THREAD_MEMBER_UPDATE">>, + <<"data">> => #{<<"id">> => <<"5">>} + }, + #{<<"event">> => <<"MESSAGE_CREATE">>, <<"data">> => #{<<"id">> => <<"6">>}} + ] + }, + ?assertEqual( + true, gateway_rpc_guild_lifecycle:handle(<<"guild.dispatch_many">>, Params) + ), + receive + {'$gen_cast', {dispatch_many, Requests}} -> + ?assertEqual( + [ + #{event => thread_update, data => #{<<"id">> => <<"5">>}}, + #{event => thread_member_update, data => #{<<"id">> => <<"5">>}}, + #{event => message_create, data => #{<<"id">> => <<"6">>}} + ], + Requests + ) + after 1000 -> ?assert(false) + end + end). + +dispatch_many_rejects_malformed_payloads_without_casting_test() -> + with_guild_pid(fun() -> + Malformed = [ + #{ + <<"guild_id">> => <<"1">>, + <<"events">> => [#{<<"event">> => <<"THREAD_UPDATE">>}] + }, + #{ + <<"guild_id">> => <<"1">>, + <<"events">> => [ + #{<<"event">> => <<"THREAD_UPDATE">>, <<"data">> => #{}}, + #{<<"event">> => 7, <<"data">> => #{}} + ] + }, + #{<<"guild_id">> => <<"1">>, <<"events">> => #{}}, + #{<<"guild_id">> => <<"1">>} + ], + lists:foreach( + fun(Params) -> + ?assertError( + {gateway_rpc_error, validation_invalid_params}, + gateway_rpc_guild_lifecycle:handle(<<"guild.dispatch_many">>, Params) + ) + end, + Malformed + ), + ?assertError( + {validation, validation_invalid_snowflake}, + gateway_rpc_guild_lifecycle:handle( + <<"guild.dispatch_many">>, #{<<"guild_id">> => <<"0">>, <<"events">> => []} + ) + ), + receive + {'$gen_cast', {dispatch_many, _}} -> ?assert(false) + after 50 -> ok + end + end). + +dispatch_many_reaches_an_overloaded_guild_test() -> + GuildId = 987654321, + guild_manager_cache:ensure_guild_pid_cache(), + ok = guild_ets_owner:ensure_table(guild_health_status, [named_table, public, set]), + Pending = {make_ref(), erlang:monotonic_time(millisecond), 2500}, + true = ets:insert(guild_pid_cache, {GuildId, self()}), + true = ets:insert(guild_health_status, {self(), GuildId, true, undefined, Pending}), + try + ?assert(guild_health:is_overloaded(self())), + Params = #{ + <<"guild_id">> => integer_to_binary(GuildId), + <<"events">> => [ + #{<<"event">> => <<"THREAD_CREATE">>, <<"data">> => #{<<"id">> => <<"5">>}}, + #{ + <<"event">> => <<"THREAD_MEMBER_UPDATE">>, + <<"data">> => #{<<"id">> => <<"5">>} + } + ] + }, + ?assertEqual( + true, gateway_rpc_guild_lifecycle:handle(<<"guild.dispatch_many">>, Params) + ), + receive + {'$gen_cast', {dispatch_many, Requests}} -> ?assertEqual(2, length(Requests)) + after 1000 -> ?assert(false) + end + after + ets:delete(guild_health_status, self()), + ets:delete(guild_pid_cache, GuildId) + end. diff --git a/fluxer_gateway/test/guild_availability_tests.erl b/fluxer_gateway/test/guild_availability_tests.erl index f8bb223f7..2c08355d5 100644 --- a/fluxer_gateway/test/guild_availability_tests.erl +++ b/fluxer_gateway/test/guild_availability_tests.erl @@ -276,3 +276,36 @@ state_for_unavailability_transition_test(GuildId, NonStaffPid, StaffPid) -> }. -endif. + +guild_create_resend_carries_threads_only_for_viewers_test() -> + Tab = guild_thread_store:new(), + try + Data = guild_data_index:normalize_map(#{ + <<"guild">> => #{<<"id">> => <<"77">>, <<"owner_id">> => <<"1">>}, + <<"roles">> => [#{<<"id">> => <<"77">>, <<"permissions">> => <<"1024">>}], + <<"channels">> => [#{<<"id">> => <<"5">>, <<"type">> => 0}], + <<"members">> => [#{<<"user">> => #{<<"id">> => <<"9">>}, <<"roles">> => []}], + <<"emojis">> => [], + <<"stickers">> => [] + }), + State = #{ + id => 77, + data => Data#{thread_store => Tab, thread_gate => #{active => true, version => 1}} + }, + Viewer = #{user_id => 9, pid => self(), thread_viewer => true}, + ok = guild_availability:send_guild_create_to_session(Viewer, 77, State, []), + ?assert(maps:is_key(<<"threads">>, received_guild_create())), + ok = guild_availability:send_guild_create_to_session( + Viewer#{thread_viewer => false}, 77, State, [] + ), + ?assertNot(maps:is_key(<<"threads">>, received_guild_create())) + after + guild_thread_store:destroy(Tab) + end. + +received_guild_create() -> + receive + {'$gen_cast', {dispatch, guild_create, GuildState}} when is_map(GuildState) -> + GuildState + after 1000 -> erlang:error(guild_create_not_sent) + end. diff --git a/fluxer_gateway/test/guild_connect_async_tests.erl b/fluxer_gateway/test/guild_connect_async_tests.erl index 9df576bce..acb5fb401 100644 --- a/fluxer_gateway/test/guild_connect_async_tests.erl +++ b/fluxer_gateway/test/guild_connect_async_tests.erl @@ -16,6 +16,38 @@ finalize_pending_session_tracks_connected_user_test() -> ?assertEqual(2, maps:get(UserId, maps:get(presence_subscriptions, State1))), flush_connect_result(). +finalize_records_the_thread_viewer_flag_test() -> + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, included_users => #{<<"10">> => true} + }), + try + SessionId = <<"s-thread">>, + Attempt = 2, + State0 = finalize_state(SessionId, 10, Attempt, true, #{}, sets:new(), #{}), + State1 = State0#{data => #{thread_gate => #{active => true, version => 1}}}, + Computed0 = finalize_computed(SessionId, 10), + Request = (maps:get(request, Computed0))#{thread_channels_capable => true}, + Computed = Computed0#{request => Request, thread_viewer => true}, + State2 = guild_connect_async:finalize_session_connect_async( + SessionId, Attempt, {ok, #{}}, Computed, State1 + ), + Session = maps:get(SessionId, maps:get(sessions, State2)), + ?assertEqual(true, maps:get(thread_viewer, Session)), + ?assertEqual(true, maps:get(thread_capable, Session)), + receive + {thread_flip_resend, SessionId} -> ?assert(false) + after 0 -> ok + end, + flush_connect_result() + after + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + finalize_connected_session_does_not_double_count_test() -> SessionId = <<"s1">>, UserId = 10, diff --git a/fluxer_gateway/test/guild_data_tests.erl b/fluxer_gateway/test/guild_data_tests.erl index 4392b4fb1..a07bd5352 100644 --- a/fluxer_gateway/test/guild_data_tests.erl +++ b/fluxer_gateway/test/guild_data_tests.erl @@ -31,6 +31,31 @@ read_model_preserves_query_results_test() -> cleanup_read_model(State) end. +read_model_resolves_thread_channels_test() -> + State0 = read_model_state(), + Store = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Store, #{ + <<"id">> => <<"600">>, <<"parent_id">> => <<"500">>, <<"type">> => 11 + }), + #{data := Data} = State0, + State = State0#{ + data => Data#{thread_gate => #{active => true, version => 1}, thread_store => Store} + }, + Request = #{user_id => 200, channel_id => 600}, + try + ok = guild_read_model:put_state(State), + {reply, Expected, _} = guild_data:get_auth_context(Request, State), + ?assertMatch( + #{auth_context := #{<<"parent_channel">> := #{<<"id">> := 500}}}, Expected + ), + ?assertEqual( + {ok, Expected}, guild_read_model:query(100, {get_guild_auth_context, Request}) + ) + after + cleanup_read_model(State), + guild_thread_store:destroy(Store) + end. + read_model_uses_live_member_rows_without_copying_members_test() -> State = read_model_state(), #{data := #{members_ets := Tab}} = State, diff --git a/fluxer_gateway/test/guild_handoff_freeze_tests.erl b/fluxer_gateway/test/guild_handoff_freeze_tests.erl index 267579832..65c914b71 100644 --- a/fluxer_gateway/test/guild_handoff_freeze_tests.erl +++ b/fluxer_gateway/test/guild_handoff_freeze_tests.erl @@ -26,7 +26,9 @@ transfer_test_() -> instantiate(fun abort_after_forwarding_reports_the_forwarded_count/1), instantiate(fun late_source_stop_keeps_the_committed_target/1), instantiate(fun controller_death_after_commit_keeps_the_target/1), - instantiate(fun lost_guard_aborts_before_other_nodes_route_to_the_target/1) + instantiate(fun lost_guard_aborts_before_other_nodes_route_to_the_target/1), + instantiate(fun thread_state_survives_the_frozen_export/1), + instantiate(fun stale_thread_sessions_do_not_break_a_sessionless_restore/1) ]}. instantiate(Test) -> @@ -242,6 +244,47 @@ lost_guard_aborts_before_other_nodes_route_to_the_target(#{ ?assertEqual(false, guild_handoff_freeze:is_frozen(Src)), ?assertEqual([a], gen_server:call(Src, get_log, 5000)). +thread_state_survives_the_frozen_export(#{src := Src, src_shard := SrcShard}) -> + ok = gen_server:call(Src, {put_session, <<"s1">>, #{pid => self(), user_id => 7}}), + ok = gen_server:call(Src, install_threads), + Result = guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), #{}), + ?assertMatch({ok, #{new_pid := _}}, Result), + {ok, #{new_pid := NewPid}} = Result, + ?assertEqual(ok, wait_dead(Src)), + State = guild_init:init_base_state(gen_server:call(NewPid, get_export)), + try + ?assertMatch(#{<<"id">> := 10}, guild_thread_gate:thread(10, State)), + ?assert(guild_thread_gate:active(State)), + ?assertEqual([<<"s1">>], maps:keys(maps:get(thread_resend_pending, State))), + ?assertEqual([<<"s1">>], maps:keys(maps:get(thread_sync_pending, State))), + ?assertEqual([10], maps:keys(maps:get(thread_list_dirty, State))), + ?assertEqual( + thread_member_list_flush, receive_tagged(thread_member_list_flush, 2000) + ) + after + guild_thread_store:destroy(guild_thread_gate:store(State)) + end. + +stale_thread_sessions_do_not_break_a_sessionless_restore(#{src := Src, src_shard := SrcShard}) -> + ok = gen_server:call(Src, {put_session, <<"s1">>, #{pid => self(), user_id => 7}}), + ok = gen_server:call(Src, install_threads), + Opts = #{transfer_sessions => false}, + {ok, #{new_pid := NewPid}} = + guild_handoff_freeze:transfer(?GUILD_ID, Src, SrcShard, node(), Opts), + Export = gen_server:call(NewPid, get_export), + ?assertEqual(#{}, maps:get(sessions, Export)), + ?assertEqual([<<"s1">>], maps:get(thread_handoff_resends, Export)), + State = guild_init:init_base_state(Export), + try + ?assertMatch(#{<<"id">> := 10}, guild_thread_gate:thread(10, State)), + Resent = guild_thread_flip:resend(<<"s1">>, State), + ?assertNot(maps:is_key(thread_resend_pending, Resent)), + Synced = guild_thread_subscriptions:handle_tick(<<"s1">>, Resent), + ?assertNot(maps:is_key(thread_sync_pending, Synced)) + after + guild_thread_store:destroy(guild_thread_gate:store(State)) + end. + hold(Pid, Ms) -> Test = self(), spawn(fun() -> @@ -373,6 +416,18 @@ handle_call(flush_down_on_terminate, _From, #{role := guild} = State) -> {reply, ok, State#{flush_down => {Helper, erlang:monitor(process, Helper)}}}; handle_call(cast_late_on_terminate, _From, #{role := guild} = State) -> {reply, ok, State#{cast_late => true}}; +handle_call(install_threads, _From, #{role := guild, data := Data} = State) -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + {reply, ok, State#{ + data => Data#{thread_store => Tab, thread_gate => #{active => true, version => 1}}, + thread_load => #{status => ready, queue => [], queued => 0, attempt => 0}, + thread_resend_pending => #{<<"s1">> => true}, + thread_sync_pending => #{<<"s1">> => scheduled}, + thread_list_dirty => #{10 => true} + }}; +handle_call(get_export, _From, #{role := guild} = State) -> + {reply, maps:remove(role, State), State}; handle_call(get_sessions, _From, #{role := guild, sessions := Sessions} = State) -> {reply, Sessions, State}; handle_call(get_log, _From, #{role := guild} = State) -> diff --git a/fluxer_gateway/test/guild_handoff_tests.erl b/fluxer_gateway/test/guild_handoff_tests.erl index 4820bddec..fc2a74c2d 100644 --- a/fluxer_gateway/test/guild_handoff_tests.erl +++ b/fluxer_gateway/test/guild_handoff_tests.erl @@ -126,3 +126,59 @@ session_loop() -> after infinity -> ok end. + +export_handoff_state_replaces_the_thread_store_with_rows_test() -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + try + State = #{ + id => 12345, + data => #{thread_store => Tab, thread_gate => #{active => true, version => 1}}, + sessions => #{} + }, + Data = maps:get(data, guild_handoff:export_handoff_state(State)), + ?assertNot(maps:is_key(thread_store, Data)), + ?assertEqual(#{active => true, version => 1}, maps:get(thread_gate, Data)), + ?assertEqual(guild_thread_store:export(Tab), maps:get(thread_rows, Data)), + ?assert(is_integer(maps:get(thread_rows_at, Data))) + after + guild_thread_store:destroy(Tab) + end. + +init_from_export_queues_a_flip_when_the_gate_disagrees_with_config_test() -> + Tab = guild_thread_store:new(), + ok = guild_thread_store:put_thread(Tab, #{<<"id">> => 10, <<"parent_id">> => 1}), + Key = channel_threads_config, + Previous = persistent_term:get(Key, undefined), + try + Exported = guild_handoff:export_handoff_state(#{ + id => 12345, + data => #{thread_store => Tab, thread_gate => #{active => true, version => 4}}, + sessions => #{} + }), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, config_version => 5 + }), + _ = guild_init:init_base_state(Exported), + ?assertEqual([5], drain_flips([])), + persistent_term:put(Key, (channel_threads_config:default_config())#{ + enabled => true, config_version => 5, enabled_guilds => #{<<"12345">> => true} + }), + _ = guild_init:init_base_state(Exported), + ?assertEqual([], drain_flips([])), + persistent_term:erase(Key), + _ = guild_init:init_base_state(Exported), + ?assertEqual([], drain_flips([])) + after + guild_thread_store:destroy(Tab), + case Previous of + undefined -> persistent_term:erase(Key); + _ -> persistent_term:put(Key, Previous) + end + end. + +drain_flips(Acc) -> + receive + {thread_gate_flip, Version} -> drain_flips([Version | Acc]) + after 0 -> lists:reverse(Acc) + end. diff --git a/fluxer_gateway/test/guild_manager_shard_lifecycle_tests.erl b/fluxer_gateway/test/guild_manager_shard_lifecycle_tests.erl index 45d9def22..5d3152d91 100644 --- a/fluxer_gateway/test/guild_manager_shard_lifecycle_tests.erl +++ b/fluxer_gateway/test/guild_manager_shard_lifecycle_tests.erl @@ -47,6 +47,10 @@ normalize_transferred_guild_state_keeps_only_transferable_fields_test() -> sessions => #{<<"s1">> => #{pid => self()}}, voice_states => #{<<"v1">> => #{}}, virtual_channel_access => #{10 => sets:from_list([20])}, + thread_handoff_resends => [<<"s1">>], + thread_handoff_syncs => [<<"s2">>], + thread_handoff_lists => [30], + thread_resend_pending => #{<<"s1">> => true}, member_list_engine => make_ref(), channel_member_list_engines => #{<<"500">> => make_ref()}, voice_server_pid => self() @@ -60,6 +64,10 @@ normalize_transferred_guild_state_keeps_only_transferable_fields_test() -> ?assertEqual(#{<<"v1">> => #{}}, maps:get(voice_states, Normalized)), ?assertNot(maps:is_key(member_list_engine, Normalized)), ?assertNot(maps:is_key(channel_member_list_engines, Normalized)), + ?assertEqual([<<"s1">>], maps:get(thread_handoff_resends, Normalized)), + ?assertEqual([<<"s2">>], maps:get(thread_handoff_syncs, Normalized)), + ?assertEqual([30], maps:get(thread_handoff_lists, Normalized)), + ?assertNot(maps:is_key(thread_resend_pending, Normalized)), ?assertNot(maps:is_key(voice_server_pid, Normalized)). register_and_monitor_race_kills_duplicate_test_() -> diff --git a/fluxer_gateway/test/guild_session_tracking_tests.erl b/fluxer_gateway/test/guild_session_tracking_tests.erl index c8a03eeee..7670eaaf0 100644 --- a/fluxer_gateway/test/guild_session_tracking_tests.erl +++ b/fluxer_gateway/test/guild_session_tracking_tests.erl @@ -205,7 +205,7 @@ fresh_pending_sibling_keeps_mutual_members() -> ?assertEqual( [?USER_C], guild_subscription_mutual_channels:filter_member_ids( - ?USER_A, [?USER_C], State + #{}, ?USER_A, [?USER_C], State ) ) end) diff --git a/fluxer_gateway/test/guild_thread_push_tests.erl b/fluxer_gateway/test/guild_thread_push_tests.erl new file mode 100644 index 000000000..8c4331ce8 --- /dev/null +++ b/fluxer_gateway/test/guild_thread_push_tests.erl @@ -0,0 +1,194 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_thread_push_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(G, 1). +-define(A, 10). +-define(B, 11). +-define(C, 12). +-define(OUT, 13). +-define(AUTHOR, 14). +-define(TEXT, 100). +-define(THREAD, 500). +-define(VIEW, 1024). +-define(SEND, 2048). +-define(CONFIG_KEY, channel_threads_config). + +with_config(Fun) -> + Previous = persistent_term:get(?CONFIG_KEY, undefined), + Config = (channel_threads_config:default_config())#{ + enabled => true, + included_users => + #{integer_to_binary(Id) => true || Id <- [?A, ?B, ?C, ?AUTHOR]} + }, + persistent_term:put(?CONFIG_KEY, Config), + try + Fun() + after + case Previous of + undefined -> persistent_term:erase(?CONFIG_KEY); + _ -> persistent_term:put(?CONFIG_KEY, Previous) + end + end. + +raw_data() -> + Gid = integer_to_binary(?G), + Member = fun(Id) -> + #{ + <<"user">> => #{<<"id">> => integer_to_binary(Id)}, + <<"roles">> => [], + <<"joined_at">> => <<"2026-01-01T00:00:00Z">> + } + end, + #{ + <<"guild">> => #{<<"id">> => Gid, <<"owner_id">> => <<"999">>, <<"features">> => []}, + <<"roles">> => [ + #{ + <<"id">> => Gid, + <<"permissions">> => integer_to_binary(?VIEW bor ?SEND), + <<"position">> => 0 + } + ], + <<"channels">> => [ + #{<<"id">> => <<"100">>, <<"type">> => 0, <<"permission_overwrites">> => []} + ], + <<"members">> => [Member(Id) || Id <- [?A, ?B, ?C, ?OUT, ?AUTHOR]], + <<"emojis">> => [], + <<"stickers">> => [] + }. + +thread_member(UserId) -> + #{ + <<"id">> => integer_to_binary(?THREAD), + <<"user_id">> => integer_to_binary(UserId), + <<"join_timestamp">> => <<"2026-01-01T00:00:00.000Z">>, + <<"flags">> => 1 + }. + +state(Active) -> + Payload = guild_thread_load:load_payload(#{ + <<"threads">> => [ + #{ + <<"id">> => integer_to_binary(?THREAD), + <<"guild_id">> => integer_to_binary(?G), + <<"parent_id">> => integer_to_binary(?TEXT), + <<"type">> => 11, + <<"name">> => <<"ideas">>, + <<"thread_metadata">> => #{<<"archived">> => false, <<"locked">> => false} + } + ], + <<"thread_members">> => [thread_member(?A), thread_member(?OUT)] + }), + Base = #{ + id => ?G, + member_count => 5, + data => guild_data_index:normalize_map(raw_data()), + sessions => #{ + <<"b">> => #{session_id => <<"b">>, user_id => ?B, pid => self()}, + <<"a">> => #{session_id => <<"a">>, user_id => ?A, pid => self()} + } + }, + Gate = + case Active of + true -> + #{ + thread_gate => #{active => true, version => 1}, + thread_tainted => true, + thread_load => {ok, Payload} + }; + false -> + #{} + end, + guild_thread_load:install(Gate, Base). + +with_state(Active, Fun) -> + with_config(fun() -> + State = state(Active), + try + Fun(State) + after + guild_thread_store:destroy(guild_thread_gate:store(State)) + end + end). + +mention_request(ChannelId, Overrides) -> + maps:merge( + #{ + channel_id => ChannelId, + author_id => ?AUTHOR, + mention_everyone => false, + mention_here => false, + role_ids => [], + user_ids => [] + }, + Overrides + ). + +everyone(Request, State) -> + {reply, #{everyone_user_ids := Ids}, _} = + guild_members_mutation:resolve_mention_sources(Request, State), + lists:sort(Ids). + +everyone_in_a_thread_reaches_recipient_active_members_only_test() -> + with_state(true, fun(State) -> + Request = mention_request(?THREAD, #{mention_everyone => true}), + ?assertEqual([?A], everyone(Request, State)), + ?assertEqual( + [?A, ?B, ?C, ?OUT], + everyone(mention_request(?TEXT, #{mention_everyone => true}), State) + ) + end). + +here_in_a_thread_needs_membership_and_a_connection_test() -> + with_state(true, fun(State) -> + Request = mention_request(?THREAD, #{mention_here => true}), + ?assertEqual([?A], everyone(Request, State)), + ?assertEqual([?A, ?B], everyone(mention_request(?TEXT, #{mention_here => true}), State)) + end). + +direct_mentions_in_a_thread_skip_inactive_recipients_test() -> + with_state(true, fun(State) -> + {reply, #{direct_user_ids := Ids}, _} = + guild_members_mutation:resolve_mention_sources( + mention_request(?THREAD, #{user_ids => [?B, ?OUT]}), State + ), + ?assertEqual([?B], lists:sort(Ids)) + end). + +paged_everyone_in_a_thread_lists_members_only_test() -> + with_state(true, fun(State) -> + Request = (mention_request(?THREAD, #{mention_everyone => true}))#{ + limit => 100, cursor => undefined + }, + {reply, #{mentions := Entries}, _} = + guild_members_mutation:resolve_mention_sources_page(Request, State), + ?assertEqual([?A], lists:sort([entry_user_id(E) || E <- Entries])) + end). + +inactive_guild_mentions_are_unchanged_test() -> + with_state(false, fun(State) -> + ?assertEqual(undefined, guild_thread_push:mention_universe(?THREAD, State)), + ?assertEqual( + [?A, ?B, ?C, ?OUT], + everyone(mention_request(?TEXT, #{mention_everyone => true}), State) + ) + end). + +passive_threads_follow_membership_test() -> + with_state(true, fun(State) -> + ?assertMatch( + [#{<<"name">> := <<"ideas">>}], + guild_thread_push:passive_threads(#{thread_viewer => true}, ?A, State) + ), + ?assertEqual( + [], guild_thread_push:passive_threads(#{thread_viewer => true}, ?B, State) + ), + ?assertEqual([], guild_thread_push:passive_threads(#{}, ?A, State)) + end). + +entry_user_id(#{user_id := UserId}) -> UserId; +entry_user_id(#{<<"user_id">> := UserId}) -> snowflake_id:parse_optional(UserId); +entry_user_id(UserId) when is_integer(UserId) -> UserId; +entry_user_id(Other) -> Other. diff --git a/fluxer_gateway/test/guild_threads_tests.erl b/fluxer_gateway/test/guild_threads_tests.erl new file mode 100644 index 000000000..667f0e05b --- /dev/null +++ b/fluxer_gateway/test/guild_threads_tests.erl @@ -0,0 +1,1986 @@ +%% SPDX-License-Identifier: AGPL-3.0-or-later + +-module(guild_threads_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(G, 1). +-define(A, 10). +-define(C, 11). +-define(X, 12). +-define(P, 13). +-define(V, 14). +-define(M, 15). +-define(OWNER, 999). +-define(TEXT, 100). +-define(FORUM, 200). +-define(CATEGORY, 300). +-define(HIDDEN, 400). +-define(T1, 500). +-define(T2, 600). +-define(MOD_ROLE, 50). +-define(VIEW, 1024). +-define(SEND, 2048). +-define(HISTORY, 65536). +-define(IN_THREADS, (1 bsl 38)). +-define(MANAGE_THREADS, (1 bsl 34)). +-define(CONFIG_KEY, channel_threads_config). + +with_config(Fun) -> + Previous = persistent_term:get(?CONFIG_KEY, undefined), + Config = (channel_threads_config:default_config())#{ + enabled => true, + included_users => + #{ + integer_to_binary(Id) => true + || Id <- [?A, ?P, ?V, ?M, ?OWNER] + } + }, + persistent_term:put(?CONFIG_KEY, Config), + try + Fun() + after + case Previous of + undefined -> persistent_term:erase(?CONFIG_KEY); + _ -> persistent_term:put(?CONFIG_KEY, Previous) + end + end. + +raw_data() -> + Gid = integer_to_binary(?G), + Everyone = integer_to_binary(?VIEW bor ?SEND bor ?HISTORY bor ?IN_THREADS), + Member = fun(Id, Roles, Bot) -> + #{ + <<"user">> => #{<<"id">> => integer_to_binary(Id), <<"bot">> => Bot}, + <<"roles">> => Roles, + <<"joined_at">> => <<"2026-01-01T00:00:00Z">> + } + end, + #{ + <<"guild">> => #{ + <<"id">> => Gid, <<"owner_id">> => integer_to_binary(?OWNER), <<"features">> => [] + }, + <<"roles">> => [ + #{<<"id">> => Gid, <<"permissions">> => Everyone, <<"position">> => 0}, + #{ + <<"id">> => integer_to_binary(?MOD_ROLE), + <<"permissions">> => integer_to_binary(?MANAGE_THREADS), + <<"position">> => 1 + } + ], + <<"channels">> => [ + #{<<"id">> => <<"100">>, <<"type">> => 0, <<"permission_overwrites">> => []}, + #{ + <<"id">> => <<"300">>, + <<"type">> => 4, + <<"permission_overwrites">> => [ + #{ + <<"id">> => Gid, + <<"type">> => 0, + <<"allow">> => <<"0">>, + <<"deny">> => <<"1024">> + } + ] + }, + #{ + <<"id">> => <<"200">>, + <<"type">> => 15, + <<"parent_id">> => <<"300">>, + <<"default_thread_rate_limit_per_user">> => 5, + <<"permission_overwrites">> => [ + #{ + <<"id">> => Gid, + <<"type">> => 0, + <<"allow">> => <<"1024">>, + <<"deny">> => <<"0">> + } + ] + }, + #{ + <<"id">> => <<"400">>, + <<"type">> => 0, + <<"permission_overwrites">> => [ + #{ + <<"id">> => integer_to_binary(?P), + <<"type">> => 1, + <<"allow">> => <<"0">>, + <<"deny">> => <<"1024">> + } + ] + } + ], + <<"members">> => [ + Member(?A, [], false), + Member(?C, [], false), + Member(?X, [integer_to_binary(?MOD_ROLE)], true), + Member(?P, [], false), + Member(?V, [], false), + Member(?M, [integer_to_binary(?MOD_ROLE)], false), + Member(?OWNER, [], false) + ], + <<"emojis">> => [], + <<"stickers">> => [] + }. + +thread(Id, Parent, Type) -> + #{ + <<"id">> => integer_to_binary(Id), + <<"guild_id">> => integer_to_binary(?G), + <<"parent_id">> => integer_to_binary(Parent), + <<"owner_id">> => integer_to_binary(?A), + <<"type">> => Type, + <<"name">> => <<"thread">>, + <<"last_message_id">> => null, + <<"rate_limit_per_user">> => 0, + <<"flags">> => 0, + <<"message_count">> => 0, + <<"total_message_sent">> => 0, + <<"member_count">> => 1, + <<"thread_metadata">> => #{ + <<"archived">> => false, + <<"locked">> => false, + <<"auto_archive_duration">> => 4320, + <<"archive_timestamp">> => <<"2026-01-01T00:00:00.000Z">>, + <<"create_timestamp">> => <<"2026-01-01T00:00:00.000Z">> + } + }. + +thread_member(ThreadId, UserId) -> + #{ + <<"id">> => integer_to_binary(ThreadId), + <<"user_id">> => integer_to_binary(UserId), + <<"join_timestamp">> => <<"2026-01-01T00:00:00.000Z">>, + <<"flags">> => 1, + <<"muted">> => false, + <<"mute_config">> => null + }. + +active_state(Pids) -> + Payload = guild_thread_load:load_payload(#{ + <<"threads">> => [thread(?T1, ?TEXT, 11), thread(?T2, ?TEXT, 12)], + <<"thread_members">> => [thread_member(?T1, ?A), thread_member(?T2, ?A)] + }), + Base = #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(raw_data()), + sessions => sessions(Pids), + member_list_subscriptions => guild_member_list_subs:new() + }, + guild_thread_load:install( + #{ + thread_gate => #{active => true, version => 1}, + thread_tainted => true, + thread_load => {ok, Payload} + }, + Base + ). + +inactive_state(Pids) -> + guild_thread_load:install(#{}, #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(raw_data()), + sessions => sessions(Pids), + member_list_subscriptions => guild_member_list_subs:new() + }). + +sessions(Pids) -> + maps:from_list([ + {atom_to_binary(Tag), session(Tag, UserId, Bot, Capable, Pid)} + || {Tag, UserId, Bot, Capable, Pid} <- [ + {a, ?A, false, true, maps:get(a, Pids)}, + {c, ?C, false, true, maps:get(c, Pids)}, + {x, ?X, true, false, maps:get(x, Pids)}, + {p, ?P, false, true, maps:get(p, Pids)}, + {v, ?V, false, true, maps:get(v, Pids)}, + {i, ?A, false, false, maps:get(i, Pids)} + ] + ]). + +session(Tag, UserId, Bot, Capable, Pid) -> + #{ + session_id => atom_to_binary(Tag), + user_id => UserId, + pid => Pid, + bot => Bot, + is_staff => false, + pending_connect => false, + active_guilds => sets:from_list([?G]), + thread_capable => Capable + }. + +start_captures() -> + Parent = self(), + maps:from_list([ + {Tag, spawn(fun() -> capture_loop(Tag, Parent) end)} + || Tag <- [a, c, x, p, v, i] + ]). + +stop_captures(Pids) -> + maps:foreach(fun(_Tag, Pid) -> Pid ! stop end, Pids). + +capture_loop(Tag, Parent) -> + receive + stop -> + ok; + {'$gen_cast', {dispatch, Event, Payload}} -> + Parent ! {captured, Tag, Event, decode(Payload)}, + capture_loop(Tag, Parent); + _Other -> + capture_loop(Tag, Parent) + end. + +decode({pre_encoded, Bin}) -> + json:decode(Bin); +decode(Map) when is_map(Map) -> + json:decode(iolist_to_binary(json:encode(guild_data_wire:payload(Map)))). + +drain() -> + drain([]). + +drain(Acc) -> + receive + {captured, Tag, Event, Payload} -> drain([{Tag, Event, Payload} | Acc]) + after 150 -> + lists:reverse(Acc) + end. + +received(Tag, Event, Events) -> + [Payload || {T, E, Payload} <- Events, T =:= Tag, E =:= Event]. + +with_guild(Fun) -> + with_config(fun() -> + Pids = start_captures(), + State = active_state(Pids), + try + Fun(State) + after + guild_thread_store:destroy(guild_thread_gate:store(State)), + stop_captures(Pids), + settle_mailbox_age() + end + end). + +settle_mailbox_age() -> + receive + {guild_mailbox_age, Seq} -> + ok = guild_mailbox_age:handle_mark(Seq), + settle_mailbox_age() + after 0 -> + ok + end. + +dispatch(Event, Data, State) -> + {noreply, NewState} = guild_dispatch:handle_dispatch(Event, Data, State), + NewState. + +viewer_flags_follow_capability_bot_and_enrolment_test() -> + with_guild(fun(State) -> + Sessions = maps:get(sessions, State), + Viewer = fun(Sid) -> guild_thread_gate:session_viewer(maps:get(Sid, Sessions)) end, + ?assert(Viewer(<<"a">>)), + ?assertNot(Viewer(<<"c">>)), + ?assert(Viewer(<<"x">>)), + ?assertNot(Viewer(<<"i">>)) + end). + +untainted_inactive_guild_create_is_unchanged_test() -> + with_config(fun() -> + Pids = start_captures(), + try + State = inactive_state(Pids), + meck:new(guild_thread_view, [passthrough]), + try + Plain = guild_data:get_guild_state(?A, State), + Opts = guild_data:get_guild_state(?A, State, #{thread_viewer => false}), + ?assertEqual(Plain, Opts), + ?assertNot(maps:is_key(<<"threads">>, Plain)), + ?assertEqual(0, meck:num_calls(guild_thread_view, '_', '_')) + after + meck:unload(guild_thread_view) + end, + ?assertEqual( + #{}, + maps:with([thread_gate, thread_store, thread_tainted], maps:get(data, State)) + ) + after + stop_captures(Pids) + end + end). + +tainted_inactive_reload_masks_role_bits_test() -> + Pids = start_captures(), + try + {_Channels, Extras} = guild_thread_load:channels_collection(#{ + <<"channels">> => [], <<"thread_tainted">> => true + }), + State = guild_thread_load:install(Extras, #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(raw_data()), + sessions => sessions(Pids), + member_list_subscriptions => guild_member_list_subs:new() + }), + ?assertNot(guild_thread_gate:active(State)), + ?assert(guild_thread_gate:needs_variant(State)), + GuildState = guild_data:get_guild_state(?C, State, #{thread_viewer => false}), + [ + ?assertEqual(0, maps:get(<<"permissions">>, R) band ?IN_THREADS) + || R <- maps:get(<<"roles">>, GuildState) + ], + Untainted = guild_data:get_guild_state( + ?C, inactive_state(Pids), #{thread_viewer => false} + ), + ?assert( + lists:any( + fun(R) -> maps:get(<<"permissions">>, R) band ?IN_THREADS =/= 0 end, + maps:get(<<"roles">>, Untainted) + ) + ) + after + stop_captures(Pids) + end. + +viewer_guild_create_carries_joined_threads_test() -> + with_guild(fun(State) -> + GuildState = guild_data:get_guild_state(?A, State, #{ + thread_viewer => true, bot => false + }), + Threads = maps:get(<<"threads">>, GuildState), + ?assertEqual([?T1, ?T2], lists:sort([maps:get(<<"id">>, T) || T <- Threads])), + [T1] = [T || T <- Threads, maps:get(<<"id">>, T) =:= ?T1], + Member = maps:get(<<"member">>, T1), + ?assertNot(maps:is_key(<<"id">>, Member)), + ?assertNot(maps:is_key(<<"user_id">>, Member)), + ?assertNot(maps:is_key(<<"member_ids_preview">>, T1)), + VState = guild_data:get_guild_state(?V, State, #{thread_viewer => true, bot => false}), + ?assertEqual([], maps:get(<<"threads">>, VState)), + XState = guild_data:get_guild_state(?X, State, #{thread_viewer => true, bot => true}), + ?assertEqual( + [?T1, ?T2], + lists:sort([maps:get(<<"id">>, T) || T <- maps:get(<<"threads">>, XState)]) + ) + end). + +control_session_guild_create_hides_forums_and_bits_test() -> + with_guild(fun(State) -> + lists:foreach( + fun(UserId) -> + GuildState = guild_data:get_guild_state(UserId, State, #{thread_viewer => false}), + ?assertNot(maps:is_key(<<"threads">>, GuildState)), + Ids = [maps:get(<<"id">>, C) || C <- maps:get(<<"channels">>, GuildState)], + ?assertNot(lists:member(?FORUM, Ids)), + ?assertEqual(UserId =:= ?OWNER, lists:member(?CATEGORY, Ids)), + [ + ?assertEqual(0, maps:get(<<"permissions">>, R) band ?IN_THREADS) + || R <- maps:get(<<"roles">>, GuildState) + ] + end, + [?C, ?OWNER] + ), + Viewer = guild_data:get_guild_state(?A, State, #{thread_viewer => true}), + ViewerIds = [maps:get(<<"id">>, C) || C <- maps:get(<<"channels">>, Viewer)], + ?assert(lists:member(?FORUM, ViewerIds)), + ?assert(lists:member(?CATEGORY, ViewerIds)) + end). + +permissions_fail_closed_for_unknown_ids_test() -> + with_guild(fun(State) -> + ?assertEqual(0, guild_permissions:get_member_permissions(?A, 424242, State)), + Inactive = State#{data => maps:remove(thread_gate, maps:get(data, State))}, + ?assertNotEqual(0, guild_permissions:get_member_permissions(?A, 424242, Inactive)) + end). + +thread_permissions_derive_from_parent_test() -> + with_guild(fun(State) -> + Perms = guild_permissions:get_member_permissions(?V, ?T1, State), + ?assert(permission_bits:has(Perms, ?VIEW)), + ?assert(permission_bits:has(Perms, ?SEND)), + ?assertNot( + permission_bits:has(guild_permissions:get_member_permissions(?V, ?T2, State), ?VIEW) + ), + ?assert( + permission_bits:has(guild_permissions:get_member_permissions(?A, ?T2, State), ?VIEW) + ), + ?assert( + permission_bits:has(guild_permissions:get_member_permissions(?M, ?T2, State), ?VIEW) + ), + {reply, #{auth_context := Context}, _} = guild_data:get_auth_context( + #{user_id => ?V, channel_id => ?T1}, State + ), + ?assertEqual(?TEXT, maps:get(<<"id">>, maps:get(<<"parent_channel">>, Context))) + end). + +thread_create_recipients_and_shaping_test() -> + with_guild(fun(State) -> + Data = (thread(700, ?TEXT, 11))#{ + <<"newly_created">> => true, + <<"_fluxer_member_ids_preview">> => [integer_to_binary(?A)], + <<"_fluxer_members">> => [thread_member(700, ?A)] + }, + _ = dispatch(thread_create, Data, State), + Events = drain(), + [ForA] = received(a, thread_create, Events), + ?assertEqual(true, maps:get(<<"newly_created">>, ForA)), + ?assertEqual( + integer_to_binary(?A), maps:get(<<"user_id">>, maps:get(<<"member">>, ForA)) + ), + ?assertNot(maps:is_key(<<"_fluxer_members">>, ForA)), + ?assertNot(maps:is_key(<<"member_ids_preview">>, ForA)), + [ForX] = received(x, thread_create, Events), + ?assertNot(maps:is_key(<<"member">>, ForX)), + ?assertEqual([], received(v, thread_create, Events)), + ?assertEqual([], received(c, thread_create, Events)), + ?assertEqual([], received(i, thread_create, Events)) + end). + +private_thread_create_never_reaches_subscribed_non_members_test() -> + with_guild(fun(State0) -> + State = guild_thread_subscriptions:update(<<"v">>, #{threads => true}, State0), + _ = dispatch(thread_create, thread(701, ?TEXT, 12), State), + Events = drain(), + ?assertEqual([], received(v, thread_create, Events)), + ?assertMatch([_], received(x, thread_create, Events)), + _ = dispatch(thread_create, thread(702, ?TEXT, 11), State), + ?assertMatch([_], received(v, thread_create, drain())) + end). + +members_update_synthesizes_create_and_suppresses_creator_duplicate_test() -> + with_guild(fun(State0) -> + Created = (thread(703, ?TEXT, 12))#{<<"_fluxer_members">> => [thread_member(703, ?A)]}, + State1 = dispatch(thread_create, Created, State0), + _ = drain(), + CreatorAdded = #{ + <<"id">> => <<"703">>, + <<"member_count">> => 1, + <<"added_members">> => [thread_member(703, ?A)], + <<"_fluxer_thread">> => thread(703, ?TEXT, 12) + }, + State2 = dispatch(thread_members_update, CreatorAdded, State1), + Events1 = drain(), + ?assertEqual([], received(a, thread_create, Events1)), + ?assertMatch([_], received(a, thread_members_update, Events1)), + VAdded = CreatorAdded#{ + <<"member_count">> => 2, <<"added_members">> => [thread_member(703, ?V)] + }, + _ = dispatch(thread_members_update, VAdded, State2), + Events2 = drain(), + [Create] = received(v, thread_create, Events2), + ?assertNot(maps:is_key(<<"newly_created">>, Create)), + ?assertEqual( + integer_to_binary(?V), maps:get(<<"user_id">>, maps:get(<<"member">>, Create)) + ), + [Update] = received(v, thread_members_update, Events2), + [Added] = maps:get(<<"added_members">>, Update), + ?assertEqual( + integer_to_binary(?V), + maps:get(<<"id">>, maps:get(<<"user">>, maps:get(<<"member">>, Added))) + ), + ?assert(maps:is_key(<<"presence">>, Added)), + ?assertNot(maps:is_key(<<"muted">>, Added)), + ?assertNot(maps:is_key(<<"mute_config">>, Added)), + ?assertNot(maps:is_key(<<"_fluxer_thread">>, Update)), + ?assertMatch([_], received(a, thread_members_update, Events2)), + ?assertEqual([], received(i, thread_members_update, Events2)), + ?assertMatch([_], received(x, thread_members_update, Events2)), + ?assertEqual([], received(p, thread_members_update, Events2)), + ?assertEqual( + thread_create, + element(2, hd([E || {v, _, _} = E <- Events2])) + ) + end). + +removed_user_without_view_still_gets_members_update_test() -> + with_guild(fun(State) -> + Removal = #{ + <<"id">> => integer_to_binary(?T2), + <<"member_count">> => 1, + <<"removed_member_ids">> => [integer_to_binary(?P)] + }, + _ = dispatch(thread_members_update, Removal, State), + [Update] = received(p, thread_members_update, drain()), + ?assertEqual([integer_to_binary(?P)], maps:get(<<"removed_member_ids">>, Update)) + end). + +thread_delete_payload_is_exact_test() -> + with_guild(fun(State) -> + Data = #{ + <<"id">> => integer_to_binary(?T1), + <<"parent_id">> => integer_to_binary(?TEXT), + <<"type">> => 11 + }, + State1 = dispatch(thread_delete, Data, State), + Events = drain(), + [ForA] = received(a, thread_delete, Events), + ?assertEqual( + lists:sort([<<"guild_id">>, <<"id">>, <<"parent_id">>, <<"type">>]), + lists:sort(maps:keys(ForA)) + ), + ?assertMatch([_], received(x, thread_delete, Events)), + ?assertEqual([], received(c, thread_delete, Events)), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, State1)) + end). + +archived_private_thread_delete_reaches_listed_members_test() -> + with_guild(fun(State0) -> + Archived = (thread(?T2, ?TEXT, 12))#{ + <<"thread_metadata">> => #{<<"archived">> => true, <<"locked">> => false} + }, + State1 = dispatch(thread_update, Archived, State0), + drain(), + ?assertEqual(undefined, guild_thread_gate:thread(?T2, State1)), + Data = #{ + <<"id">> => integer_to_binary(?T2), + <<"parent_id">> => integer_to_binary(?TEXT), + <<"type">> => 12 + }, + _ = dispatch(thread_delete, Data, State1), + ?assertEqual([], received(a, thread_delete, drain())), + _ = dispatch( + thread_delete, Data#{<<"_fluxer_member_ids">> => [integer_to_binary(?A)]}, State1 + ), + Events = drain(), + [ForA] = received(a, thread_delete, Events), + ?assertNot(maps:is_key(<<"_fluxer_member_ids">>, ForA)), + ?assertEqual([], received(v, thread_delete, Events)) + end). + +unknown_thread_uses_internal_context_test() -> + with_guild(fun(State) -> + Data = #{ + <<"id">> => <<"800">>, + <<"member_count">> => 1, + <<"added_members">> => [thread_member(800, ?V)], + <<"_fluxer_thread">> => thread(800, ?TEXT, 11) + }, + State1 = dispatch(thread_members_update, Data, State), + ?assertNotEqual(undefined, guild_thread_gate:thread(800, State1)), + Events = drain(), + ?assertMatch([_], received(v, thread_create, Events)), + ?assertMatch([_], received(v, thread_members_update, Events)) + end). + +archive_removes_thread_and_unarchive_orders_update_before_member_update_test() -> + with_guild(fun(State0) -> + Archived = (thread(?T1, ?TEXT, 11))#{ + <<"thread_metadata">> => #{<<"archived">> => true, <<"locked">> => false} + }, + State1 = dispatch(thread_update, Archived, State0), + ?assertMatch([_], received(a, thread_update, drain())), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, State1)), + Unarchived = (thread(?T1, ?TEXT, 11))#{ + <<"_fluxer_members">> => [thread_member(?T1, ?A)] + }, + {noreply, State2} = guild:handle_cast( + {dispatch_many, [#{event => thread_update, data => Unarchived}]}, State1 + ), + Drained = drain(), + ?assertEqual([thread_update, thread_member_update], [E || {a, E, _} <- Drained]), + [Member] = received(a, thread_member_update, Drained), + ?assertEqual(integer_to_binary(?G), maps:get(<<"guild_id">>, Member)), + ?assertEqual(integer_to_binary(?A), maps:get(<<"user_id">>, Member)), + ?assertEqual([], received(v, thread_member_update, Drained)), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, State2)) + end). + +message_create_in_thread_reaches_members_and_bots_only_test() -> + with_guild(fun(State) -> + Message = #{ + <<"id">> => <<"900">>, + <<"channel_id">> => integer_to_binary(?T1), + <<"type">> => 0, + <<"content">> => <<"hi">>, + <<"author">> => #{<<"id">> => integer_to_binary(?X)}, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + State1 = dispatch(message_create, Message, State), + Events = drain(), + ?assertMatch([_], received(a, message_create, Events)), + ?assertMatch([_], received(x, message_create, Events)), + ?assertEqual([], received(v, message_create, Events)), + ?assertEqual([], received(c, message_create, Events)), + ?assertEqual([], received(i, message_create, Events)), + Thread = guild_thread_gate:thread(?T1, State1), + ?assertEqual(1, maps:get(<<"message_count">>, Thread)) + end). + +type_18_and_thread_only_updates_are_hidden_from_control_test() -> + with_guild(fun(State) -> + Created = #{ + <<"id">> => <<"901">>, + <<"channel_id">> => integer_to_binary(?TEXT), + <<"type">> => 18, + <<"author">> => #{<<"id">> => integer_to_binary(?A)}, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + _ = dispatch(message_create, Created, State), + Events1 = drain(), + ?assertMatch([_], received(a, message_create, Events1)), + ?assertEqual([], received(c, message_create, Events1)), + Update = #{ + <<"id">> => <<"902">>, + <<"channel_id">> => integer_to_binary(?TEXT), + <<"flags">> => 16#20, + <<"thread">> => #{<<"id">> => <<"903">>}, + <<"__thread_only_update">> => true + }, + _ = dispatch(message_update, Update, State), + Events2 = drain(), + [ForA] = received(a, message_update, Events2), + ?assertNot(maps:is_key(<<"__thread_only_update">>, ForA)), + ?assertEqual([], received(c, message_update, Events2)), + Reply = #{ + <<"id">> => <<"904">>, + <<"channel_id">> => integer_to_binary(?TEXT), + <<"flags">> => 16#20 bor 4, + <<"thread">> => #{<<"id">> => <<"905">>}, + <<"referenced_message">> => #{<<"type">> => 18, <<"id">> => <<"906">>} + }, + _ = dispatch(message_update, Reply, State), + Events3 = drain(), + [Masked] = received(c, message_update, Events3), + ?assertEqual(4, maps:get(<<"flags">>, Masked)), + ?assertNot(maps:is_key(<<"thread">>, Masked)), + ?assertEqual(null, maps:get(<<"referenced_message">>, Masked)), + [Full] = received(a, message_update, Events3), + ?assertEqual(16#24, maps:get(<<"flags">>, Full)) + end). + +forum_channel_events_reach_viewers_only_test() -> + with_guild(fun(State) -> + Forum = #{ + <<"id">> => <<"210">>, + <<"type">> => 15, + <<"permission_overwrites">> => [] + }, + _ = dispatch(channel_create, Forum, State), + Events = drain(), + ?assertMatch([_], received(a, channel_create, Events)), + ?assertEqual([], received(c, channel_create, Events)), + ?assertEqual([], received(i, channel_create, Events)) + end). + +inactive_guild_drops_gated_events_test() -> + with_config(fun() -> + Pids = start_captures(), + try + State = inactive_state(Pids), + _ = dispatch(thread_create, thread(700, ?TEXT, 11), State), + _ = dispatch(channel_create, #{<<"id">> => <<"210">>, <<"type">> => 15}, State), + ?assertEqual([], drain()) + after + stop_captures(Pids) + end + end). + +live_audit_entries_on_thread_targets_are_viewer_only_test() -> + with_guild(fun(State0) -> + Sessions = maps:get(sessions, State0), + Passive = maps:map(fun(_Sid, S) -> S#{active_guilds => sets:new()} end, Sessions), + Owner = maps:get(<<"a">>, Passive), + State = State0#{ + member_count => 300, + sessions => Passive#{ + <<"c">> => (maps:get(<<"c">>, Passive))#{user_id => ?OWNER}, + <<"a">> => Owner#{user_id => ?OWNER, thread_viewer => true} + } + }, + Entry = #{ + <<"id">> => <<"1000">>, + <<"action_type">> => 110, + <<"target_id">> => integer_to_binary(?T1), + <<"__thread_scoped">> => integer_to_binary(?G) + }, + _ = dispatch(guild_audit_log_entry_create, Entry, State), + Events = drain(), + [ForA] = received(a, guild_audit_log_entry_create, Events), + ?assertNot(maps:is_key(<<"__thread_scoped">>, ForA)), + ?assertEqual([], received(c, guild_audit_log_entry_create, Events)) + end). + +passive_sessions_get_self_directed_thread_events_test() -> + with_guild(fun(State0) -> + Sessions = maps:get(sessions, State0), + Passive = maps:map(fun(_Sid, S) -> S#{active_guilds => sets:new()} end, Sessions), + State1 = guild_thread_subscriptions:update( + <<"v">>, #{threads => true}, State0#{member_count => 300, sessions => Passive} + ), + _ = dispatch(thread_member_update, (thread_member(?T1, ?A))#{<<"flags">> => 3}, State1), + ?assertMatch([_], received(a, thread_member_update, drain())), + Created = (thread(704, ?TEXT, 11))#{<<"_fluxer_members">> => [thread_member(704, ?A)]}, + State2 = dispatch(thread_create, Created, State1), + Events = drain(), + ?assertMatch([_], received(a, thread_create, Events)), + ?assertEqual([], received(v, thread_create, Events)), + _ = dispatch( + thread_members_update, + #{ + <<"id">> => <<"704">>, + <<"member_count">> => 2, + <<"added_members">> => [thread_member(704, ?V)], + <<"_fluxer_thread">> => thread(704, ?TEXT, 11) + }, + State2 + ), + Events2 = drain(), + ?assertMatch([_], received(v, thread_members_update, Events2)), + ?assertEqual([], received(a, thread_members_update, Events2)), + Unarchive = (thread(?T1, ?TEXT, 11))#{<<"name">> => <<"renamed">>}, + _ = dispatch(thread_update, Unarchive, State2), + Events3 = drain(), + ?assertMatch([_], received(a, thread_update, Events3)), + ?assertEqual([], received(v, thread_update, Events3)) + end). + +permission_loss_keeps_membership_and_sends_no_delete_test() -> + with_guild(fun(State) -> + Gid = integer_to_binary(?G), + Update = #{ + <<"id">> => integer_to_binary(?TEXT), + <<"type">> => 0, + <<"permission_overwrites">> => [ + #{ + <<"id">> => integer_to_binary(?A), + <<"type">> => 1, + <<"allow">> => <<"0">>, + <<"deny">> => <<"1024">> + }, + #{ + <<"id">> => Gid, + <<"type">> => 0, + <<"allow">> => <<"0">>, + <<"deny">> => <<"0">> + } + ] + }, + State1 = dispatch(channel_update, Update, State), + Events = drain(), + ?assertEqual([], received(a, thread_delete, Events)), + Tab = guild_thread_gate:store(State1), + ?assert(guild_thread_store:is_member(Tab, ?T1, ?A)), + Message = #{ + <<"id">> => <<"910">>, + <<"channel_id">> => integer_to_binary(?T1), + <<"type">> => 0, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + _ = dispatch(message_create, Message, State1), + ?assertEqual([], received(a, message_create, drain())) + end). + +op14_threads_subscription_is_viewer_only_and_syncs_test() -> + with_guild(fun(State0) -> + State1 = guild_thread_subscriptions:update(<<"c">>, #{threads => true}, State0), + ?assertNot( + maps:is_key(thread_subscribed, maps:get(<<"c">>, maps:get(sessions, State1))) + ), + State2 = guild_thread_subscriptions:update(<<"v">>, #{threads => true}, State1), + _ = guild_thread_subscriptions:handle_tick(<<"v">>, State2), + [Sync] = received(v, thread_list_sync, drain()), + ?assertNot(maps:is_key(<<"channel_ids">>, Sync)), + ?assertEqual([integer_to_binary(?T1)], [ + maps:get(<<"id">>, T) + || T <- maps:get(<<"threads">>, Sync) + ]), + ?assertEqual([], maps:get(<<"members">>, Sync)), + State3 = guild_thread_subscriptions:update( + <<"a">>, #{member_lists => [?T1, ?T2, 424242]}, State2 + ), + Lists = received(a, thread_member_list_update, drain()), + ?assertEqual(2, length(Lists)), + [First | _] = Lists, + [Entry] = maps:get(<<"members">>, First), + ?assertEqual(integer_to_binary(?A), maps:get(<<"user_id">>, Entry)), + ?assert(maps:is_key(<<"presence">>, Entry)), + Passive = guild_sessions_passive:set_session_passive_guild(<<"v">>, ?G, State3), + ?assertNot( + maps:is_key(thread_subscribed, maps:get(<<"v">>, maps:get(sessions, Passive))) + ) + end). + +access_gain_sends_list_sync_with_channel_ids_test() -> + with_guild(fun(State0) -> + State1 = dispatch(thread_create, thread(705, ?HIDDEN, 11), State0), + Events0 = drain(), + ?assertEqual([], received(p, thread_create, Events0)), + Lifted = #{ + <<"id">> => integer_to_binary(?HIDDEN), + <<"type">> => 0, + <<"permission_overwrites">> => [] + }, + _ = dispatch(channel_update, Lifted, State1), + [Sync] = received(p, thread_list_sync, drain()), + ?assertEqual([integer_to_binary(?HIDDEN)], maps:get(<<"channel_ids">>, Sync)), + ?assertEqual([<<"705">>], [maps:get(<<"id">>, T) || T <- maps:get(<<"threads">>, Sync)]) + end). + +op16_from_control_session_ignores_forums_and_threads_test() -> + with_guild(fun(State) -> + Request = #{session_id => <<"c">>, user_id => ?C, channel_ids => [?FORUM, ?T1]}, + {reply, #{counts := Counts}, _} = guild_query_handler:handle_call( + {get_channel_member_counts, Request}, {self(), make_ref()}, State + ), + ?assertEqual([], Counts) + end). + +public_online_count_ignores_forums_test() -> + with_guild(fun(State) -> + Data = maps:get(data, State), + WithoutForums = guild_data_index:put_channels( + [ + C + || C <- guild_data_index:channel_list(Data), + maps:get(<<"type">>, C) =/= 15 + ], + Data + ), + ?assertEqual( + guild_public_online:compute_count(State#{data => WithoutForums}), + guild_public_online:compute_count(State) + ) + end). + +flip_deactivation_resends_only_changed_sessions_test_() -> + {timeout, 30, fun flip_deactivation_resends_only_changed_sessions/0}. + +flip_deactivation_resends_only_changed_sessions() -> + with_guild(fun(State) -> + Flip = #{ + <<"thread_gate">> => #{<<"active">> => false, <<"config_version">> => 2}, + <<"thread_tainted">> => true, + <<"channels">> => [ + C + || C <- maps:get(<<"channels">>, raw_data()), maps:get(<<"type">>, C) =/= 15 + ], + <<"roles">> => maps:get(<<"roles">>, raw_data()) + }, + Off = guild_thread_flip:apply_flip(Flip, State), + ?assertNot(guild_thread_gate:active(Off)), + ?assertEqual(undefined, guild_thread_gate:store(Off)), + Sessions = maps:get(sessions, Off), + ?assertNot(lists:any(fun guild_thread_gate:session_viewer/1, maps:values(Sessions))), + Resends = collect_resends([]), + ?assertEqual([<<"a">>, <<"p">>, <<"v">>, <<"x">>], lists:sort(Resends)), + _ = guild_thread_flip:resend(<<"a">>, Off), + Events = drain(), + [GuildCreate] = received(a, guild_create, Events), + ?assertNot(maps:is_key(<<"threads">>, GuildCreate)), + ?assertEqual([], received(a, guild_update, Events)) + end). + +collect_resends(Acc) -> + receive + {thread_flip_resend, SessionId} -> collect_resends([SessionId | Acc]) + after 11000 -> + Acc + end. + +flip_activation_marks_viewers_and_loads_threads_test() -> + with_config(fun() -> + Pids = start_captures(), + try + State = inactive_state(Pids), + Flip = #{ + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 3}, + <<"thread_tainted">> => true, + <<"channels">> => maps:get(<<"channels">>, raw_data()), + <<"roles">> => maps:get(<<"roles">>, raw_data()), + <<"threads">> => [thread(?T1, ?TEXT, 11)], + <<"thread_members">> => [thread_member(?T1, ?A)] + }, + On = guild_thread_flip:apply_flip(Flip, State), + ?assert(guild_thread_gate:active(On)), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, On)), + ?assert( + guild_thread_gate:session_viewer(maps:get(<<"a">>, maps:get(sessions, On))) + ), + ?assertNot( + guild_thread_gate:session_viewer(maps:get(<<"c">>, maps:get(sessions, On))) + ), + guild_thread_store:destroy(guild_thread_gate:store(On)) + after + stop_captures(Pids) + end + end). + +flip_result_keeps_live_channels_and_replays_held_events_test() -> + with_config(fun() -> + Pids = start_captures(), + try + On = run_flip_with_held_events(3, Pids), + ?assert(guild_thread_gate:active(On)), + ?assertNot(maps:is_key(thread_flip_held, On)), + ?assertNot(maps:is_key(thread_flip_retry, On)), + Index = guild_data_index:channel_index(maps:get(data, On)), + ?assertMatch(#{<<"type">> := 15}, maps:get(?FORUM, Index)), + ?assertEqual([], maps:get(<<"permission_overwrites">>, maps:get(?HIDDEN, Index))), + ?assertEqual(2, length(guild_data_index:role_list(maps:get(data, On)))), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, On)), + guild_thread_store:destroy(guild_thread_gate:store(On)) + after + stop_captures(Pids) + end + end). + +flip_result_from_a_stale_api_rearms_the_flip_test() -> + with_config(fun() -> + Pids = start_captures(), + try + On = run_flip_with_held_events(2, Pids), + ?assert(guild_thread_gate:active(On)), + #{thread_flip_retry := {Timer, 1}} = On, + ?assert(erlang:read_timer(Timer) > 0), + _ = erlang:cancel_timer(Timer), + guild_thread_store:destroy(guild_thread_gate:store(On)) + after + stop_captures(Pids) + end + end). + +held_event_without_an_id_is_skipped_on_replay_test() -> + with_config(fun() -> + Pids = start_captures(), + try + Raw = raw_data(), + Snapshot = [ + C + || C <- maps:get(<<"channels">>, Raw), maps:get(<<"type">>, C) =/= 15 + ], + Ref = make_ref(), + Held = [ + {thread_create, thread(?T1, ?FORUM, 11)}, + {thread_update, #{<<"guild_id">> => integer_to_binary(?G)}} + ], + State = (inactive_state(Pids))#{ + sessions => #{}, + data => guild_data_index:normalize_map(Raw), + thread_flip => {Ref, 3}, + thread_flip_held => {2, Held} + }, + Flip = #{ + <<"config_version">> => 3, + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 3}, + <<"thread_tainted">> => true, + <<"channels">> => Snapshot, + <<"roles">> => [], + <<"threads">> => [], + <<"thread_members">> => [] + }, + On = guild_thread_flip:handle_result(Ref, {ok, Flip}, State), + ?assert(guild_thread_gate:active(On)), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, On)), + guild_thread_store:destroy(guild_thread_gate:store(On)) + after + stop_captures(Pids) + end + end). + +run_flip_with_held_events(ApiVersion, Pids) -> + Raw = raw_data(), + [Forum] = [C || C <- maps:get(<<"channels">>, Raw), maps:get(<<"type">>, C) =:= 15], + Snapshot = [C || C <- maps:get(<<"channels">>, Raw), maps:get(<<"type">>, C) =/= 15], + Inactive = inactive_state(Pids), + Ref = make_ref(), + State0 = Inactive#{ + sessions => #{}, + data => guild_data_index:normalize_map(Raw#{<<"channels">> => Snapshot}), + thread_flip => {Ref, 3}, + thread_flip_held => {0, []} + }, + Lifted = #{ + <<"id">> => integer_to_binary(?HIDDEN), + <<"type">> => 0, + <<"permission_overwrites">> => [] + }, + State1 = dispatch(channel_create, Forum, State0), + State2 = dispatch(thread_create, thread(?T1, ?FORUM, 11), State1), + State3 = dispatch(channel_update, Lifted, State2), + ?assertMatch(#{thread_flip_held := {2, _}}, State3), + Flip = #{ + <<"config_version">> => ApiVersion, + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => ApiVersion}, + <<"thread_tainted">> => true, + <<"channels">> => Snapshot, + <<"roles">> => [], + <<"threads">> => [], + <<"thread_members">> => [] + }, + guild_thread_flip:handle_result(Ref, {ok, Flip}, State3). + +thread_store_retire_destroys_only_non_current_stores_test() -> + Current = guild_thread_store:new(), + Retired = guild_thread_store:new(), + State = #{data => #{thread_store => Current}}, + ?assertEqual({noreply, State}, guild:handle_info({thread_store_retire, Current}, State)), + ?assertEqual({noreply, State}, guild:handle_info({thread_store_retire, Retired}, State)), + ?assertNotEqual(undefined, ets:info(Current)), + ?assertEqual(undefined, ets:info(Retired)), + guild_thread_store:destroy(Current). + +flip_activation_without_sessions_rebuilds_the_permission_cache_test() -> + with_config(fun() -> + Pids = start_captures(), + try + Raw = raw_data(), + NoForum = Raw#{ + <<"channels">> => [ + C + || C <- maps:get(<<"channels">>, Raw), maps:get(<<"type">>, C) =/= 15 + ] + }, + Inactive = inactive_state(Pids), + State = Inactive#{sessions => #{}, data => guild_data_index:normalize_map(NoForum)}, + ok = guild_permission_cache:put_state(State), + Flip = #{ + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 3}, + <<"thread_tainted">> => true, + <<"channels">> => maps:get(<<"channels">>, raw_data()), + <<"roles">> => maps:get(<<"roles">>, raw_data()), + <<"threads">> => [], + <<"thread_members">> => [] + }, + On = guild_thread_flip:apply_flip(Flip, State), + {ok, #{data := Cached}} = guild_permission_cache:get_snapshot(?G), + ?assertMatch(#{active := true}, maps:get(thread_gate, Cached, undefined)), + ?assert( + lists:any( + fun(C) -> maps:get(<<"type">>, C) =:= 15 end, + guild_data_index:channel_list(Cached) + ) + ), + guild_permission_cache:delete(?G), + guild_thread_store:destroy(guild_thread_gate:store(On)) + after + stop_captures(Pids) + end + end). + +non_viewer_visibility_of_indexed_channels_uses_type_only_test() -> + with_guild(fun(State) -> + [C] = [S || S <- maps:values(maps:get(sessions, State)), maps:get(user_id, S) =:= ?C], + ?assertNot(guild_thread_gate:session_viewer(C)), + Tab = guild_thread_gate:store(State), + Fake = guild_data_normalize:thread(thread(?TEXT, ?TEXT, 11)), + guild_thread_store:put_thread(Tab, Fake), + ?assertNotEqual(undefined, guild_thread_gate:thread(?TEXT, State)), + ?assert(guild_thread_gate:channel_visible(C, ?TEXT, State)), + ?assertNot(guild_thread_gate:channel_visible(C, ?FORUM, State)), + ?assertNot(guild_thread_gate:channel_visible(C, ?T1, State)), + Data = maps:get(data, State), + Index = guild_data_index:channel_index(Data), + Indexed = State#{ + data => Data#{ + <<"channel_index">> => Index#{?T2 => #{<<"id">> => ?T2, <<"type">> => 12}} + } + }, + ?assertNot(guild_thread_gate:channel_visible(C, ?T2, Indexed)) + end). + +forum_only_category_update_skips_non_viewers_test() -> + with_guild(fun(State) -> + [Category] = [ + C + || C <- guild_data_index:channel_list(maps:get(data, State)), + maps:get(<<"type">>, C) =:= 4 + ], + Update = guild_data_wire:payload(Category#{<<"name">> => <<"renamed">>}), + _ = dispatch(channel_update, Update, State), + Events = drain(), + ?assertMatch([_], received(a, channel_update, Events)), + ?assertMatch([_], received(x, channel_update, Events)), + ?assertEqual([], received(c, channel_update, Events)), + ?assertEqual([], received(i, channel_update, Events)) + end). + +visibility_gain_channel_create_is_masked_for_non_viewers_test() -> + with_guild(fun(State) -> + Gid = integer_to_binary(?G), + Everyone = #{ + <<"id">> => Gid, + <<"type">> => 0, + <<"allow">> => integer_to_binary(?IN_THREADS), + <<"deny">> => <<"0">> + }, + DenyC = #{ + <<"id">> => integer_to_binary(?C), + <<"type">> => 1, + <<"allow">> => <<"0">>, + <<"deny">> => integer_to_binary(?VIEW) + }, + WithText = fun(Overwrites) -> + Data = maps:get(data, State), + Channels = [ + case maps:get(<<"id">>, C) of + ?TEXT -> + C#{ + <<"permission_overwrites">> => Overwrites, + <<"default_thread_rate_limit_per_user">> => 7 + }; + _ -> + C + end + || C <- guild_data_index:channel_list(Data) + ], + State#{data => guild_data_index:put_channels(Channels, Data)} + end, + Old = WithText([Everyone, DenyC]), + New = WithText([Everyone]), + _ = guild_visibility_overwrites:compute_and_dispatch_visibility_changes_for_users( + [?C], Old, New + ), + [Create] = received(c, channel_create, drain()), + ?assertEqual(integer_to_binary(?TEXT), maps:get(<<"id">>, Create)), + ?assertNot(maps:is_key(<<"default_thread_rate_limit_per_user">>, Create)), + [Overwrite] = maps:get(<<"permission_overwrites">>, Create), + ?assertEqual(<<"0">>, maps:get(<<"allow">>, Overwrite)) + end). + +handoff_export_carries_rows_within_budget_test() -> + with_guild(fun(State) -> + Exported = guild_handoff:export_handoff_state(State), + Data = maps:get(data, Exported), + ?assertNot(maps:is_key(thread_store, Data)), + ?assert(is_list(maps:get(thread_rows, Data))), + Restored = guild_init:init_base_state(Exported), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, Restored)), + ?assert( + guild_thread_gate:session_viewer(maps:get(<<"a">>, maps:get(sessions, Restored))) + ), + guild_thread_store:destroy(guild_thread_gate:store(Restored)) + end). + +handoff_strips_rows_over_budget_test() -> + Tab = guild_thread_store:new(), + try + true = ets:insert(Tab, [{{p, 1, N}, true} || N <- lists:seq(1, 50001)]), + Exported = guild_thread_load:export_handoff(#{thread_store => Tab}), + ?assertEqual(#{}, Exported) + after + guild_thread_store:destroy(Tab) + end. + +memory_fixture_of_a_thousand_threads_test() -> + Tab = guild_thread_store:new(), + try + Threads = [thread(10000 + N, ?TEXT, 11) || N <- lists:seq(1, 1000)], + Members = [ + thread_member(10000 + N, 20000 + U) + || N <- lists:seq(1, 1000), U <- lists:seq(1, 50) + ], + ok = guild_thread_store:load( + Tab, + [guild_data_normalize:thread(T) || T <- Threads], + [guild_data_normalize:thread_member(M) || M <- Members] + ), + ?assertEqual(1000, guild_thread_store:thread_count(Tab)), + ?assertEqual(1000 * 2 + 1000 * 50 * 2, guild_thread_store:row_count(Tab)), + Words = ets:info(Tab, memory), + ?assert(Words * erlang:system_info(wordsize) < 64 * 1024 * 1024) + after + guild_thread_store:destroy(Tab) + end. + +can_view_channel_call_sites_are_enumerated_test() -> + Known = lists:sort([ + guild, + guild_data_channels, + guild_dispatch, + guild_dispatch_push, + guild_dispatch_send, + guild_member_list_channel_engine, + guild_member_list_connected, + guild_members_common, + guild_members_query, + guild_passive_sync, + guild_permissions, + guild_permissions_check, + guild_query_handler, + guild_request_forum_unreads, + guild_member_list_subscribe, + guild_sessions, + guild_subscription_handler, + guild_thread_gate, + guild_visibility_channels, + guild_visibility_roles, + guild_members_search, + guild_members_mutation + ]), + Dir = filename:join([code:lib_dir(fluxer_gateway), "..", "..", "..", "..", "src", "guild"]), + Files = filelib:wildcard(filename:join(source_dir(Dir), "*.erl")), + ?assert(length(Files) > 50), + Modules = lists:usort([ + list_to_atom(filename:basename(F, ".erl")) + || F <- Files, + calls_view_primitive(F) + ]), + ?assertEqual([], Modules -- Known). + +source_dir(Fallback) -> + Candidates = [ + "src/guild", + filename:join([code:lib_dir(fluxer_gateway), "src", "guild"]), + Fallback + ], + hd([D || D <- Candidates, filelib:is_dir(D)] ++ [Fallback]). + +calls_view_primitive(File) -> + {ok, Bin} = file:read_file(File), + binary:match(Bin, [ + <<"can_view_channel(">>, + <<"session_can_view_channel(">>, + <<"session_can_view_channel_members(">> + ]) =/= nomatch. + +load_race_replays_events_queued_while_loading_test_() -> + {spawn, {timeout, 20, fun load_race_replays_events_queued_while_loading/0}}. + +load_race_replays_events_queued_while_loading() -> + with_config(fun() -> + Pids = start_captures(), + Base = #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(raw_data()), + sessions => sessions(Pids) + }, + Loading = guild_thread_load:install( + #{thread_gate => #{active => true, version => 1}}, Base + ), + try + ?assert(guild_thread_load:loading(Loading)), + #{thread_load := #{timer := Timer}} = Loading, + erlang:cancel_timer(Timer), + flush_load_retries(), + Ref = make_ref(), + Fetching = Loading#{thread_load => (maps:get(thread_load, Loading))#{ref => Ref}}, + Created = (thread(706, ?TEXT, 11))#{ + <<"_fluxer_members">> => [thread_member(706, ?A)] + }, + Retrying = dispatch(thread_create, Created, Fetching), + ?assertMatch([_], received(a, thread_create, drain())), + Payload = guild_thread_load:load_payload(#{ + <<"threads">> => [thread(?T1, ?TEXT, 11)], + <<"thread_members">> => [thread_member(?T1, ?A)] + }), + Loaded = guild_thread_load:handle_load_result(Ref, {ok, Payload}, Retrying), + ?assertNot(guild_thread_load:loading(Loaded)), + ?assertNotEqual(undefined, guild_thread_gate:thread(?T1, Loaded)), + ?assertNotEqual(undefined, guild_thread_gate:thread(706, Loaded)), + Ticks = lists:sort(sync_ticks(6000)), + ?assertEqual([<<"a">>, <<"x">>], Ticks), + Synced = lists:foldl( + fun(Sid, Acc) -> guild_thread_subscriptions:handle_tick(Sid, Acc) end, + Loaded, + Ticks + ), + Events = drain(), + [Sync] = received(a, thread_list_sync, Events), + ?assertNot(maps:is_key(<<"channel_ids">>, Sync)), + ?assertEqual( + lists:sort([integer_to_binary(?T1), <<"706">>]), + lists:sort([maps:get(<<"id">>, T) || T <- maps:get(<<"threads">>, Sync)]) + ), + ?assertMatch([_], received(x, thread_list_sync, Events)), + ?assertEqual([], received(c, thread_list_sync, Events)), + guild_thread_store:destroy(guild_thread_gate:store(Synced)) + after + stop_captures(Pids) + end + end). + +sync_ticks(Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + sync_ticks(Deadline, []). + +sync_ticks(Deadline, Acc) -> + Wait = max(0, Deadline - erlang:monotonic_time(millisecond)), + receive + {thread_list_sync_tick, Sid} -> sync_ticks(Deadline, [Sid | Acc]) + after Wait -> Acc + end. + +flush_load_retries() -> + receive + thread_load_retry -> flush_load_retries() + after 0 -> ok + end. + +first_fetch_after_a_stripped_handoff_is_immediate_test() -> + flush_load_retries(), + Installed = guild_thread_load:install( + #{thread_gate => #{active => true, version => 1}}, + #{id => ?G, data => #{}, sessions => #{}} + ), + try + ?assert(guild_thread_load:loading(Installed)), + receive + thread_load_retry -> ok + after 100 -> ?assert(false) + end, + Ref = make_ref(), + Loading = Installed#{thread_load => (maps:get(thread_load, Installed))#{ref => Ref}}, + Failed = guild_thread_load:handle_load_result(Ref, {error, down}, Loading), + #{thread_load := #{timer := Timer, attempt := 1}} = Failed, + Remaining = erlang:cancel_timer(Timer), + ?assert(is_integer(Remaining) andalso Remaining > 29000) + after + guild_thread_load:deactivate(Installed) + end. + +active_channels_collection_loads_threads_asynchronously_test() -> + flush_load_retries(), + {Channels, Extras} = guild_thread_load:channels_collection(#{ + <<"channels">> => [#{<<"id">> => <<"1">>}], + <<"thread_gate">> => #{<<"active">> => true, <<"config_version">> => 1}, + <<"threads">> => [thread(?T1, ?TEXT, 11)] + }), + ?assertEqual([#{<<"id">> => <<"1">>}], Channels), + ?assertNot(maps:is_key(thread_load, Extras)), + Installed = guild_thread_load:install(Extras, #{id => ?G, data => #{}, sessions => #{}}), + ?assert(guild_thread_load:loading(Installed)), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, Installed)), + receive + thread_load_retry -> ok + after 100 -> ?assert(false) + end, + guild_thread_load:deactivate(Installed). + +flip_resend_skips_sessions_of_an_unavailable_guild_test() -> + with_guild(fun(State) -> + Data = maps:get(data, State), + Guild = maps:get(<<"guild">>, Data), + Unavailable = State#{ + data => Data#{ + <<"guild">> => Guild#{<<"features">> => [<<"UNAVAILABLE_FOR_EVERYONE">>]} + } + }, + _ = guild_thread_flip:resend(<<"a">>, Unavailable), + ?assertEqual([], received(a, guild_create, drain())), + _ = guild_thread_flip:resend(<<"a">>, State), + ?assertMatch([_], received(a, guild_create, drain())) + end). + +flip_resend_keeps_a_degraded_guild_degraded_test() -> + with_guild(fun(State) -> + ok = guild_ets_owner:ensure_table(guild_health_status, [named_table, public, set]), + _ = guild_thread_flip:resend(<<"a">>, State), + [Healthy] = received(a, guild_create, drain()), + ?assertNot(maps:is_key(<<"degraded">>, Healthy)), + true = ets:insert(guild_health_status, {self(), ?G, true, undefined, undefined}), + try + _ = guild_thread_flip:resend(<<"a">>, State), + ?assertMatch([#{<<"degraded">> := true}], received(a, guild_create, drain())) + after + ets:delete(guild_health_status, self()) + end + end). + +gate_only_reload_keeps_the_ready_store_while_it_refetches_test() -> + with_guild(fun(State) -> + Store = guild_thread_gate:store(State), + Reloaded = guild_thread_flip:reinstall( + #{thread_gate => #{active => true, version => 1}}, State, State + ), + flush_load_retries(), + ?assertEqual(Store, guild_thread_gate:store(Reloaded)), + ?assertMatch(#{<<"id">> := _}, guild_thread_gate:thread(?T1, Reloaded)), + ?assert(guild_thread_load:loading(Reloaded)), + Ref = make_ref(), + Fresh = guild_thread_load:handle_load_result( + Ref, + {ok, guild_thread_load:load_payload(#{<<"threads">> => [thread(?T2, ?TEXT, 12)]})}, + thread_load_ref(Ref, Reloaded) + ), + ?assertNot(guild_thread_load:loading(Fresh)), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, Fresh)), + ?assertMatch(#{<<"id">> := _}, guild_thread_gate:thread(?T2, Fresh)), + guild_thread_store:destroy(guild_thread_gate:store(Fresh)) + end). + +changed_gate_reload_starts_an_empty_store_test() -> + with_guild(fun(State) -> + Reloaded = guild_thread_flip:reinstall( + #{thread_gate => #{active => true, version => 2}}, State, State + ), + flush_load_retries(), + try + ?assertNotEqual(guild_thread_gate:store(State), guild_thread_gate:store(Reloaded)), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, Reloaded)) + after + guild_thread_store:destroy(guild_thread_gate:store(Reloaded)) + end + end). + +thread_load_ref(Ref, #{thread_load := Load} = State) -> + State#{thread_load => Load#{ref => Ref}}. + +reactivation_clears_thread_subscriptions_so_op14_resyncs_test_() -> + {spawn, {timeout, 20, fun reactivation_clears_thread_subscriptions_so_op14_resyncs/0}}. + +reactivation_clears_thread_subscriptions_so_op14_resyncs() -> + with_guild(fun(State) -> + Sessions = maps:get(sessions, State), + A = maps:get(<<"a">>, Sessions), + Subscribed = State#{ + sessions => Sessions#{ + <<"a">> => A#{thread_subscribed => true, thread_member_lists => [?T1]} + } + }, + Off = guild_thread_flip:reinstall( + #{thread_gate => #{active => false, version => 2}}, Subscribed, Subscribed + ), + OffA = maps:get(<<"a">>, maps:get(sessions, Off)), + ?assertNot(maps:is_key(thread_subscribed, OffA)), + ?assertNot(maps:is_key(thread_member_lists, OffA)), + On = guild_thread_flip:reinstall( + #{ + thread_gate => #{active => true, version => 3}, + thread_load => {ok, #{threads => [], members => [], parent_settings => []}} + }, + Off, + Off + ), + try + OnA = maps:get(<<"a">>, maps:get(sessions, On)), + ?assert(guild_thread_gate:session_viewer(OnA)), + Updated = guild_thread_subscriptions:update(<<"a">>, #{threads => true}, On), + ?assertEqual( + true, + maps:get(thread_subscribed, maps:get(<<"a">>, maps:get(sessions, Updated))) + ), + receive + {thread_list_sync_tick, <<"a">>} -> ok + after 6000 -> ?assert(false) + end + after + guild_thread_store:destroy(guild_thread_gate:store(On)), + flush_flip_resends() + end + end). + +flush_flip_resends() -> + receive + {thread_flip_resend, _} -> flush_flip_resends(); + {thread_list_sync_tick, _} -> flush_flip_resends() + after 0 -> ok + end. + +thread_member_list_follows_presence_changes_test() -> + with_guild(fun(State) -> + Subscribed = guild_thread_subscriptions:update( + <<"a">>, #{member_lists => [?T1]}, State + ), + ?assertMatch(#{thread_presence_users := #{?A := [?T1]}}, Subscribed), + [_] = received(a, thread_member_list_update, drain()), + Changed = lists:foldl( + fun guild_thread_subscriptions:presence_changed/2, Subscribed, [?A, ?A, ?A] + ), + ?assertEqual([], received(a, thread_member_list_update, drain())), + _ = guild_thread_subscriptions:handle_list_flush(Changed), + ?assertMatch([_], received(a, thread_member_list_update, drain())), + ?assertEqual(Subscribed, guild_thread_subscriptions:presence_changed(?C, Subscribed)), + ?assertEqual(State, guild_thread_subscriptions:presence_changed(?A, State)) + end). + +handoff_through_an_old_node_round_trips_test() -> + with_guild(fun(State) -> + Exported = guild_handoff:export_handoff_state(State), + OldShape = Exported#{ + data => maps:without( + [thread_gate, thread_tainted, thread_rows, thread_rows_at], + maps:get(data, Exported) + ) + }, + FromOld = guild_init:init_base_state(OldShape), + ?assertEqual(undefined, guild_thread_gate:store(FromOld)), + ?assertNot(guild_thread_load:loading(FromOld)), + ?assertNot( + lists:any( + fun guild_thread_gate:session_viewer/1, + maps:values(maps:get(sessions, FromOld)) + ) + ), + ReExported = guild_handoff:export_handoff_state(FromOld), + ?assertNot(maps:is_key(thread_rows, maps:get(data, ReExported))), + Stale = Exported#{ + data => (maps:get(data, Exported))#{thread_rows_at => 0} + }, + flush_load_retries(), + FromStale = guild_init:init_base_state(Stale), + try + ?assert(guild_thread_load:loading(FromStale)), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, FromStale)), + receive + thread_load_retry -> ok + after 100 -> ?assert(false) + end + after + guild_thread_load:deactivate(FromStale) + end + end). + +excluded_bot_is_a_control_session_test() -> + with_config(fun() -> + Config = persistent_term:get(?CONFIG_KEY), + persistent_term:put(?CONFIG_KEY, Config#{ + excluded_users => #{integer_to_binary(?X) => true} + }), + Pids = start_captures(), + State = active_state(Pids), + try + ?assertNot( + guild_thread_gate:session_viewer(maps:get(<<"x">>, maps:get(sessions, State))) + ), + _ = dispatch(thread_create, thread(707, ?TEXT, 11), State), + ?assertEqual([], received(x, thread_create, drain())) + after + guild_thread_store:destroy(guild_thread_gate:store(State)), + stop_captures(Pids) + end + end). + +thread_message_permission_work_ignores_non_member_sessions_test() -> + with_guild(fun(State) -> + Message = #{ + <<"id">> => <<"920">>, + <<"channel_id">> => integer_to_binary(?T1), + <<"type">> => 0, + <<"content">> => <<"hi">>, + <<"author">> => #{<<"id">> => integer_to_binary(?X)}, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + Crowd = maps:from_list([ + { + <<"crowd", (integer_to_binary(N))/binary>>, + (session(crowd, UserId, false, true, self()))#{ + session_id => <<"crowd", (integer_to_binary(N))/binary>> + } + } + || N <- lists:seq(1, 1000), UserId <- [lists:nth(1 + N rem 2, [?C, ?V])] + ]), + Crowded = guild_thread_gate:recompute_session_viewers(State#{ + sessions => maps:merge(maps:get(sessions, State), Crowd) + }), + meck:new(guild_thread_permissions, [passthrough, no_link]), + try + _ = dispatch(message_create, Message, State), + Base = meck:num_calls(guild_thread_permissions, resolve, '_'), + ?assertMatch([_], received(a, message_create, drain())), + meck:reset(guild_thread_permissions), + _ = dispatch(message_create, Message#{<<"id">> => <<"921">>}, Crowded), + ?assertEqual(Base, meck:num_calls(guild_thread_permissions, resolve, '_')), + ?assertMatch([_], received(a, message_create, drain())) + after + meck:unload(guild_thread_permissions) + end, + flush_crowd() + end). + +flush_crowd() -> + receive + {'$gen_cast', _} -> flush_crowd() + after 0 -> ok + end. + +tainted_inactive_guild_drops_thread_message_deletes_test() -> + with_config(fun() -> + Pids = start_captures(), + try + {_Channels, Extras} = guild_thread_load:channels_collection(#{ + <<"channels">> => [], <<"thread_tainted">> => true + }), + State = guild_thread_load:install(Extras, #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(raw_data()), + sessions => sessions(Pids), + member_list_subscriptions => guild_member_list_subs:new() + }), + Bulk = #{<<"channel_id">> => integer_to_binary(?T1), <<"ids">> => [<<"930">>]}, + _ = dispatch(message_delete_bulk, Bulk, State), + _ = dispatch( + message_delete_bulk, + Bulk#{<<"_fluxer_thread">> => thread(?T1, ?TEXT, 11)}, + State + ), + ?assertEqual([], drain()), + Text = #{<<"channel_id">> => integer_to_binary(?TEXT), <<"ids">> => [<<"931">>]}, + _ = dispatch(message_delete_bulk, Text, State), + ?assertMatch([_], received(c, message_delete_bulk, drain())) + after + stop_captures(Pids) + end + end). + +archived_thread_message_delete_uses_the_internal_context_test() -> + with_guild(fun(State0) -> + Archived = (thread(?T1, ?TEXT, 11))#{ + <<"thread_metadata">> => #{<<"archived">> => true, <<"locked">> => false} + }, + State1 = guild_thread_subscriptions:update( + <<"v">>, #{threads => true}, dispatch(thread_update, Archived, State0) + ), + drain(), + ?assertEqual(undefined, guild_thread_gate:thread(?T1, State1)), + Delete = #{<<"channel_id">> => integer_to_binary(?T1), <<"id">> => <<"940">>}, + _ = dispatch(message_delete, Delete, State1), + ?assertEqual([], drain()), + Context = #{ + <<"id">> => integer_to_binary(?T1), + <<"parent_id">> => integer_to_binary(?TEXT), + <<"type">> => 11 + }, + _ = dispatch(message_delete, Delete#{<<"_fluxer_thread">> => Context}, State1), + Events = drain(), + [ForV] = received(v, message_delete, Events), + ?assertNot(maps:is_key(<<"_fluxer_thread">>, ForV)), + ?assertMatch([_], received(x, message_delete, Events)), + ?assertEqual([], received(c, message_delete, Events)), + ?assertEqual([], received(i, message_delete, Events)) + end). + +archived_thread_member_update_checks_parent_access_test() -> + with_guild(fun(State0) -> + State1 = dispatch(thread_create, thread(708, ?HIDDEN, 11), State0), + Archived = (thread(708, ?HIDDEN, 11))#{ + <<"thread_metadata">> => #{<<"archived">> => true, <<"locked">> => false} + }, + State2 = dispatch(thread_update, Archived, State1), + drain(), + ?assertEqual(undefined, guild_thread_gate:thread(708, State2)), + Update = fun(UserId) -> + (thread_member(708, UserId))#{<<"_fluxer_parent_id">> => integer_to_binary(?HIDDEN)} + end, + _ = dispatch(thread_member_update, Update(?V), State2), + [ForV] = received(v, thread_member_update, drain()), + ?assertNot(maps:is_key(<<"_fluxer_parent_id">>, ForV)), + _ = dispatch(thread_member_update, Update(?P), State2), + ?assertEqual([], received(p, thread_member_update, drain())), + _ = dispatch(thread_member_update, thread_member(708, ?V), State2), + ?assertEqual([], received(v, thread_member_update, drain())) + end). + +handoff_restores_thread_member_list_presence_tracking_test() -> + with_guild(fun(State) -> + Subscribed = guild_thread_subscriptions:update( + <<"a">>, #{member_lists => [?T1]}, State + ), + drain(), + Exported = guild_manager_shard_lifecycle:normalize_transferred_guild_state( + maps:get(id, Subscribed), guild_handoff:export_handoff_state(Subscribed) + ), + Restored = guild_init:init_base_state(Exported), + try + ?assertMatch(#{thread_presence_users := #{?A := [?T1]}}, Restored), + _ = guild_thread_subscriptions:handle_list_flush( + guild_thread_subscriptions:presence_changed(?A, Restored) + ), + ?assertMatch([_], received(a, thread_member_list_update, drain())) + after + guild_thread_store:destroy(guild_thread_gate:store(Restored)) + end + end). + +capable_and_incapable_sessions_of_one_user_split_test() -> + with_guild(fun(State) -> + Sessions = maps:get(sessions, State), + ?assert(guild_thread_gate:session_viewer(maps:get(<<"a">>, Sessions))), + ?assertNot(guild_thread_gate:session_viewer(maps:get(<<"i">>, Sessions))), + ?assert( + maps:is_key( + <<"threads">>, + guild_data:get_guild_state( + ?A, State, guild_thread_gate:state_opts(maps:get(<<"a">>, Sessions)) + ) + ) + ), + ?assertNot( + maps:is_key( + <<"threads">>, + guild_data:get_guild_state( + ?A, State, guild_thread_gate:state_opts(maps:get(<<"i">>, Sessions)) + ) + ) + ), + _ = dispatch( + thread_create, + (thread(709, ?TEXT, 11))#{<<"_fluxer_members">> => [thread_member(709, ?A)]}, + State + ), + Created = drain(), + ?assertMatch([_], received(a, thread_create, Created)), + ?assertEqual([], received(i, thread_create, Created)), + Update = #{ + <<"id">> => <<"950">>, + <<"channel_id">> => integer_to_binary(?TEXT), + <<"flags">> => 16#20 bor 4, + <<"thread">> => #{<<"id">> => <<"709">>} + }, + _ = dispatch(message_update, Update, State), + Updated = drain(), + [Full] = received(a, message_update, Updated), + ?assertEqual(16#24, maps:get(<<"flags">>, Full)), + ?assert(maps:is_key(<<"thread">>, Full)), + [Masked] = received(i, message_update, Updated), + ?assertEqual(4, maps:get(<<"flags">>, Masked)), + ?assertNot(maps:is_key(<<"thread">>, Masked)) + end). + +forum_only_shared_channel_grants_no_presence_to_control_sessions_test() -> + with_guild(fun(State0) -> + Data0 = maps:get(data, State0), + Forum = [ + maps:remove(<<"parent_id">>, C) + || C <- guild_data_index:channel_list(Data0), maps:get(<<"type">>, C) =:= 15 + ], + State = State0#{data => guild_data_index:put_channels(Forum, Data0)}, + Sessions = maps:get(sessions, State), + ?assertEqual( + [?V], + guild_subscription_mutual_channels:filter_member_ids( + maps:get(<<"a">>, Sessions), ?A, [?V], State + ) + ), + ?assertEqual( + [], + guild_subscription_mutual_channels:filter_member_ids( + maps:get(<<"c">>, Sessions), ?C, [?V], State + ) + ), + ?assertEqual( + #{<<"a">> => [?V], <<"c">> => []}, + guild_subscription_mutual_channels:filter_session_member_ids( + [{<<"a">>, ?A, [?V]}, {<<"c">>, ?C, [?V]}], State + ) + ) + end). + +-define(NEWS, 110). +-define(NEWS_THREAD, 800). + +news_raw_data() -> + Raw = raw_data(), + News = #{<<"id">> => <<"110">>, <<"type">> => 5, <<"permission_overwrites">> => []}, + Raw#{<<"channels">> => maps:get(<<"channels">>, Raw) ++ [News]}. + +news_state(Pids) -> + Payload = guild_thread_load:load_payload(#{ + <<"threads">> => [thread(?T1, ?TEXT, 11), thread(?NEWS_THREAD, ?NEWS, 10)], + <<"thread_members">> => [thread_member(?T1, ?A), thread_member(?NEWS_THREAD, ?A)] + }), + guild_thread_load:install( + #{ + thread_gate => #{active => true, version => 1}, + thread_tainted => true, + thread_load => {ok, Payload} + }, + #{ + id => ?G, + member_count => 10, + data => guild_data_index:normalize_map(news_raw_data()), + sessions => sessions(Pids), + member_list_subscriptions => guild_member_list_subs:new() + } + ). + +with_news_guild(Fun) -> + with_config(fun() -> + Pids = start_captures(), + State = news_state(Pids), + try + Fun(State) + after + guild_thread_store:destroy(guild_thread_gate:store(State)), + stop_captures(Pids), + settle_mailbox_age() + end + end). + +news_thread_create_reaches_viewers_only_test() -> + with_news_guild(fun(State0) -> + Data = (thread(801, ?NEWS, 10))#{ + <<"newly_created">> => true, + <<"_fluxer_members">> => [thread_member(801, ?A)] + }, + State1 = dispatch(thread_create, Data, State0), + Events1 = drain(), + [ForA] = received(a, thread_create, Events1), + ?assertEqual(10, maps:get(<<"type">>, ForA)), + ?assertEqual(integer_to_binary(?NEWS), maps:get(<<"parent_id">>, ForA)), + ?assertMatch([_], received(x, thread_create, Events1)), + ?assertEqual([], received(v, thread_create, Events1)), + ?assertEqual([], received(c, thread_create, Events1)), + ?assertEqual([], received(i, thread_create, Events1)), + State2 = guild_thread_subscriptions:update(<<"v">>, #{threads => true}, State1), + _ = dispatch(thread_create, thread(802, ?NEWS, 10), State2), + Events2 = drain(), + ?assertMatch([_], received(v, thread_create, Events2)), + ?assertEqual([], received(c, thread_create, Events2)), + ?assertEqual([], received(i, thread_create, Events2)) + end). + +news_thread_permissions_derive_from_announcement_parent_test() -> + with_news_guild(fun(State) -> + Perms = guild_permissions:get_member_permissions(?V, ?NEWS_THREAD, State), + ?assert(permission_bits:has(Perms, ?VIEW)), + ?assert(permission_bits:has(Perms, ?SEND)), + {reply, #{auth_context := Context}, _} = guild_data:get_auth_context( + #{user_id => ?V, channel_id => ?NEWS_THREAD}, State + ), + ?assertEqual(?NEWS, maps:get(<<"id">>, maps:get(<<"parent_channel">>, Context))), + [C] = [S || S <- maps:values(maps:get(sessions, State)), maps:get(user_id, S) =:= ?C], + ?assert(guild_thread_gate:channel_visible(C, ?NEWS, State)), + ?assertNot(guild_thread_gate:channel_visible(C, ?NEWS_THREAD, State)) + end). + +news_thread_permission_loss_on_parent_hides_thread_test() -> + with_news_guild(fun(State0) -> + Gid = integer_to_binary(?G), + Hidden = #{ + <<"id">> => integer_to_binary(?NEWS), + <<"type">> => 5, + <<"permission_overwrites">> => [ + #{ + <<"id">> => Gid, + <<"type">> => 0, + <<"allow">> => <<"0">>, + <<"deny">> => <<"1024">> + } + ] + }, + State1 = dispatch(channel_update, Hidden, State0), + _ = drain(), + ?assertNot( + permission_bits:has( + guild_permissions:get_member_permissions(?A, ?NEWS_THREAD, State1), ?VIEW + ) + ), + ?assertNotEqual(undefined, guild_thread_gate:thread(?NEWS_THREAD, State1)), + _ = dispatch(thread_update, thread(?NEWS_THREAD, ?NEWS, 10), State1), + ?assertEqual([], received(a, thread_update, drain())) + end). + +news_guild_create_carries_type_10_for_viewers_only_test() -> + with_news_guild(fun(State) -> + Viewer = guild_data:get_guild_state(?A, State, #{thread_viewer => true, bot => false}), + [News] = [ + T + || T <- maps:get(<<"threads">>, Viewer), maps:get(<<"id">>, T) =:= ?NEWS_THREAD + ], + ?assertEqual(10, maps:get(<<"type">>, News)), + ?assert(maps:is_key(<<"member">>, News)), + Bot = guild_data:get_guild_state(?X, State, #{thread_viewer => true, bot => true}), + ?assert( + lists:member(?NEWS_THREAD, [ + maps:get(<<"id">>, T) + || T <- maps:get(<<"threads">>, Bot) + ]) + ), + Control = guild_data:get_guild_state(?C, State, #{thread_viewer => false}), + ?assertNot(maps:is_key(<<"threads">>, Control)), + Ids = [maps:get(<<"id">>, C) || C <- maps:get(<<"channels">>, Control)], + ?assert(lists:member(?NEWS, Ids)), + ?assertNot(lists:member(?NEWS_THREAD, Ids)) + end). + +news_thread_messages_and_crosspost_updates_hide_threads_from_control_test() -> + with_news_guild(fun(State) -> + Message = #{ + <<"id">> => <<"910">>, + <<"channel_id">> => integer_to_binary(?NEWS_THREAD), + <<"type">> => 0, + <<"content">> => <<"hi">>, + <<"author">> => #{<<"id">> => integer_to_binary(?X)}, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + _ = dispatch(message_create, Message, State), + Events1 = drain(), + ?assertMatch([_], received(a, message_create, Events1)), + ?assertEqual([], received(c, message_create, Events1)), + ?assertEqual([], received(i, message_create, Events1)), + Created = #{ + <<"id">> => <<"911">>, + <<"channel_id">> => integer_to_binary(?NEWS), + <<"type">> => 18, + <<"author">> => #{<<"id">> => integer_to_binary(?A)}, + <<"mentions">> => [], + <<"mention_roles">> => [] + }, + _ = dispatch(message_create, Created, State), + Events2 = drain(), + ?assertMatch([_], received(a, message_create, Events2)), + ?assertEqual([], received(c, message_create, Events2)), + Crossposted = #{ + <<"id">> => <<"912">>, + <<"channel_id">> => integer_to_binary(?NEWS), + <<"flags">> => 16#20 bor 1, + <<"thread">> => thread(912, ?NEWS, 10) + }, + _ = dispatch(message_update, Crossposted, State), + Events3 = drain(), + [Full] = received(a, message_update, Events3), + ?assertEqual(16#21, maps:get(<<"flags">>, Full)), + ?assertEqual(10, maps:get(<<"type">>, maps:get(<<"thread">>, Full))), + lists:foreach( + fun(Tag) -> + [Masked] = received(Tag, message_update, Events3), + ?assertEqual(1, maps:get(<<"flags">>, Masked)), + ?assertNot(maps:is_key(<<"thread">>, Masked)) + end, + [c, i] + ) + end). + +news_thread_list_sync_includes_type_10_test() -> + with_news_guild(fun(State0) -> + State1 = guild_thread_subscriptions:update(<<"v">>, #{threads => true}, State0), + _ = guild_thread_subscriptions:handle_tick(<<"v">>, State1), + [Sync] = received(v, thread_list_sync, drain()), + ?assertEqual( + [integer_to_binary(?T1), integer_to_binary(?NEWS_THREAD)], + lists:sort([maps:get(<<"id">>, T) || T <- maps:get(<<"threads">>, Sync)]) + ) + end). + +inactive_guild_drops_news_threads_and_keeps_announcement_channels_test() -> + with_config(fun() -> + Pids = start_captures(), + try + State = inactive_state(Pids), + _ = dispatch(thread_create, thread(803, ?TEXT, 10), State), + _ = dispatch(channel_create, #{<<"id">> => <<"804">>, <<"type">> => 10}, State), + ?assertEqual([], drain()), + News = #{<<"id">> => <<"805">>, <<"type">> => 5, <<"permission_overwrites">> => []}, + _ = dispatch(channel_create, News, State), + Events = drain(), + [ForI] = received(i, channel_create, Events), + ?assertEqual(5, maps:get(<<"type">>, ForI)), + ?assertMatch([_], received(c, channel_create, Events)) + after + stop_captures(Pids) + end + end). + +news_thread_handoff_round_trips_test() -> + with_news_guild(fun(State) -> + Exported = guild_handoff:export_handoff_state(State), + Restored = guild_init:init_base_state(Exported), + try + ?assertMatch( + #{<<"type">> := 10}, guild_thread_gate:thread(?NEWS_THREAD, Restored) + ), + ?assert( + permission_bits:has( + guild_permissions:get_member_permissions(?V, ?NEWS_THREAD, Restored), ?VIEW + ) + ) + after + guild_thread_store:destroy(guild_thread_gate:store(Restored)) + end + end). diff --git a/fluxer_gateway/test/session_lifecycle_tests.erl b/fluxer_gateway/test/session_lifecycle_tests.erl index 472efd8da..6eea9e09d 100644 --- a/fluxer_gateway/test/session_lifecycle_tests.erl +++ b/fluxer_gateway/test/session_lifecycle_tests.erl @@ -31,6 +31,9 @@ serialize_state_test() -> ?assertEqual({2, 8}, maps:get(shard, Serialized)), ?assert(sets:is_element(123, maps:get(active_guilds, Serialized))), ?assertEqual(10, maps:get(seq, Serialized)), + ?assertEqual(false, maps:get(thread_channels_capable, Serialized)), + Capable = session_lifecycle:serialize_state(State#{thread_channels_capable => true}), + ?assertEqual(true, maps:get(thread_channels_capable, Capable)), ok. serialize_transfer_state_includes_resume_fields_test() -> @@ -55,6 +58,7 @@ serialize_transfer_state_includes_resume_fields_test() -> ignored_events => #{<<"TYPING_START">> => true}, initial_guild_id => 123, debounce_reactions => true, + thread_channels_capable => true, channels => #{}, relationships => #{}, seq => 10, @@ -73,7 +77,8 @@ serialize_transfer_state_includes_resume_fields_test() -> ?assertEqual(10, maps:get(seq, TransferState)), ?assertEqual(8, maps:get(ack_seq, TransferState)), ?assertEqual(8, maps:get(replay_floor, TransferState)), - ?assertEqual([#{seq => 9}], maps:get(buffer, TransferState)). + ?assertEqual([#{seq => 9}], maps:get(buffer, TransferState)), + ?assertEqual(true, maps:get(thread_channels_capable, TransferState)). serialize_transfer_state_strips_socket_pid_test() -> State = #{ @@ -108,7 +113,8 @@ serialize_transfer_state_strips_socket_pid_test() -> collected_presences => [] }, TransferState = session_lifecycle:serialize_transfer_state(State), - ?assertEqual(undefined, maps:get(socket_pid, TransferState)). + ?assertEqual(undefined, maps:get(socket_pid, TransferState)), + ?assertEqual(false, maps:get(thread_channels_capable, TransferState)). serialize_transfer_state_preserves_deque_replay_buffer_test() -> Deque = lists:foldl( diff --git a/fluxer_gateway/test/session_manager_shard_drain_tests.erl b/fluxer_gateway/test/session_manager_shard_drain_tests.erl index db2d8e6f5..1d0ba6867 100644 --- a/fluxer_gateway/test/session_manager_shard_drain_tests.erl +++ b/fluxer_gateway/test/session_manager_shard_drain_tests.erl @@ -15,6 +15,62 @@ should_debounce_reactions_test() -> ?assertEqual(false, session_manager_shard_drain:should_debounce_reactions(#{flags => -1})), ok. +thread_channels_capable_reads_identify_flag_bit_two_test() -> + ?assertEqual(false, session_manager_shard_drain:thread_channels_capable(#{})), + ?assertEqual(false, session_manager_shard_drain:thread_channels_capable(#{flags => 0})), + ?assertEqual(false, session_manager_shard_drain:thread_channels_capable(#{flags => 2})), + ?assertEqual(true, session_manager_shard_drain:thread_channels_capable(#{flags => 4})), + ?assertEqual(true, session_manager_shard_drain:thread_channels_capable(#{flags => 6})), + ?assertEqual(false, session_manager_shard_drain:thread_channels_capable(#{flags => -4})), + ?assertEqual( + false, session_manager_shard_drain:thread_channels_capable(#{flags => <<"4">>}) + ). + +build_session_data_carries_thread_channels_capability_test() -> + UserDataMap = #{ + <<"id">> => <<"123">>, + <<"username">> => <<"tester">>, + <<"discriminator">> => <<"0001">>, + <<"avatar">> => null, + <<"flags">> => 0 + }, + Data = #{<<"guilds">> => [], <<"user">> => UserDataMap, <<"user_settings">> => null}, + Base = #{properties => #{}, token => <<"token">>, presence => null}, + Capable = build_test_session_data(Data, Base#{flags => 6}, UserDataMap), + Incapable = build_test_session_data(Data, Base#{flags => 2}, UserDataMap), + Unflagged = build_test_session_data(Data, Base, UserDataMap), + ?assertEqual(true, maps:get(thread_channels_capable, Capable)), + ?assertEqual(true, maps:get(debounce_reactions, Capable)), + ?assertEqual(false, maps:get(thread_channels_capable, Incapable)), + ?assertEqual(false, maps:get(thread_channels_capable, Unflagged)). + +session_rpc_sends_thread_channels_capable_only_when_flagged_test() -> + meck:new(api_rpc_client, [passthrough, no_link]), + Self = self(), + meck:expect(api_rpc_client, call_with_retry, fun(Request, _Config) -> + Self ! {session_rpc_request, Request}, + {ok, #{}} + end), + try + Identify = #{token => <<"token">>, properties => #{}}, + {ok, _} = session_manager_shard_drain:fetch_rpc_data( + #{identify_data => Identify#{flags => 4}, version => 1}, <<"127.0.0.1">> + ), + ?assertMatch(#{<<"thread_channels_capable">> := true}, receive_session_rpc_request()), + {ok, _} = session_manager_shard_drain:fetch_rpc_data( + #{identify_data => Identify#{flags => 2}, version => 1}, <<"127.0.0.1">> + ), + ?assertNot(maps:is_key(<<"thread_channels_capable">>, receive_session_rpc_request())) + after + meck:unload(api_rpc_client) + end. + +receive_session_rpc_request() -> + receive + {session_rpc_request, Request} -> Request + after 1000 -> erlang:error(session_rpc_request_not_sent) + end. + reconnect_drain_casts_each_active_session_test() -> TestPid = self(), SessionPidA = spawn(fun() -> reconnect_drain_test_session(TestPid) end), diff --git a/fluxer_gateway/test/thread_permission_cases.json b/fluxer_gateway/test/thread_permission_cases.json new file mode 100644 index 000000000..3cd08ae1a --- /dev/null +++ b/fluxer_gateway/test/thread_permission_cases.json @@ -0,0 +1,1457 @@ +[ + { + "name": "owner is a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": [], + "isOwner": true + }, + "expect": true + }, + { + "name": "administrator is a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": ["ADMINISTRATOR"] + }, + "expect": true + }, + { + "name": "manage threads is a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "expect": true + }, + { + "name": "timed out manage threads is not a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"], + "timedOut": true + }, + "expect": false + }, + { + "name": "timed out administrator stays a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": ["ADMINISTRATOR"], + "timedOut": true + }, + "expect": true + }, + { + "name": "plain member is not a moderator", + "fn": "isThreadModerator", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "expect": false + }, + { + "name": "public thread needs parent view", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "expect": null + }, + { + "name": "public thread without parent view", + "fn": "canViewThread", + "actor": { + "perms": [] + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "private thread visible to members", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL"], + "isMember": true + }, + "thread": { + "type": 12 + }, + "expect": null + }, + { + "name": "private thread hidden from non-members", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "thread": { + "type": 12 + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "private thread visible to moderators", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL", "MANAGE_THREADS"] + }, + "thread": { + "type": 12 + }, + "expect": null + }, + { + "name": "private thread hidden from timed out moderators", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL", "MANAGE_THREADS"], + "timedOut": true + }, + "thread": { + "type": 12 + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "private thread visible to administrators", + "fn": "canViewThread", + "actor": { + "perms": ["ADMINISTRATOR"] + }, + "thread": { + "type": 12 + }, + "expect": null + }, + { + "name": "private thread member without parent view", + "fn": "canViewThread", + "actor": { + "perms": [], + "isMember": true + }, + "thread": { + "type": 12 + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "announcement thread visible with parent view", + "fn": "canViewThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "thread": { + "type": 10 + }, + "expect": null + }, + { + "name": "announcement thread hidden without parent view", + "fn": "canViewThread", + "actor": { + "perms": [] + }, + "thread": { + "type": 10 + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "send in an open thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "args": { + "action": "send" + }, + "expect": null + }, + { + "name": "send in an archived unlocked thread unarchives", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "action": "send" + }, + "expect": null + }, + { + "name": "send in a locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "locked": true + }, + "args": { + "action": "send" + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "send in an archived locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true, + "locked": true + }, + "args": { + "action": "send" + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "moderator sends in a locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true, + "locked": true + }, + "args": { + "action": "send" + }, + "expect": null + }, + { + "name": "timed out send", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "timedOut": true + }, + "args": { + "action": "send" + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "timed out owner sends", + "fn": "threadWriteBlock", + "actor": { + "perms": [], + "timedOut": true, + "isOwner": true + }, + "args": { + "action": "send" + }, + "expect": null + }, + { + "name": "react in an archived thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "action": "react" + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "moderator reacts in an archived thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true + }, + "args": { + "action": "react" + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "pin in a locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "locked": true + }, + "args": { + "action": "pin" + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "moderator pins in a locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "locked": true + }, + "args": { + "action": "pin" + }, + "expect": null + }, + { + "name": "edit in an open thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "args": { + "action": "edit" + }, + "expect": null + }, + { + "name": "edit in an archived thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "action": "edit" + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "delete in an archived locked thread", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true, + "locked": true + }, + "args": { + "action": "delete" + }, + "expect": null + }, + { + "name": "timed out delete", + "fn": "threadWriteBlock", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "timedOut": true + }, + "args": { + "action": "delete" + }, + "expect": null + }, + { + "name": "member unarchives an unlocked thread", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true + }, + "expect": null + }, + { + "name": "non-member cannot unarchive", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "creator who left unarchives an unlocked thread", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "archived": true + }, + "expect": null + }, + { + "name": "member without send in threads unarchives an unlocked thread", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true + }, + "expect": null + }, + { + "name": "member cannot unarchive a locked thread", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true, + "locked": true + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "moderator unarchives a locked thread", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true, + "locked": true + }, + "expect": null + }, + { + "name": "timed out member cannot unarchive", + "fn": "canUnarchive", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true, + "timedOut": true + }, + "thread": { + "archived": true + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "thread owner renames", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": null + }, + { + "name": "other member cannot rename", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "moderator renames", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": null + }, + { + "name": "thread owner changes auto archive", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "auto_archive_duration": 60 + } + }, + "expect": null + }, + { + "name": "thread owner archives", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "archived": true + } + }, + "expect": null + }, + { + "name": "member cannot archive", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "args": { + "patch": { + "archived": true + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "member unarchives through patch", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "archived": false + } + }, + "expect": null + }, + { + "name": "non-member cannot unarchive through patch", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "archived": false + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "creator who left unarchives through patch", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "archived": false + } + }, + "expect": null + }, + { + "name": "member without send in threads unarchives through patch", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "archived": false + } + }, + "expect": null + }, + { + "name": "member cannot unarchive a locked thread through patch", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true, + "locked": true + }, + "args": { + "patch": { + "archived": false + } + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "renaming an archived thread", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "renaming while unarchiving", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true + }, + "args": { + "patch": { + "name": "x", + "archived": false + } + }, + "expect": null + }, + { + "name": "thread owner locks", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "locked": true + } + }, + "expect": null + }, + { + "name": "thread owner cannot unlock", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "locked": true + }, + "args": { + "patch": { + "locked": false + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "member cannot lock", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "args": { + "patch": { + "locked": true + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "moderator unlocks", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "locked": true + }, + "args": { + "patch": { + "locked": false + } + }, + "expect": null + }, + { + "name": "thread owner renames a locked thread", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "locked": true + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "thread owner changes slowmode", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "rate_limit_per_user": 10 + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "moderator changes slowmode", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "args": { + "patch": { + "rate_limit_per_user": 10 + } + }, + "expect": null + }, + { + "name": "thread owner changes invitable", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "type": 12 + }, + "args": { + "patch": { + "invitable": false + } + }, + "expect": null + }, + { + "name": "member cannot change invitable", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "type": 12 + }, + "args": { + "patch": { + "invitable": false + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "thread owner cannot pin", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "patch": { + "flags": 2 + }, + "currentFlags": 0 + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "unchanged flags need nothing", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "args": { + "patch": { + "flags": 2 + }, + "currentFlags": 2 + }, + "expect": null + }, + { + "name": "moderator pins", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "args": { + "patch": { + "flags": 2 + }, + "currentFlags": 0 + }, + "expect": null + }, + { + "name": "timed out thread owner renames", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true, + "timedOut": true + }, + "args": { + "patch": { + "name": "x" + } + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "empty patch on an archived thread", + "fn": "threadPatchRequirement", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "patch": {} + }, + "expect": null + }, + { + "name": "thread owner sets plain tags", + "fn": "canSetTags", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "touchesModeratedTag": false + }, + "expect": null + }, + { + "name": "thread owner cannot set moderated tags", + "fn": "canSetTags", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "args": { + "touchesModeratedTag": true + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "moderator sets moderated tags", + "fn": "canSetTags", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "args": { + "touchesModeratedTag": true + }, + "expect": null + }, + { + "name": "member cannot set tags", + "fn": "canSetTags", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "args": { + "touchesModeratedTag": false + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "join a public thread", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "expect": null + }, + { + "name": "join an archived thread", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "thread": { + "archived": true + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "join a locked thread", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "thread": { + "locked": true + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "moderator joins a locked thread", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL", "MANAGE_THREADS"] + }, + "thread": { + "locked": true + }, + "expect": null + }, + { + "name": "join without parent view", + "fn": "canJoinThread", + "actor": { + "perms": [] + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "timed out user joins", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL"], + "timedOut": true + }, + "expect": null + }, + { + "name": "join an announcement thread", + "fn": "canJoinThread", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "thread": { + "type": 10 + }, + "expect": null + }, + { + "name": "member adds to a public thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": null + }, + { + "name": "add without send in threads", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES"] + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "add to an archived thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "archived": true + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "add to a locked thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "locked": true + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": "THREAD_LOCKED" + }, + { + "name": "add to an invitable private thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "type": 12, + "invitable": true + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": null + }, + { + "name": "add to a non-invitable private thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "type": 12, + "invitable": false + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "moderator adds to a non-invitable private thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "type": 12, + "invitable": false + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": null + }, + { + "name": "add a moderator to a non-invitable private thread", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "type": 12, + "invitable": false + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": true + } + }, + "expect": null + }, + { + "name": "add a target without parent view", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "args": { + "target": { + "canViewParent": false, + "isModerator": false + } + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "timed out add", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "timedOut": true + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "add to an announcement thread ignores invitable", + "fn": "canAddThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "thread": { + "type": 10, + "invitable": false + }, + "args": { + "target": { + "canViewParent": true, + "isModerator": false + } + }, + "expect": null + }, + { + "name": "moderator removes", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "expect": null + }, + { + "name": "private thread owner removes", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "type": 12 + }, + "expect": null + }, + { + "name": "public thread owner cannot remove", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "remove from an archived thread", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "thread": { + "archived": true + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "timed out moderator removes", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"], + "timedOut": true + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "announcement thread owner cannot remove", + "fn": "canRemoveThreadMember", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "thread": { + "type": 10 + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "member leaves", + "fn": "canLeave", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "expect": null + }, + { + "name": "non-member leaves", + "fn": "canLeave", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"] + }, + "expect": "UNKNOWN_THREAD_MEMBER" + }, + { + "name": "leave an archived thread", + "fn": "canLeave", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true + }, + "thread": { + "archived": true + }, + "expect": "THREAD_ARCHIVED" + }, + { + "name": "timed out member leaves", + "fn": "canLeave", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isMember": true, + "timedOut": true + }, + "expect": null + }, + { + "name": "moderator deletes", + "fn": "canDeleteThread", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "expect": null + }, + { + "name": "thread owner cannot delete", + "fn": "canDeleteThread", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY"], + "isThreadOwner": true + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "timed out moderator deletes", + "fn": "canDeleteThread", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES_IN_THREADS", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"], + "timedOut": true + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "administrator deletes", + "fn": "canDeleteThread", + "actor": { + "perms": ["ADMINISTRATOR"] + }, + "expect": null + }, + { + "name": "create from a message", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS", "READ_MESSAGE_HISTORY"] + }, + "args": { + "kind": "from_message" + }, + "expect": null + }, + { + "name": "create from a message without history", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS"] + }, + "args": { + "kind": "from_message" + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "create a public thread", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS"] + }, + "args": { + "kind": "public" + }, + "expect": null + }, + { + "name": "create a public thread without the bit", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES"] + }, + "args": { + "kind": "public" + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "create a private thread", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PRIVATE_THREADS"] + }, + "args": { + "kind": "private" + }, + "expect": null + }, + { + "name": "create a private thread with only the public bit", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS"] + }, + "args": { + "kind": "private" + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "forum post with send messages", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "SEND_MESSAGES"] + }, + "args": { + "kind": "forum_post" + }, + "expect": null + }, + { + "name": "forum post ignores create public threads", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS", "SEND_MESSAGES_IN_THREADS"] + }, + "args": { + "kind": "forum_post" + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "timed out create", + "fn": "canCreateThread", + "actor": { + "perms": ["VIEW_CHANNEL", "CREATE_PUBLIC_THREADS"], + "timedOut": true + }, + "args": { + "kind": "public" + }, + "expect": "COMMUNICATION_DISABLED" + }, + { + "name": "create without parent view", + "fn": "canCreateThread", + "actor": { + "perms": ["CREATE_PUBLIC_THREADS"] + }, + "args": { + "kind": "public" + }, + "expect": "MISSING_ACCESS" + }, + { + "name": "administrator creates a private thread", + "fn": "canCreateThread", + "actor": { + "perms": ["ADMINISTRATOR"] + }, + "args": { + "kind": "private" + }, + "expect": null + }, + { + "name": "public archived list", + "fn": "canListArchivedThreads", + "actor": { + "perms": ["VIEW_CHANNEL", "READ_MESSAGE_HISTORY"] + }, + "args": { + "privateThreads": false + }, + "expect": null + }, + { + "name": "public archived list without history", + "fn": "canListArchivedThreads", + "actor": { + "perms": ["VIEW_CHANNEL"] + }, + "args": { + "privateThreads": false + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "private archived list for members", + "fn": "canListArchivedThreads", + "actor": { + "perms": ["VIEW_CHANNEL", "READ_MESSAGE_HISTORY"] + }, + "args": { + "privateThreads": true + }, + "expect": "MISSING_PERMISSIONS" + }, + { + "name": "private archived list for moderators", + "fn": "canListArchivedThreads", + "actor": { + "perms": ["VIEW_CHANNEL", "READ_MESSAGE_HISTORY", "MANAGE_THREADS"] + }, + "args": { + "privateThreads": true + }, + "expect": null + } +] diff --git a/fluxer_messages/src/router_impl.rs b/fluxer_messages/src/router_impl.rs index 2153f33cc..04adedc9f 100644 --- a/fluxer_messages/src/router_impl.rs +++ b/fluxer_messages/src/router_impl.rs @@ -75,9 +75,10 @@ impl RouterService for MessagesRouter { nonce, tts, include_hidden, + threads_mask, .. } => Some(format!( - "api-get:{channel_id}:{message_id}:{viewer_user_id}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{nonce:?}:{tts:?}:{include_hidden}" + "api-get:{channel_id}:{message_id}:{viewer_user_id}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{nonce:?}:{tts:?}:{include_hidden}:{threads_mask}" )), MessageRequest::BuildResponse { .. } => None, MessageRequest::BuildResponses { .. } => None, @@ -94,9 +95,11 @@ impl RouterService for MessagesRouter { media_endpoint, include_reactions, include_hidden, + threads_mask, + exclude_types, .. } => Some(format!( - "api-list:{channel_id}:{viewer_user_id}:{limit}:{before_id:?}:{after_id:?}:{around_id:?}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{include_hidden}" + "api-list:{channel_id}:{viewer_user_id}:{limit}:{before_id:?}:{after_id:?}:{around_id:?}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{include_hidden}:{threads_mask}:{exclude_types:?}" )), MessageRequest::ExtractMentions { .. } => None, } @@ -123,4 +126,58 @@ mod tests { MessagesRouter::coalesce_key(&before) ); } + + fn list_request(threads_mask: bool, exclude_types: Vec) -> MessageRequest { + MessageRequest::ListResponses { + channel_id: "42".to_owned(), + viewer_user_id: "7".to_owned(), + limit: 50, + before_id: None, + after_id: None, + around_id: None, + source_guild_id: Some("9".to_owned()), + message_history_cutoff_ms: None, + can_read_message_history: true, + media_endpoint: "https://media.test".to_owned(), + media_proxy_secret_key: "secret".to_owned(), + attachment_url_secret_base64: None, + include_reactions: None, + include_hidden: false, + threads_mask, + exclude_types, + } + } + + fn get_request(threads_mask: bool) -> MessageRequest { + MessageRequest::GetResponseById { + channel_id: "42".to_owned(), + message_id: "43".to_owned(), + viewer_user_id: "7".to_owned(), + source_guild_id: Some("9".to_owned()), + message_history_cutoff_ms: None, + can_read_message_history: true, + media_endpoint: "https://media.test".to_owned(), + media_proxy_secret_key: "secret".to_owned(), + attachment_url_secret_base64: None, + include_reactions: None, + nonce: None, + tts: None, + include_hidden: false, + threads_mask, + } + } + + #[test] + fn coalesce_key_separates_thread_masked_and_type_excluded_lists() { + let control = MessagesRouter::coalesce_key(&list_request(false, Vec::new())); + let masked = MessagesRouter::coalesce_key(&list_request(true, Vec::new())); + let excluded = MessagesRouter::coalesce_key(&list_request(false, vec![18])); + assert_ne!(control, masked); + assert_ne!(control, excluded); + assert_ne!(masked, excluded); + assert_ne!( + MessagesRouter::coalesce_key(&get_request(false)), + MessagesRouter::coalesce_key(&get_request(true)) + ); + } } diff --git a/fluxer_messages/src/shard_impl.rs b/fluxer_messages/src/shard_impl.rs index f47c16244..5afb2345e 100644 --- a/fluxer_messages/src/shard_impl.rs +++ b/fluxer_messages/src/shard_impl.rs @@ -89,6 +89,9 @@ fn copy_source(message: &Message) -> Option<(i64, i64)> { } const MESSAGE_FLAG_SUPPRESS_EMBEDS: i64 = 1 << 2; +const MESSAGE_TYPE_THREAD_CREATED: i32 = 18; +const THREAD_MESSAGE_FLAG_MASK: i64 = 0x120; +const THREAD_FEATURE_CHANNEL_TYPES: [i32; 5] = [10, 11, 12, 15, 16]; const EMBED_MEDIA_OWNED_ATTACHMENT_FLAG: i32 = 1 << 30; #[cfg(test)] const USER_FLAG_DELETED: i64 = 1_i64 << 34; @@ -103,6 +106,8 @@ const ENRICHMENT_QUERY_CONCURRENCY: usize = 16; const REACTION_MESSAGE_BATCH_SIZE: usize = 64; const ATTACHMENT_DECAY_BATCH_SIZE: usize = 128; const BUCKET_INDEX_PAGE_SIZE: u32 = 200; +const EXCLUDED_TYPE_REFILL_PAGE_SIZE: u32 = 100; +const EXCLUDED_TYPE_SCAN_ROW_CAP: usize = 5_000; const JS_MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; fn assert_safe_byte_size(value: i64) -> i64 { @@ -397,6 +402,22 @@ struct ResponseBuildOptions { nonce: Option, tts: bool, include_hidden: bool, + threads_mask: bool, + excluded_types: Vec, +} + +impl ResponseBuildOptions { + fn hides(&self, message: &Message) -> bool { + self.threads_mask && message.message_type == MESSAGE_TYPE_THREAD_CREATED + } + + fn mask_flags(&self, flags: i64) -> i64 { + if self.threads_mask { + flags & !THREAD_MESSAGE_FLAG_MASK + } else { + flags + } + } } #[derive(Debug, Default)] @@ -697,11 +718,51 @@ impl MessagesShard { channel_id: i64, cursor: PageCursor, limit: u32, + floor: Option, + excluded_types: &[i32], ) -> anyhow::Result> { + if excluded_types.is_empty() { + return match cursor { + PageCursor::Latest => self.get_latest(channel_id, limit).await, + PageCursor::Before(before_id) => { + self.get_before(channel_id, before_id, limit).await + } + PageCursor::After(after_id) => self.get_after(channel_id, after_id, limit).await, + }; + } match cursor { - PageCursor::Latest => self.get_latest(channel_id, limit).await, - PageCursor::Before(before_id) => self.get_before(channel_id, before_id, limit).await, - PageCursor::After(after_id) => self.get_after(channel_id, after_id, limit).await, + PageCursor::Latest => { + collect_excluding(limit, excluded_types, None, |cursor, size| async move { + match cursor { + Some(cursor) => self.get_before(channel_id, cursor, size).await, + None => self.get_latest(channel_id, size).await, + } + }) + .await + } + PageCursor::Before(before_id) => { + collect_excluding( + limit, + excluded_types, + Some(before_id), + |cursor, size| async move { + let mut page = self + .get_before(channel_id, cursor.unwrap_or(before_id), size) + .await?; + if let Some(floor) = floor { + page.retain(|message| message.message_id > floor); + } + Ok(page) + }, + ) + .await + } + PageCursor::After(after_id) => { + collect_excluding(limit, excluded_types, Some(after_id), |cursor, size| { + self.get_after(channel_id, cursor.unwrap_or(after_id), size) + }) + .await + } } } @@ -721,7 +782,15 @@ impl MessagesShard { if limit == 0 { break; } - let page = self.fetch_page(channel_id, cursor, page_size).await?; + let page = self + .fetch_page( + channel_id, + cursor, + page_size, + floor, + &options.excluded_types, + ) + .await?; let exhausted = page.len() < page_size as usize; let reached_floor = floor.is_some_and(|floor| page.iter().any(|message| message.message_id <= floor)); @@ -776,7 +845,9 @@ impl MessagesShard { let hidden = self .hidden_message_ids(std::slice::from_ref(&target), options, &mut target_users) .await; - Ok((!hidden.contains(&target.message_id)).then_some(target)) + Ok((!hidden.contains(&target.message_id) + && !options.excluded_types.contains(&target.message_type)) + .then_some(target)) }, self.visible_page( channel_id, @@ -885,6 +956,9 @@ impl MessagesShard { let Some(message) = self.get_by_id(channel_id, message_id).await? else { return Ok(None); }; + if options.hides(&message) { + return Ok(None); + } if message.author_id.is_none() && message.webhook_id.is_none() { self.cleanup_orphaned_messages(vec![message]).await; return Ok(None); @@ -909,7 +983,9 @@ impl MessagesShard { message: Message, options: ResponseBuildOptions, ) -> anyhow::Result> { - if !self.is_message_visible_to_requester(message.message_id, &options) { + if !self.is_message_visible_to_requester(message.message_id, &options) + || options.hides(&message) + { return Ok(None); } if message.author_id.is_none() && message.webhook_id.is_none() { @@ -936,7 +1012,10 @@ impl MessagesShard { ) -> anyhow::Result> { let (messages, orphaned_messages): (Vec<_>, Vec<_>) = messages .into_iter() - .filter(|message| self.is_message_visible_to_requester(message.message_id, &options)) + .filter(|message| { + self.is_message_visible_to_requester(message.message_id, &options) + && !options.hides(message) + }) .partition(|message| message.author_id.is_some() || message.webhook_id.is_some()); self.cleanup_orphaned_messages(orphaned_messages).await; let mut users = UserLookup::default(); @@ -1279,10 +1358,13 @@ impl MessagesShard { channel_ids.into_iter().map(|id| id.to_string()).collect(); channel_ids.sort(); channel_ids.dedup(); - let payload = serde_json::json!({ + let mut payload = serde_json::json!({ "guild_id": guild_id.to_string(), "channel_ids": channel_ids, }); + if options.threads_mask { + payload["exclude_thread_channels"] = serde_json::Value::Bool(true); + } let payload_bytes = serde_json::to_vec(&payload).unwrap_or_default(); let response = self .transport @@ -1308,6 +1390,10 @@ impl MessagesShard { .and_then(|result| result.channels) .unwrap_or_default() .into_iter() + .filter(|channel| { + !(options.threads_mask + && THREAD_FEATURE_CHANNEL_TYPES.contains(&channel.channel_type)) + }) .map(|channel| { let item = ApiChannelMentionResponse { id: channel.id, @@ -1397,6 +1483,7 @@ impl MessagesShard { context .referenced_messages .get(&(channel_id, message_id)) + .filter(|referenced| !options.hides(referenced)) .map(|referenced| { let mut referenced_options = options.clone(); referenced_options.nonce = None; @@ -1418,7 +1505,7 @@ impl MessagesShard { author, webhook_id: message.webhook_id.map(|id| id.to_string()), message_type: message.message_type, - flags: message.flags.unwrap_or_default(), + flags: options.mask_flags(message.flags.unwrap_or_default()), content: message.content.clone().unwrap_or_default(), timestamp: epoch_millis_to_iso(snowflake_to_epoch_millis(message.message_id)), edited_timestamp: message.edited_timestamp.map(epoch_millis_to_iso), @@ -1762,7 +1849,7 @@ impl MessagesShard { .as_ref() .map(|stickers| stickers.iter().filter_map(map_sticker).collect::>()), snapshot_type: snapshot.snapshot_type.unwrap_or_default(), - flags: snapshot.flags.unwrap_or_default(), + flags: options.mask_flags(i64::from(snapshot.flags.unwrap_or_default())) as i32, } } } @@ -2499,6 +2586,7 @@ impl ShardService for MessagesShard { nonce, tts, include_hidden, + threads_mask, } => { let channel_id = parse_i64(&channel_id, "channel_id")?; let message_id = parse_i64(&message_id, "message_id")?; @@ -2525,6 +2613,8 @@ impl ShardService for MessagesShard { nonce, tts: tts.unwrap_or(false), include_hidden, + threads_mask, + excluded_types: Vec::new(), }, ) .await?; @@ -2546,6 +2636,7 @@ impl ShardService for MessagesShard { nonce, tts, include_hidden, + threads_mask, } => { let viewer_user_id = parse_i64(&viewer_user_id, "viewer_user_id")?; let source_guild_id = source_guild_id @@ -2569,6 +2660,8 @@ impl ShardService for MessagesShard { nonce, tts: tts.unwrap_or(false), include_hidden, + threads_mask, + excluded_types: Vec::new(), }, ) .await?; @@ -2588,6 +2681,7 @@ impl ShardService for MessagesShard { attachment_url_secret_base64, include_reactions, include_hidden, + threads_mask, } => { let viewer_user_id = parse_i64(&viewer_user_id, "viewer_user_id")?; let source_guild_id = source_guild_id @@ -2611,6 +2705,8 @@ impl ShardService for MessagesShard { nonce: None, tts: false, include_hidden, + threads_mask, + excluded_types: Vec::new(), }, ) .await?; @@ -2631,8 +2727,13 @@ impl ShardService for MessagesShard { attachment_url_secret_base64, include_reactions, include_hidden, + threads_mask, + mut exclude_types, } => { let channel_id = parse_i64(&channel_id, "channel_id")?; + if threads_mask && !exclude_types.contains(&MESSAGE_TYPE_THREAD_CREATED) { + exclude_types.push(MESSAGE_TYPE_THREAD_CREATED); + } let viewer_user_id = parse_i64(&viewer_user_id, "viewer_user_id")?; let before_id = before_id .as_deref() @@ -2671,6 +2772,8 @@ impl ShardService for MessagesShard { nonce: None, tts: false, include_hidden, + threads_mask, + excluded_types: exclude_types, }, ) .await?; @@ -2809,6 +2912,40 @@ where Ok(()) } +async fn collect_excluding( + limit: u32, + excluded_types: &[i32], + mut cursor: Option, + fetch: Fetch, +) -> anyhow::Result> +where + Fetch: Fn(Option, u32) -> Fut, + Fut: std::future::Future>>, +{ + let mut collected = Vec::new(); + let mut scanned = 0_usize; + let mut page_size = limit; + while collected.len() < limit as usize && scanned < EXCLUDED_TYPE_SCAN_ROW_CAP { + let page = fetch(cursor, page_size).await?; + let exhausted = page.len() < page_size as usize; + let Some(last) = page.last() else { + break; + }; + cursor = Some(last.message_id); + scanned += page.len(); + collected.extend( + page.into_iter() + .filter(|message| !excluded_types.contains(&message.message_type)), + ); + if exhausted { + break; + } + page_size = limit.max(EXCLUDED_TYPE_REFILL_PAGE_SIZE); + } + collected.truncate(limit as usize); + Ok(collected) +} + fn around_window_limits(limit: u32) -> (u32, u32) { let newer_limit = limit / 2; let older_limit = limit.saturating_sub(1).saturating_sub(newer_limit); @@ -4110,6 +4247,8 @@ mod tests { nonce: None, tts: false, include_hidden: false, + threads_mask: false, + excluded_types: Vec::new(), } } @@ -4916,6 +5055,522 @@ mod tests { ); } + fn typed_message(message_id: i64, message_type: i32, flags: i64) -> Message { + serde_json::from_value(json!({ + "message_id": message_id.to_string(), + "channel_id": "10", + "bucket": 416, + "author_id": "1472426752046002208", + "type": message_type, + "flags": flags, + "version": 1 + })) + .unwrap() + } + + fn masked_options() -> ResponseBuildOptions { + ResponseBuildOptions { + threads_mask: true, + ..build_options() + } + } + + #[tokio::test] + async fn excluded_types_refill_the_page_past_hidden_rows() { + let channel = [ + (100, 18), + (99, 0), + (98, 18), + (97, 18), + (96, 0), + (95, 0), + (94, 0), + ] + .map(|(id, message_type)| typed_message(id, message_type, 0)); + let fetches = std::sync::Mutex::new(Vec::new()); + + let collected = collect_excluding(3, &[18], None, |cursor, size| { + let fetches = &fetches; + let channel = &channel; + async move { + fetches.lock().unwrap().push((cursor, size)); + Ok(channel + .iter() + .filter(|message| cursor.is_none_or(|cursor| message.message_id < cursor)) + .take(size as usize) + .cloned() + .collect()) + } + }) + .await + .unwrap(); + + assert_eq!( + collected.iter().map(|m| m.message_id).collect::>(), + vec![99, 96, 95] + ); + assert_eq!( + *fetches.lock().unwrap(), + vec![(None, 3), (Some(98), EXCLUDED_TYPE_REFILL_PAGE_SIZE)] + ); + } + + #[tokio::test] + async fn excluded_types_stop_at_the_end_of_the_channel() { + let channel = [(100, 18), (99, 0), (98, 18)] + .map(|(id, message_type)| typed_message(id, message_type, 0)); + let fetches = std::sync::Mutex::new(0_usize); + + let collected = collect_excluding(2, &[18], None, |cursor, size| { + let fetches = &fetches; + let channel = &channel; + async move { + *fetches.lock().unwrap() += 1; + Ok(channel + .iter() + .filter(|message| cursor.is_none_or(|cursor| message.message_id < cursor)) + .take(size as usize) + .cloned() + .collect()) + } + }) + .await + .unwrap(); + + assert_eq!( + collected.iter().map(|m| m.message_id).collect::>(), + vec![99] + ); + assert_eq!(*fetches.lock().unwrap(), 2); + } + + #[tokio::test] + async fn excluded_types_reach_past_a_long_hidden_run() { + let channel = (0..3_000_i64) + .map(|offset| typed_message(10_000 - offset, 18, 0)) + .chain([typed_message(6_999, 0, 0), typed_message(6_998, 0, 0)]) + .collect::>(); + + let collected = collect_excluding(2, &[18], None, |cursor, size| { + let channel = &channel; + async move { + Ok(channel + .iter() + .filter(|message| cursor.is_none_or(|cursor| message.message_id < cursor)) + .take(size as usize) + .cloned() + .collect()) + } + }) + .await + .unwrap(); + + assert_eq!( + collected.iter().map(|m| m.message_id).collect::>(), + vec![6_999, 6_998] + ); + } + + #[tokio::test] + async fn excluded_types_stop_at_the_scan_row_cap() { + let scanned = std::sync::Mutex::new(0_usize); + + let collected = collect_excluding(2, &[18], None, |cursor, size| { + let scanned = &scanned; + async move { + *scanned.lock().unwrap() += size as usize; + let top = cursor.unwrap_or(1_000_000); + Ok((1..=i64::from(size)) + .map(|offset| typed_message(top - offset, 18, 0)) + .collect()) + } + }) + .await + .unwrap(); + + assert!(collected.is_empty()); + let scanned = *scanned.lock().unwrap(); + assert!(scanned >= EXCLUDED_TYPE_SCAN_ROW_CAP); + assert!(scanned < EXCLUDED_TYPE_SCAN_ROW_CAP + EXCLUDED_TYPE_REFILL_PAGE_SIZE as usize); + } + + const LISTED_BASE_ID: i64 = 1_509_197_195_776_110_592; + + async fn listed_ids( + types: &[i32], + limit: u32, + before: Option, + after: Option, + around: Option, + excluded_types: &[i32], + ) -> Vec { + let rows = types + .iter() + .enumerate() + .map(|(index, message_type)| { + typed_message(LISTED_BASE_ID + index as i64 + 1, *message_type, 0) + }) + .collect::>(); + let shard = MessagesShard { + storage: MessagesStorage::Memory(std::sync::Arc::new(rows)), + transport: InMemoryTransport::new(), + }; + let users = stub_user_service(&shard.transport).await; + let at = |offset: i64| LISTED_BASE_ID + offset; + let responses = shard + .list_api_responses( + 10, + limit, + before.map(at), + after.map(at), + around.map(at), + ResponseBuildOptions { + excluded_types: excluded_types.to_vec(), + ..build_options() + }, + ) + .await + .unwrap(); + users.abort(); + responses + .iter() + .map(|response| response.id.parse::().unwrap() - LISTED_BASE_ID) + .collect() + } + + #[tokio::test] + async fn excluded_types_list_the_latest_page_past_hidden_rows() { + let types = [0, 0, 18, 0, 18, 18, 0, 18]; + + assert_eq!( + listed_ids(&types, 3, None, None, None, &[18]).await, + [7, 4, 2] + ); + assert_eq!( + listed_ids(&types, 3, None, None, None, &[]).await, + [8, 7, 6] + ); + } + + #[tokio::test] + async fn excluded_types_list_after_an_id_in_ascending_pages() { + let types = [18, 18, 18, 0, 18, 0, 0, 0]; + + assert_eq!( + listed_ids(&types, 3, None, Some(0), None, &[18]).await, + [7, 6, 4] + ); + assert_eq!( + listed_ids(&types, 3, None, Some(0), None, &[]).await, + [3, 2, 1] + ); + } + + #[tokio::test] + async fn excluded_types_keep_a_before_and_after_window_inside_its_bounds() { + let types = [0, 0, 0, 18, 0, 18, 18, 0]; + + assert_eq!( + listed_ids(&types, 3, Some(9), Some(1), None, &[18]).await, + [8, 5, 3] + ); + assert_eq!( + listed_ids(&types, 3, Some(9), Some(4), None, &[18]).await, + [8, 5] + ); + assert_eq!( + listed_ids(&types, 3, Some(9), Some(1), None, &[]).await, + [8, 7, 6] + ); + } + + #[tokio::test] + async fn excluded_types_drop_a_hidden_around_target_and_refill_both_sides() { + let types = [0, 0, 18, 0, 18, 0, 18, 0, 0]; + + assert_eq!( + listed_ids(&types, 5, None, None, Some(5), &[18]).await, + [8, 6, 4, 2] + ); + assert_eq!( + listed_ids(&types, 5, None, None, Some(5), &[]).await, + [7, 6, 5, 4, 3] + ); + } + + #[tokio::test] + async fn threads_mask_drops_thread_created_rows_and_clears_thread_flags() { + let deleted = DeletedMessageKeys::default(); + let shard = recording_shard(&deleted); + let users = stub_user_service(&shard.transport).await; + let mut forwarded = typed_message(1_509_197_195_776_110_594, 0, 0x120 | 0x4); + forwarded.message_snapshots = Some(vec![ + serde_json::from_value(json!({"content": "x", "type": 0, "flags": 0x124})).unwrap(), + ]); + let messages = vec![ + forwarded, + typed_message(1_509_197_195_776_110_593, MESSAGE_TYPE_THREAD_CREATED, 0), + ]; + + let masked = shard + .build_api_responses_from_messages(messages.clone(), masked_options()) + .await + .unwrap(); + let control = shard + .build_api_responses_from_messages(messages, build_options()) + .await + .unwrap(); + users.abort(); + + assert_eq!(masked.len(), 1); + assert_eq!(masked[0].flags, 0x4); + assert_eq!(masked[0].message_snapshots.as_ref().unwrap()[0].flags, 0x4); + assert_eq!(control.len(), 2); + assert_eq!(control[0].flags, 0x124); + assert_eq!( + control[0].message_snapshots.as_ref().unwrap()[0].flags, + 0x124 + ); + assert_eq!(control[1].message_type, MESSAGE_TYPE_THREAD_CREATED); + assert!(recorded_deletions(&deleted).is_empty()); + } + + #[tokio::test] + async fn threads_mask_drops_a_single_thread_created_row() { + let deleted = DeletedMessageKeys::default(); + let shard = recording_shard(&deleted); + let users = stub_user_service(&shard.transport).await; + let starter = typed_message(1_509_197_195_776_110_593, MESSAGE_TYPE_THREAD_CREATED, 0); + + let masked = shard + .build_api_response_from_message(starter.clone(), masked_options()) + .await + .unwrap(); + let control = shard + .build_api_response_from_message(starter, build_options()) + .await + .unwrap(); + users.abort(); + + assert!(masked.is_none()); + assert!(control.is_some()); + } + + #[test] + fn threads_mask_clears_nested_referenced_flags_and_nulls_a_thread_created_reference() { + let shard = recording_shard(&DeletedMessageKeys::default()); + let reply_to = |target: i64| { + let mut reply = typed_message(1_509_197_195_776_110_600, 19, 0x20); + reply.message_reference = Some(MessageReference { + channel_id: Some(10), + message_id: Some(target), + guild_id: None, + reference_type: Some(0), + }); + reply + }; + let referenced = typed_message(1_509_197_195_776_110_590, 0, 0x120); + let thread_created = + typed_message(1_509_197_195_776_110_591, MESSAGE_TYPE_THREAD_CREATED, 0); + let context = ResponseContext { + referenced_messages: HashMap::from([ + ((10, referenced.message_id), referenced.clone()), + ((10, thread_created.message_id), thread_created.clone()), + ]), + ..ResponseContext::default() + }; + + let masked = shard.map_message_response( + &reply_to(referenced.message_id), + &masked_options(), + &context, + true, + ); + assert_eq!(masked.flags, 0); + let nested = masked.referenced_message.unwrap().unwrap(); + assert_eq!(nested.flags, 0); + + let control = shard.map_message_response( + &reply_to(referenced.message_id), + &build_options(), + &context, + true, + ); + assert_eq!(control.flags, 0x20); + assert_eq!(control.referenced_message.unwrap().unwrap().flags, 0x120); + + let masked_starter = shard.map_message_response( + &reply_to(thread_created.message_id), + &masked_options(), + &context, + true, + ); + assert!(matches!(masked_starter.referenced_message, Some(None))); + let control_starter = shard.map_message_response( + &reply_to(thread_created.message_id), + &build_options(), + &context, + true, + ); + assert!(matches!(control_starter.referenced_message, Some(Some(_)))); + } + + #[test] + fn thread_options_default_to_off_for_older_callers() { + let list: MessageRequest = serde_json::from_value(json!({ + "op": "ListResponses", + "channel_id": "10", + "viewer_user_id": "1", + "limit": 50, + "can_read_message_history": true, + "media_endpoint": "https://media.test", + "media_proxy_secret_key": "secret", + })) + .unwrap(); + match list { + MessageRequest::ListResponses { + threads_mask, + exclude_types, + .. + } => { + assert!(!threads_mask); + assert!(exclude_types.is_empty()); + } + other => panic!("unexpected request {other:?}"), + } + let get: MessageRequest = serde_json::from_value(json!({ + "op": "GetResponseById", + "channel_id": "10", + "message_id": "11", + "viewer_user_id": "1", + "can_read_message_history": true, + "media_endpoint": "https://media.test", + "media_proxy_secret_key": "secret", + })) + .unwrap(); + match get { + MessageRequest::GetResponseById { threads_mask, .. } => assert!(!threads_mask), + other => panic!("unexpected request {other:?}"), + } + let options = build_options(); + assert_eq!(options.mask_flags(0x1ff), 0x1ff); + assert!(!options.hides(&typed_message(1, MESSAGE_TYPE_THREAD_CREATED, 0))); + } + + #[test] + fn a_reference_without_a_message_id_round_trips() { + let message: Message = serde_json::from_value(json!({ + "message_id": "1509197195776110592", + "channel_id": "10", + "bucket": 416, + "author_id": "1472426752046002208", + "type": MESSAGE_TYPE_THREAD_CREATED, + "version": 1, + "message_reference": {"channel_id": "1509197195776110500", "guild_id": "7", "type": 0} + })) + .unwrap(); + let reference = message.message_reference.as_ref().unwrap(); + assert_eq!(reference.message_id, None); + + let mapped = map_message_reference(reference).expect("a thread reference maps"); + assert_eq!(mapped.message_id, None); + let encoded = serde_json::to_value(&mapped).unwrap(); + assert_eq!( + encoded, + json!({"channel_id": "1509197195776110500", "guild_id": "7", "type": 0}) + ); + let decoded: ApiMessageReferenceResponse = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.message_id, None); + + let shard = recording_shard(&DeletedMessageKeys::default()); + let response = shard.map_message_response( + &message, + &build_options(), + &ResponseContext::default(), + true, + ); + assert!(response.referenced_message.is_none()); + let packed = rmp_serde::to_vec_named(&response).unwrap(); + let unpacked: ApiMessageResponse = rmp_serde::from_slice(&packed).unwrap(); + let unpacked_reference = unpacked.message_reference.unwrap(); + assert_eq!(unpacked_reference.message_id, None); + assert_eq!(unpacked_reference.channel_id, "1509197195776110500"); + + let reply = MessageReference { + channel_id: Some(1), + message_id: Some(2), + guild_id: None, + reference_type: None, + }; + assert_eq!( + serde_json::to_value(map_message_reference(&reply).unwrap()).unwrap(), + json!({"channel_id": "1", "message_id": "2", "type": 0}) + ); + } + + async fn stub_channel_mentions( + transport: &InMemoryTransport, + payloads: std::sync::Arc>>, + ) -> tokio::task::JoinHandle<()> { + let mut subscriber = transport + .subscribe("rpc.gateway.guild.resolve_channel_mentions") + .await + .unwrap(); + let transport = transport.clone(); + tokio::spawn(async move { + while let Some(message) = subscriber.next().await { + payloads + .lock() + .unwrap() + .push(serde_json::from_slice(TransportMessage::payload(&message)).unwrap()); + let reply = json!({"ok": true, "result": {"channels": [ + {"id": "20", "name": "general", "type": 0}, + {"id": "21", "name": "thread", "type": 11}, + {"id": "22", "name": "forum", "type": 15}, + {"id": "23", "name": "news-thread", "type": 10} + ]}}); + let _ = reply_message(&message, &transport, reply.to_string().as_bytes()).await; + } + }) + } + + #[tokio::test] + async fn threads_mask_excludes_thread_channels_from_mention_resolution() { + let shard = recording_shard(&DeletedMessageKeys::default()); + let payloads = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let stub = stub_channel_mentions(&shard.transport, payloads.clone()).await; + let ids = HashSet::from([20, 21, 22, 23]); + + let masked = shard + .resolve_channel_mentions( + ids.clone(), + &ResponseBuildOptions { + source_guild_id: Some(7), + ..masked_options() + }, + ) + .await; + let control = shard + .resolve_channel_mentions( + ids, + &ResponseBuildOptions { + source_guild_id: Some(7), + ..build_options() + }, + ) + .await; + stub.abort(); + + let mut masked_ids = masked.into_keys().collect::>(); + masked_ids.sort(); + assert_eq!(masked_ids, vec!["20"]); + assert_eq!(control.len(), 4); + assert_eq!(control["23"].channel_type, 10); + let payloads = payloads.lock().unwrap(); + assert_eq!(payloads[0]["exclude_thread_channels"], json!(true)); + assert!(payloads[1].get("exclude_thread_channels").is_none()); + } + const CROSSPOST_SOURCE_CHANNEL_ID: i64 = 500; const CROSSPOST_SOURCE_GUILD_ID: i64 = 600; const CROSSPOST_SOURCE_MESSAGE_ID: i64 = 1_509_197_195_776_110_590; diff --git a/fluxer_messages/src/types.rs b/fluxer_messages/src/types.rs index 2ecbe6051..2c06c5e4e 100644 --- a/fluxer_messages/src/types.rs +++ b/fluxer_messages/src/types.rs @@ -105,6 +105,8 @@ pub enum MessageRequest { tts: Option, #[serde(default)] include_hidden: bool, + #[serde(default)] + threads_mask: bool, }, BuildResponse { message: Message, @@ -121,6 +123,8 @@ pub enum MessageRequest { tts: Option, #[serde(default)] include_hidden: bool, + #[serde(default)] + threads_mask: bool, }, BuildResponses { messages: Vec, @@ -135,6 +139,8 @@ pub enum MessageRequest { include_reactions: Option, #[serde(default)] include_hidden: bool, + #[serde(default)] + threads_mask: bool, }, ListResponses { channel_id: String, @@ -153,6 +159,10 @@ pub enum MessageRequest { include_reactions: Option, #[serde(default)] include_hidden: bool, + #[serde(default)] + threads_mask: bool, + #[serde(default)] + exclude_types: Vec, }, ExtractMentions { contents: Vec, diff --git a/fluxer_push/src/delivery.rs b/fluxer_push/src/delivery.rs index 4cfb03ed3..8fdcf886b 100644 --- a/fluxer_push/src/delivery.rs +++ b/fluxer_push/src/delivery.rs @@ -53,6 +53,7 @@ impl Job { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum Audience { Standard, + Thread, Ring, } @@ -60,6 +61,7 @@ impl Audience { fn admits(self, subscription: &Subscription) -> bool { match self { Self::Standard => subscription.platform() != Some(Platform::IosApnsVoip), + Self::Thread => Self::Standard.admits(subscription) && subscription.thread_channels, Self::Ring => Self::rings(subscription), } } @@ -377,7 +379,11 @@ async fn run_message_job( &subscriptions, envelopes, deadline, - Audience::Standard, + if job.is_thread() { + Audience::Thread + } else { + Audience::Standard + }, ) .await; let duration_ms = elapsed_ms(started_ms); @@ -672,3 +678,34 @@ fn reap(result: Result<(), tokio::task::JoinError>) { warn!(error = %error, "push job task failed"); } } + +#[cfg(test)] +mod tests { + use super::*; + + fn subscription(platform: &str, thread_channels: bool) -> Subscription { + Subscription { + subscription_id: "sub".to_owned(), + endpoint: "token".to_owned(), + p256dh_key: None, + auth_key: None, + platform: Some(platform.to_owned()), + app_id: None, + provider_environment: None, + thread_channels, + } + } + + #[test] + fn a_thread_push_reaches_only_devices_that_registered_the_capability() { + assert!(Audience::Thread.admits(&subscription("android_fcm", true))); + assert!(!Audience::Thread.admits(&subscription("android_fcm", false))); + assert!(!Audience::Thread.admits(&subscription("ios_apns_voip", true))); + } + + #[test] + fn a_standard_push_ignores_the_capability() { + assert!(Audience::Standard.admits(&subscription("ios_apns", false))); + assert!(Audience::Standard.admits(&subscription("ios_apns", true))); + } +} diff --git a/fluxer_push/src/job.rs b/fluxer_push/src/job.rs index 36f2b826f..cdfb1b314 100644 --- a/fluxer_push/src/job.rs +++ b/fluxer_push/src/job.rs @@ -10,6 +10,8 @@ pub const QUEUE_GROUP: &str = "fluxer-push"; const SUPPORTED_VERSION: u8 = 1; const DIRECT_MESSAGE_GUILD_ID: &str = "0"; +const THREAD_KIND: &str = "thread"; +const FORUM_THREAD_CREATED_KIND: &str = "forum_thread_created"; #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct MessageJob { @@ -19,6 +21,14 @@ pub struct MessageJob { pub message_id: String, pub notification: NotificationFields, pub user_ids: Vec, + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub parent_id: Option, + #[serde(default)] + pub parent_name: Option, + #[serde(default)] + pub channel_name: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] @@ -72,6 +82,17 @@ impl MessageJob { pub fn is_direct_message(&self) -> bool { self.guild_id == DIRECT_MESSAGE_GUILD_ID } + + pub fn is_thread(&self) -> bool { + matches!( + self.kind.as_deref(), + Some(THREAD_KIND | FORUM_THREAD_CREATED_KIND) + ) + } + + pub fn is_forum_thread_created(&self) -> bool { + self.kind.as_deref() == Some(FORUM_THREAD_CREATED_KIND) + } } pub fn decode_message(bytes: &[u8]) -> Result { @@ -132,4 +153,66 @@ mod tests { "1" ); } + + const BASE: &str = r#"{"v":1,"config_version":3,"guild_id":"5","channel_id":"6","message_id":"7","notification":{"title":"t","body":"b","icon":"i","badge":"g","tag":"x","notification_tag":"y","url":"/u"},"user_ids":["8"]"#; + + #[test] + fn a_job_without_thread_fields_decodes_as_before() { + let job = decode_message(format!("{BASE}}}").as_bytes()).expect("the job decodes"); + assert!(!job.is_thread()); + assert_eq!(job.parent_id, None); + assert_eq!(job.parent_name, None); + assert_eq!(job.channel_name, None); + } + + #[test] + fn a_thread_job_decodes_its_optional_fields() { + let job = decode_message( + format!(r#"{BASE},"kind":"thread","parent_id":"9","parent_name":"general","channel_name":"plans"}}"#) + .as_bytes(), + ) + .expect("the job decodes"); + assert!(job.is_thread()); + assert_eq!(job.parent_id.as_deref(), Some("9")); + assert_eq!(job.parent_name.as_deref(), Some("general")); + assert_eq!(job.channel_name.as_deref(), Some("plans")); + } + + #[test] + fn a_gateway_thread_job_decodes_as_a_thread() { + let job = decode_message( + br#"{"v":1,"config_version":3,"guild_id":"5","channel_id":"6","message_id":"7","notification":{"title":"ada (#ideas, #general, Guild)","body":"hi","icon":"icon","badge":"badge","tag":"channel:6:7","notification_tag":"channel:6","url":"/channels/5/6/7","image_url":null},"user_ids":["8"],"kind":"thread","parent_id":"200","parent_name":"general","channel_name":"ideas"}"#, + ) + .expect("the job decodes"); + assert!(job.is_thread()); + assert_eq!(job.parent_id.as_deref(), Some("200")); + assert_eq!(job.parent_name.as_deref(), Some("general")); + assert_eq!(job.channel_name.as_deref(), Some("ideas")); + } + + #[test] + fn a_forum_thread_created_job_reaches_thread_capable_devices_only() { + let job = decode_message( + format!( + r#"{BASE},"kind":"forum_thread_created","parent_id":"9","channel_name":"post"}}"# + ) + .as_bytes(), + ) + .expect("the job decodes"); + assert!(job.is_thread()); + assert!(job.is_forum_thread_created()); + let thread = decode_message(format!(r#"{BASE},"kind":"thread"}}"#).as_bytes()) + .expect("the job decodes"); + assert!(!thread.is_forum_thread_created()); + } + + #[test] + fn thread_fields_nested_in_the_notification_do_not_mark_a_thread() { + let job = decode_message( + br#"{"v":1,"config_version":3,"guild_id":"5","channel_id":"6","message_id":"7","notification":{"title":"t","body":"b","icon":"i","badge":"g","tag":"x","notification_tag":"y","url":"/u","kind":"thread","parent_id":"200"},"user_ids":["8"]}"#, + ) + .expect("the job decodes"); + assert!(!job.is_thread()); + assert_eq!(job.parent_id, None); + } } diff --git a/fluxer_push/src/payload.rs b/fluxer_push/src/payload.rs index c21162743..41e98ace2 100644 --- a/fluxer_push/src/payload.rs +++ b/fluxer_push/src/payload.rs @@ -26,10 +26,17 @@ const RING_CALLER_NAME_KEY: &str = "caller_name"; const RING_CALLER_AVATAR_KEY: &str = "caller_avatar_url"; const RING_AVATAR_KEYS: [&str; 1] = [RING_CALLER_AVATAR_KEY]; const RING_IDENTITY_KEYS: [&str; 2] = [RING_CALLER_ID_KEY, RING_CALLER_NAME_KEY]; -const MINIMAL_DATA_KEYS: [&str; 9] = [ +const THREAD_DATA_KEYS: [&str; 3] = ["parent_id", "parent_name", "channel_name"]; +const NOTIF_TYPE_KEY: &str = "notif_type_id"; +const FORUM_THREAD_CREATED_NOTIF_TYPE: u64 = 9; +const FCM_DEFAULT_CHANNEL_ID: &str = "fluxer_default_push"; +const FCM_FORUM_THREAD_CREATED_CHANNEL_ID: &str = "fluxer_forum_thread_created"; +const MINIMAL_DATA_KEYS: [&str; 11] = [ "type", "action", "channel_id", + "parent_id", + NOTIF_TYPE_KEY, "message_id", "guild_id", "target_user_id", @@ -46,7 +53,7 @@ pub fn web_push_message(job: &MessageJob, target_user_id: &str, badge_count: u32 } else { Value::String(job.guild_id.clone()) }; - let data = json!({ + let mut data = json!({ "channel_id": job.channel_id, "author_avatar_url": fields.icon, "message_id": job.message_id, @@ -57,6 +64,7 @@ pub fn web_push_message(job: &MessageJob, target_user_id: &str, badge_count: u32 "target_user_id": target_user_id, "has_media": image_url.is_some(), }); + merge_thread_fields(&mut data, job); let notification = json!({ "title": fields.title, "body": fields.body, @@ -81,6 +89,28 @@ pub fn web_push_message(job: &MessageJob, target_user_id: &str, badge_count: u32 envelope } +fn merge_thread_fields(data: &mut Value, job: &MessageJob) { + let Some(object) = data.as_object_mut() else { + return; + }; + let fields = [ + job.parent_id.as_deref(), + job.parent_name.as_deref(), + job.channel_name.as_deref(), + ]; + for (key, value) in THREAD_DATA_KEYS.into_iter().zip(fields) { + if let Some(value) = value.filter(|value| !value.is_empty()) { + object.insert(key.to_owned(), value.into()); + } + } + if job.is_forum_thread_created() { + object.insert( + NOTIF_TYPE_KEY.to_owned(), + FORUM_THREAD_CREATED_NOTIF_TYPE.into(), + ); + } +} + pub fn web_push_clear(job: &ClearJob, badge_count: u32) -> Value { let tag = format!("channel:{}", job.channel_id); let data = json!({ @@ -200,7 +230,7 @@ fn fcm_notification_message(device_token: &str, envelope: &Value) -> Value { data.insert("image_url".to_owned(), url.into()); } let mut android_notification = json!({ - "channel_id": "fluxer_default_push", + "channel_id": fcm_channel_id(envelope), "tag": tag, "click_action": FCM_CLICK_ACTION, }); @@ -220,6 +250,18 @@ fn fcm_notification_message(device_token: &str, envelope: &Value) -> Value { }) } +fn fcm_channel_id(envelope: &Value) -> &'static str { + let notif_type = envelope + .get("data") + .and_then(|data| data.get(NOTIF_TYPE_KEY)) + .and_then(Value::as_u64); + if notif_type == Some(FORUM_THREAD_CREATED_NOTIF_TYPE) { + FCM_FORUM_THREAD_CREATED_CHANNEL_ID + } else { + FCM_DEFAULT_CHANNEL_ID + } +} + pub fn apns_payload(envelope: &Value) -> Value { let data = envelope.get("data"); let mut payload = data.and_then(Value::as_object).cloned().unwrap_or_default(); @@ -616,6 +658,21 @@ mod tests { image_url: image_url.map(str::to_owned), }, user_ids: vec![USER_ID.to_owned()], + kind: None, + parent_id: None, + parent_name: None, + channel_name: None, + } + } + + fn thread_job() -> MessageJob { + MessageJob { + guild_id: "4455".to_owned(), + kind: Some("thread".to_owned()), + parent_id: Some("7788".to_owned()), + parent_name: Some("general".to_owned()), + channel_name: Some("release plans".to_owned()), + ..message_job(None) } } @@ -669,6 +726,60 @@ mod tests { ); } + #[test] + fn a_message_outside_a_thread_carries_no_thread_keys() { + let envelope = web_push_message(&message_job(None), USER_ID, 3); + for key in THREAD_DATA_KEYS { + assert!(envelope["data"].get(key).is_none(), "{key} leaked"); + } + } + + #[test] + fn a_thread_message_carries_its_parent_and_names() { + let envelope = web_push_message(&thread_job(), USER_ID, 3); + assert_eq!(envelope["data"]["parent_id"], json!("7788")); + assert_eq!(envelope["data"]["parent_name"], json!("general")); + assert_eq!(envelope["data"]["channel_name"], json!("release plans")); + assert_eq!(apns_payload(&envelope)["parent_id"], json!("7788")); + } + + #[test] + fn only_a_forum_thread_created_message_carries_its_notification_type() { + let thread = web_push_message(&thread_job(), USER_ID, 3); + assert!(thread["data"].get(NOTIF_TYPE_KEY).is_none()); + let forum = MessageJob { + kind: Some("forum_thread_created".to_owned()), + ..thread_job() + }; + let envelope = web_push_message(&forum, USER_ID, 3); + assert_eq!(envelope["data"][NOTIF_TYPE_KEY], json!(9)); + assert_eq!(envelope["data"]["parent_id"], json!("7788")); + let message = fcm_message("device-token", &envelope); + assert_eq!(message["message"]["data"][NOTIF_TYPE_KEY], json!("9")); + assert_eq!( + message["message"]["android"]["notification"]["channel_id"], + json!(FCM_FORUM_THREAD_CREATED_CHANNEL_ID) + ); + assert_eq!( + fcm_message("device-token", &thread)["message"]["android"]["notification"]["channel_id"], + json!(FCM_DEFAULT_CHANNEL_ID) + ); + let budget = serialize(&minimal(&envelope, usize::MAX)).len(); + let (bytes, _step) = fit(&envelope, budget); + let shrunk: Value = serde_json::from_slice(&bytes).expect("the shrunk payload is json"); + assert_eq!(shrunk["data"][NOTIF_TYPE_KEY], json!(9)); + } + + #[test] + fn a_shrunk_thread_message_keeps_its_parent_id() { + let envelope = web_push_message(&thread_job(), USER_ID, 3); + let budget = serialize(&minimal(&envelope, usize::MAX)).len(); + let (bytes, step) = fit(&envelope, budget); + assert_eq!(step, Some(PayloadShrink::Minimal)); + let shrunk: Value = serde_json::from_slice(&bytes).expect("the shrunk payload is json"); + assert_eq!(shrunk["data"]["parent_id"], json!("7788")); + } + #[test] fn a_shrunk_clear_still_tells_the_client_it_is_a_clear() { let envelope = web_push_clear(&clear_job(), 3); diff --git a/fluxer_push/src/providers/mod.rs b/fluxer_push/src/providers/mod.rs index c7f204be4..f59e6403d 100644 --- a/fluxer_push/src/providers/mod.rs +++ b/fluxer_push/src/providers/mod.rs @@ -239,6 +239,7 @@ mod consent_tests { platform: Some("web_push".to_owned()), app_id: None, provider_environment: None, + thread_channels: false, } } diff --git a/fluxer_push/src/providers/own_relay.rs b/fluxer_push/src/providers/own_relay.rs index d337ba628..e54ed4ed1 100644 --- a/fluxer_push/src/providers/own_relay.rs +++ b/fluxer_push/src/providers/own_relay.rs @@ -47,6 +47,7 @@ impl Hop { } .to_owned() }), + thread_channels: sub.thread_channels, } } } diff --git a/fluxer_push/src/subscription.rs b/fluxer_push/src/subscription.rs index 83763e038..85f9fc523 100644 --- a/fluxer_push/src/subscription.rs +++ b/fluxer_push/src/subscription.rs @@ -23,6 +23,8 @@ pub struct Subscription { pub platform: Option, pub app_id: Option, pub provider_environment: Option, + #[serde(default)] + pub thread_channels: bool, } impl Subscription { diff --git a/packages/schema/src/domains/experiment/ExperimentBucketVectors.json b/packages/schema/src/domains/experiment/ExperimentBucketVectors.json new file mode 100644 index 000000000..8961dabe8 --- /dev/null +++ b/packages/schema/src/domains/experiment/ExperimentBucketVectors.json @@ -0,0 +1,38 @@ +[ + ["channel-threads-guild-v1", "0", 9122], + ["channel-threads-guild-v1", "1", 6741], + ["channel-threads-guild-v1", "1000000000000000001", 2330], + ["channel-threads-guild-v1", "1234567890123456789", 1114], + ["channel-threads-guild-v1", "1400000000000000000", 9497], + ["channel-threads-guild-v1", "18446744073709551615", 4495], + ["channel-threads-user-v1", "0", 5784], + ["channel-threads-user-v1", "1", 3403], + ["channel-threads-user-v1", "1000000000000000001", 2848], + ["channel-threads-user-v1", "1234567890123456789", 7968], + ["channel-threads-user-v1", "1400000000000000000", 1255], + ["channel-threads-user-v1", "18446744073709551615", 5433], + ["push-service-delivery-v1", "0", 2634], + ["push-service-delivery-v1", "1", 253], + ["push-service-delivery-v1", "1000000000000000001", 8258], + ["push-service-delivery-v1", "1234567890123456789", 9458], + ["push-service-delivery-v1", "1400000000000000000", 337], + ["push-service-delivery-v1", "18446744073709551615", 7207], + ["voice-ns-v1", "0", 5441], + ["voice-ns-v1", "1", 7822], + ["voice-ns-v1", "1000000000000000001", 8241], + ["voice-ns-v1", "1234567890123456789", 1217], + ["voice-ns-v1", "1400000000000000000", 3306], + ["voice-ns-v1", "18446744073709551615", 8882], + ["a", "0", 4966], + ["a", "1", 2585], + ["a", "1000000000000000001", 878], + ["a", "1234567890123456789", 3022], + ["a", "1400000000000000000", 4037], + ["a", "18446744073709551615", 8283], + ["Salt with spaces ~!@#$%^&*()", "0", 5392], + ["Salt with spaces ~!@#$%^&*()", "1", 5715], + ["Salt with spaces ~!@#$%^&*()", "1000000000000000001", 7960], + ["Salt with spaces ~!@#$%^&*()", "1234567890123456789", 2216], + ["Salt with spaces ~!@#$%^&*()", "1400000000000000000", 9599], + ["Salt with spaces ~!@#$%^&*()", "18446744073709551615", 4145] +]