%% SPDX-License-Identifier: AGPL-3.0-or-later -module(gateway_dispatch_relay_stress_tests). -typing([eqwalizer]). -include_lib("eunit/include/eunit.hrl"). -define(STATE_KEY, {gateway_dispatch_relay, state}). -define(RELAY_SESSION_COUNT, 2048). dispatch_many_direct_fallback_delivers_once_to_many_sessions_test_() -> {timeout, 30, fun dispatch_many_direct_fallback_delivers_once_to_many_sessions/0}. dispatch_many_worker_shards_deliver_once_to_many_sessions_test_() -> {timeout, 30, fun dispatch_many_worker_shards_deliver_once_to_many_sessions/0}. dispatch_many_backpressured_worker_still_relays_delivery_test_() -> {timeout, 30, fun dispatch_many_backpressured_worker_still_relays_delivery/0}. dispatch_many_direct_fallback_delivers_once_to_many_sessions() -> with_relay_state_cleared(fun() -> {Receivers, Ref} = start_receivers(?RELAY_SESSION_COUNT), try Payload = #{<<"stress">> => <<"direct">>}, ok = gateway_dispatch_relay:dispatch_many(Receivers, relay_stress_event, Payload), assert_received_once(Receivers, Ref, relay_stress_event, Payload) after stop_receivers(Receivers) end end). dispatch_many_worker_shards_deliver_once_to_many_sessions() -> Workers = gateway_dispatch_relay_batch:start_workers(8), with_relay_workers(Workers, fun() -> {Receivers, Ref} = start_receivers(?RELAY_SESSION_COUNT), try Payload = #{<<"stress">> => <<"workers">>}, ok = gateway_dispatch_relay:dispatch_many(Receivers, relay_stress_event, Payload), assert_received_once(Receivers, Ref, relay_stress_event, Payload), assert_worker_queues_drained(Workers) after stop_receivers(Receivers) end end), stop_workers(Workers). dispatch_many_backpressured_worker_still_relays_delivery() -> Worker = gateway_dispatch_relay_batch:start_worker(0), with_relay_workers([Worker], fun() -> ok = sys:suspend(Worker), backpressure_worker(Worker, 64), {Receivers, Ref} = start_receivers(512), try Payload = #{<<"stress">> => <<"backpressure">>}, ok = gateway_dispatch_relay_batch:relay_or_direct_many( Receivers, relay_stress_event, Payload ), ?assert(gateway_dispatch_relay_batch:message_queue_len(Worker) >= 64), ok = sys:resume(Worker), assert_received_once(Receivers, Ref, relay_stress_event, Payload), assert_worker_queues_drained([Worker]) after stop_receivers(Receivers) end end), stop_worker(Worker). with_relay_state_cleared(Fun) -> Previous = persistent_term:get(?STATE_KEY, undefined), _ = persistent_term:erase(?STATE_KEY), try Fun() after restore_relay_state(Previous) end. with_relay_workers(Workers, Fun) -> Previous = persistent_term:get(?STATE_KEY, undefined), persistent_term:put(?STATE_KEY, #{ workers => list_to_tuple(Workers), shard_count => length(Workers) }), try Fun() after restore_relay_state(Previous) end. restore_relay_state(undefined) -> _ = persistent_term:erase(?STATE_KEY), ok; restore_relay_state(Previous) -> persistent_term:put(?STATE_KEY, Previous), ok. start_receivers(Count) -> Ref = make_ref(), Parent = self(), Receivers = [ spawn_link(fun() -> receiver_loop(Parent, Ref) end) || _ <- lists:seq(1, Count) ], {Receivers, Ref}. receiver_loop(Parent, Ref) -> receive {'$gen_cast', {dispatch, Event, Payload}} -> Parent ! {relay_stress_received, Ref, self(), Event, Payload}, receiver_loop(Parent, Ref); stop -> ok after 30000 -> ok end. stop_receivers(Receivers) -> lists:foreach(fun(Pid) -> Pid ! stop end, Receivers). stop_workers(Workers) -> lists:foreach(fun stop_worker/1, Workers). stop_worker(Pid) -> try gen_server:stop(Pid, normal, 5000) of _ -> ok catch exit:_ -> ok end. backpressure_worker(Worker, MessageCount) -> lists:foreach(fun(I) -> Worker ! {queued, I} end, lists:seq(1, MessageCount)). assert_received_once(Receivers, Ref, Event, Payload) -> Expected = maps:from_list([{Pid, false} || Pid <- Receivers]), Seen = collect_received(length(Receivers), Ref, Event, Payload, Expected), ?assertEqual(maps:from_list([{Pid, true} || Pid <- Receivers]), Seen), assert_no_extra_received(Ref). collect_received(0, _Ref, _Event, _Payload, Seen) -> Seen; collect_received(Remaining, Ref, Event, Payload, Seen) -> receive {relay_stress_received, Ref, Pid, Event, Payload} -> ?assertEqual(false, maps:get(Pid, Seen, duplicate)), collect_received(Remaining - 1, Ref, Event, Payload, Seen#{Pid := true}); {relay_stress_received, Ref, Pid, OtherEvent, OtherPayload} -> ?assert(false, {unexpected_relay_payload, Pid, OtherEvent, OtherPayload}) after 5000 -> Missing = [Pid || {Pid, false} <- maps:to_list(Seen)], ?assert(false, {relay_stress_timeout, Remaining, length(Missing)}) end. assert_no_extra_received(Ref) -> receive {relay_stress_received, Ref, Pid, Event, Payload} -> ?assert(false, {duplicate_relay_delivery, Pid, Event, Payload}) after 100 -> ok end. assert_worker_queues_drained(Workers) -> ok = wait_until(fun() -> lists:all( fun(Pid) -> gateway_dispatch_relay_batch:message_queue_len(Pid) =:= 0 end, Workers ) end). wait_until(Pred) -> Deadline = erlang:monotonic_time(millisecond) + 5000, wait_until(Pred, Deadline). wait_until(Pred, Deadline) -> case Pred() of true -> ok; false -> case erlang:monotonic_time(millisecond) >= Deadline of true -> ?assert(false, wait_until_timeout); false -> timer:sleep(10), wait_until(Pred, Deadline) end end.