fix(gateway): keep dispatch ordered under relay backpressure (#2300)

fluxer_gateway/test/gateway_dispatch_relay_ordering_tests.erl

@@ -0,0 +1,98 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(gateway_dispatch_relay_ordering_tests).
+-typing([eqwalizer]).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(STATE_KEY, {gateway_dispatch_relay, state}).
+-define(EVENT_COUNT, 8).
+
+dispatch_preserves_order_when_worker_is_backpressured_test_() ->
+    {timeout, 30, fun dispatch_preserves_order_when_worker_is_backpressured/0}.
+
+dispatch_many_preserves_order_when_worker_is_backpressured_test_() ->
+    {timeout, 30, fun dispatch_many_preserves_order_when_worker_is_backpressured/0}.
+
+dispatch_preserves_order_when_worker_is_backpressured() ->
+    assert_ordered_delivery(fun(SessionPid, N) ->
+        gateway_dispatch_relay:dispatch(SessionPid, relay_order_event, #{<<"n">> => N})
+    end).
+
+dispatch_many_preserves_order_when_worker_is_backpressured() ->
+    assert_ordered_delivery(fun(SessionPid, N) ->
+        gateway_dispatch_relay:dispatch_many([SessionPid], relay_order_event, #{<<"n">> => N})
+    end).
+
+assert_ordered_delivery(SendFun) ->
+    Worker = gateway_dispatch_relay_batch:start_worker(0),
+    try
+        with_relay_workers([Worker], fun() -> ordered_delivery(Worker, SendFun) end)
+    after
+        stop_worker(Worker)
+    end.
+
+ordered_delivery(Worker, SendFun) ->
+    Ref = make_ref(),
+    Parent = self(),
+    SessionPid = spawn_link(fun() -> session_loop(Parent, Ref) end),
+    MaxQueue = gateway_rollout_config:gateway_dispatch_relay_max_queue(),
+    ok = sys:suspend(Worker),
+    fill_queue(Worker, MaxQueue - 1),
+    lists:foreach(fun(N) -> ok = SendFun(SessionPid, N) end, lists:seq(1, ?EVENT_COUNT)),
+    ?assert(gateway_dispatch_relay_batch:message_queue_len(Worker) >= MaxQueue),
+    ok = sys:resume(Worker),
+    Observed = collect_observed(Ref, ?EVENT_COUNT, []),
+    SessionPid ! stop,
+    ?assertEqual(lists:seq(1, ?EVENT_COUNT), Observed).
+
+fill_queue(_Worker, 0) ->
+    ok;
+fill_queue(Worker, Remaining) ->
+    Worker ! relay_order_filler,
+    fill_queue(Worker, Remaining - 1).
+
+session_loop(Parent, Ref) ->
+    receive
+        {'$gen_cast', {dispatch, relay_order_event, #{<<"n">> := N}}} ->
+            Parent ! {relay_order_received, Ref, N},
+            session_loop(Parent, Ref);
+        stop ->
+            ok
+    after 30000 ->
+        ok
+    end.
+
+collect_observed(_Ref, 0, Acc) ->
+    lists:reverse(Acc);
+collect_observed(Ref, Remaining, Acc) ->
+    receive
+        {relay_order_received, Ref, N} -> collect_observed(Ref, Remaining - 1, [N | Acc])
+    after 10000 ->
+        ?assert(false, {relay_order_timeout, Remaining, lists:reverse(Acc)})
+    end.
+
+with_relay_workers(Workers, Fun) ->
+    Previous = persistent_term:get(?STATE_KEY, undefined),
+    persistent_term:put(?STATE_KEY, #{
+        workers => list_to_tuple(Workers),
+        shard_count => length(Workers)
+    }),
+    try
+        Fun()
+    after
+        restore_relay_state(Previous)
+    end.
+
+restore_relay_state(undefined) ->
+    _ = persistent_term:erase(?STATE_KEY),
+    ok;
+restore_relay_state(Previous) ->
+    persistent_term:put(?STATE_KEY, Previous).
+
+stop_worker(Pid) ->
+    try gen_server:stop(Pid, normal, 5000) of
+        _ -> ok
+    catch
+        exit:_ -> ok
+    end.

fluxer_gateway/test/gateway_dispatch_relay_stress_tests.erl

@@ -14,8 +14,8 @@ dispatch_many_direct_fallback_delivers_once_to_many_sessions_test_() ->
 dispatch_many_worker_shards_deliver_once_to_many_sessions_test_() ->
     {timeout, 30, fun dispatch_many_worker_shards_deliver_once_to_many_sessions/0}.
 
-dispatch_many_backpressured_worker_falls_back_to_direct_delivery_test_() ->
-    {timeout, 30, fun dispatch_many_backpressured_worker_falls_back_to_direct_delivery/0}.
+dispatch_many_backpressured_worker_still_relays_delivery_test_() ->
+    {timeout, 30, fun dispatch_many_backpressured_worker_still_relays_delivery/0}.
 
 dispatch_many_direct_fallback_delivers_once_to_many_sessions() ->
     with_relay_state_cleared(fun() ->
@@ -44,24 +44,26 @@ dispatch_many_worker_shards_deliver_once_to_many_sessions() ->
     end),
     stop_workers(Workers).
 
-dispatch_many_backpressured_worker_falls_back_to_direct_delivery() ->
-    BackpressuredWorker = start_backpressured_worker(64),
-    with_relay_workers([BackpressuredWorker], fun() ->
-        ok = wait_until(fun() ->
-            gateway_dispatch_relay_batch:message_queue_len(BackpressuredWorker) >= 64
-        end),
+dispatch_many_backpressured_worker_still_relays_delivery() ->
+    Worker = gateway_dispatch_relay_batch:start_worker(0),
+    with_relay_workers([Worker], fun() ->
+        ok = sys:suspend(Worker),
+        backpressure_worker(Worker, 64),
         {Receivers, Ref} = start_receivers(512),
         try
             Payload = #{<<"stress">> => <<"backpressure">>},
             ok = gateway_dispatch_relay_batch:relay_or_direct_many(
-                Receivers, relay_stress_event, Payload, 1
+                Receivers, relay_stress_event, Payload
             ),
-            assert_received_once(Receivers, Ref, relay_stress_event, Payload)
+            ?assert(gateway_dispatch_relay_batch:message_queue_len(Worker) >= 64),
+            ok = sys:resume(Worker),
+            assert_received_once(Receivers, Ref, relay_stress_event, Payload),
+            assert_worker_queues_drained([Worker])
         after
             stop_receivers(Receivers)
         end
     end),
-    BackpressuredWorker ! stop.
+    stop_worker(Worker).
 
 with_relay_state_cleared(Fun) ->
     Previous = persistent_term:get(?STATE_KEY, undefined),
@@ -124,17 +126,8 @@ stop_worker(Pid) ->
         exit:_ -> ok
     end.
 
-start_backpressured_worker(MessageCount) ->
-    Pid = spawn_link(fun backpressured_worker_loop/0),
-    lists:foreach(fun(I) -> Pid ! {queued, I} end, lists:seq(1, MessageCount)),
-    Pid.
-
-backpressured_worker_loop() ->
-    receive
-        stop -> ok
-    after 30000 ->
-        ok
-    end.
+backpressure_worker(Worker, MessageCount) ->
+    lists:foreach(fun(I) -> Worker ! {queued, I} end, lists:seq(1, MessageCount)).
 
 assert_received_once(Receivers, Ref, Event, Payload) ->
     Expected = maps:from_list([{Pid, false} || Pid <- Receivers]),