fix(gateway): share the relay dispatch bound across producers (#2315)

fluxer_gateway/test/gateway_dispatch_relay_bound_tests.erl

@@ -15,6 +15,9 @@
 -define(PROBE_BUDGET, 2).
 -define(SHARD_COUNT, 2).
 -define(FAST_SHARD_TIMEOUT_MS, 1500).
+-define(PRODUCER_COUNTS, [1, 10, 64]).
+-define(PEAK_SAMPLES, 100).
+-define(PEAK_INTERVAL_MS, 5).
 
 dispatch_is_bounded_and_stays_ordered_test_() ->
     {timeout, 30, fun dispatch_is_bounded_and_stays_ordered/0}.
@@ -37,6 +40,12 @@ dispatch_does_not_probe_the_worker_per_event_test_() ->
 saturated_shard_does_not_delay_other_shards_test_() ->
     {timeout, 60, fun saturated_shard_does_not_delay_other_shards/0}.
 
+concurrent_producers_share_the_worker_bound_test_() ->
+    {timeout, 120, fun concurrent_producers_share_the_worker_bound/0}.
+
+restarted_worker_starts_from_an_empty_bound_test_() ->
+    {timeout, 30, fun restarted_worker_starts_from_an_empty_bound/0}.
+
 dispatch_is_bounded_and_stays_ordered() ->
     assert_bounded_and_ordered(fun send_dispatch/2).
 
@@ -105,6 +114,66 @@ saturated_shard_does_not_delay_other_shards() ->
         end)
     end).
 
+restarted_worker_starts_from_an_empty_bound() ->
+    with_max_queue(?BOUND, fun() ->
+        Saturated = saturate_worker(),
+        Ref = make_ref(),
+        Session = spawn_session(self(), Ref),
+        with_worker(fun(_Restarted) ->
+            Producer = spawn_producer(fun send_dispatch/2, Session),
+            ?assertEqual(finished, producer_status(Producer, 5000)),
+            ?assertEqual(lists:seq(1, ?EVENT_COUNT), collect_observed(Ref, ?EVENT_COUNT, []))
+        end),
+        Session ! stop,
+        stop_worker(Saturated)
+    end).
+
+saturate_worker() ->
+    Worker = gateway_dispatch_relay_batch:start_worker(0),
+    Session = spawn_session(self(), make_ref()),
+    with_relay_workers([Worker], fun() ->
+        ok = sys:suspend(Worker),
+        fill_queue(Worker, ?BOUND),
+        Producer = spawn_producer(fun send_dispatch/2, Session),
+        ?assertEqual(blocked, producer_status(Producer, 500))
+    end),
+    Session ! stop,
+    Worker.
+
+concurrent_producers_share_the_worker_bound() ->
+    lists:foreach(fun assert_peak_within_bound/1, ?PRODUCER_COUNTS).
+
+assert_peak_within_bound(Producers) ->
+    Verdict = bound_verdict(peak_queue_len(Producers)),
+    ?assertEqual({Producers, within_bound}, {Producers, Verdict}).
+
+bound_verdict(Peak) when Peak =< ?BOUND + 1 -> within_bound;
+bound_verdict(Peak) -> {over_bound, Peak}.
+
+peak_queue_len(Producers) ->
+    with_max_queue(?BOUND, fun() ->
+        with_worker(fun(Worker) -> concurrent_peak(Worker, Producers) end)
+    end).
+
+concurrent_peak(Worker, Producers) ->
+    Ref = make_ref(),
+    Session = spawn_session(self(), Ref),
+    ok = sys:suspend(Worker),
+    Pids = [spawn_producer(fun send_dispatch/2, Session) || _ <- lists:seq(1, Producers)],
+    Peak = sample_peak(Worker, ?PEAK_SAMPLES, 0),
+    ok = sys:resume(Worker),
+    lists:foreach(fun(Pid) -> ok = await_producer(Pid, blocked) end, Pids),
+    _ = collect_observed(Ref, Producers * ?EVENT_COUNT, []),
+    Session ! stop,
+    Peak.
+
+sample_peak(_Worker, 0, Peak) ->
+    Peak;
+sample_peak(Worker, Remaining, Peak) ->
+    ok = gateway_retry_timer:wait(?PEAK_INTERVAL_MS),
+    Sampled = gateway_dispatch_relay_batch:message_queue_len(Worker),
+    sample_peak(Worker, Remaining - 1, max(Peak, Sampled)).
+
 assert_bounded_and_ordered(SendFun) ->
     {Status, QueueLen, Observed} = run_over_bound(SendFun),
     ?assertEqual(lists:seq(1, ?EVENT_COUNT), Observed),