Merge pull request #15539 from rabbitmq/test-flakes-feb23-2026

Flaky test fixes

Author: michaelklishin

.github/workflows/test-make-target.yaml

@@ -105,7 +105,8 @@ jobs:
         sudo apt-get update && \
         sudo apt-get install -y \
           ldap-utils \
-          slapd
+          slapd \
+          util-linux
 
         sudo systemctl stop slapd
 

deps/rabbit/test/classic_queue_SUITE.erl

@@ -152,7 +152,14 @@ leader_locator_client_local(Config) ->
          ?assertEqual({'queue.declare_ok', Q, 0, 0},
                       declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"classic">>},
                                       {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])),
-         {ok, Leader0} = rabbit_ct_broker_helpers:rpc(Config, Server, rabbit_amqqueue, lookup, [rabbit_misc:r(<<"/">>, queue, Q)]),
+         {ok, Leader0} = ?awaitMatch(
+                            {ok, _},
+                            rabbit_ct_broker_helpers:rpc(Config,
+                                                         Server,
+                                                         rabbit_amqqueue,
+                                                         lookup,
+                                                         [rabbit_misc:r(<<"/">>, queue, Q)]),
+                            5000),
          Leader = amqqueue:qnode(Leader0),
          ?assertEqual(Server, Leader),
          ?assertMatch(#'queue.delete_ok'{},
@@ -175,12 +182,19 @@ test_leader_locator(Config, Argument, Strategies) ->
 
     [begin
          Leaders = [begin
-                        ?assertMatch({'queue.declare_ok', Q, 0, 0},
+                        ?assertEqual({'queue.declare_ok', Q, 0, 0},
                                      declare(Ch, Q,
                                              [{<<"x-queue-type">>, longstr, <<"classic">>},
                                               {Argument, longstr, Strategy}])),
 
-                        {ok, Leader0} = rabbit_ct_broker_helpers:rpc(Config, Server, rabbit_amqqueue, lookup, [rabbit_misc:r(<<"/">>, queue, Q)]),
+                        {ok, Leader0} = ?awaitMatch(
+                                           {ok, _},
+                                           rabbit_ct_broker_helpers:rpc(Config,
+                                                                        Server,
+                                                                        rabbit_amqqueue,
+                                                                        lookup,
+                                                                        [rabbit_misc:r(<<"/">>, queue, Q)]),
+                                           5000),
                         Leader = amqqueue:qnode(Leader0),
                         Leader
                     end || Q <- Qs],

deps/rabbitmq_cli/lib/rabbitmq/cli/core/distribution.ex

@@ -101,13 +101,24 @@ defmodule RabbitMQ.CLI.Core.Distribution do
         throw(err)
 
       rmq_hostname ->
-        # This limits the number of possible unique node names used by CLI tools to avoid
-        # the atom table from growing above the node limit. We must use reasonably unique IDs
-        # to allow for concurrent CLI tool execution.
-        #
-        # Enum.random/1 is constant time and space with range arguments https://hexdocs.pm/elixir/Enum.html#random/1.
-        id = Enum.random(1..1024)
+        id = generate_cli_node_id()
         String.to_atom("rabbitmqcli-#{id}-#{rmq_hostname}")
     end
   end
+
+  # The main goal is to limit the number of possible unique node names used by CLI
+  # tools to avoid the atom table from growing above the node limit.
+  # However, when RABBITMQ_CTL_UNIQUE_NODE_NAME is set, each CLI invocation gets
+  # a guaranteed unique node name. This avoids name collisions in CI environments
+  # that run many CLI commands concurrently. The unbounded atom
+  # table growth is acceptable in short-lived CI environments.
+  defp generate_cli_node_id do
+    case System.get_env("RABBITMQ_CTL_UNIQUE_NODE_NAME") do
+      nil ->
+        Enum.random(1..1024)
+
+      _ ->
+        "#{:os.getpid()}"
+    end
+  end
 end

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -1335,7 +1335,8 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
       {"RABBITMQ_MNESIA_DIR", ?config(data_dir, NodeConfig)},
       {"RABBITMQ_PLUGINS_DIR", ?config(plugins_dir, NodeConfig)},
       {"RABBITMQ_ENABLED_PLUGINS_FILE",
-        ?config(enabled_plugins_file, NodeConfig)}
+        ?config(enabled_plugins_file, NodeConfig)},
+      {"RABBITMQ_CTL_UNIQUE_NODE_NAME", "true"}
     ],
     Ret = rabbit_ct_helpers:get_config(
             NodeConfig, enabled_feature_flags_list_file),
@@ -1366,7 +1367,8 @@ rabbitmq_queues(Config, Node, Args) ->
       {"RABBITMQ_MNESIA_DIR", ?config(data_dir, NodeConfig)},
       {"RABBITMQ_PLUGINS_DIR", ?config(plugins_dir, NodeConfig)},
       {"RABBITMQ_ENABLED_PLUGINS_FILE",
-        ?config(enabled_plugins_file, NodeConfig)}
+        ?config(enabled_plugins_file, NodeConfig)},
+      {"RABBITMQ_CTL_UNIQUE_NODE_NAME", "true"}
     ],
     Ret = rabbit_ct_helpers:get_config(
             NodeConfig, enabled_feature_flags_list_file),
@@ -1390,7 +1392,8 @@ rabbitmq_streams(Config, Node, Args) ->
       {"RABBITMQ_MNESIA_DIR", ?config(data_dir, NodeConfig)},
       {"RABBITMQ_PLUGINS_DIR", ?config(plugins_dir, NodeConfig)},
       {"RABBITMQ_ENABLED_PLUGINS_FILE",
-        ?config(enabled_plugins_file, NodeConfig)}
+        ?config(enabled_plugins_file, NodeConfig)},
+      {"RABBITMQ_CTL_UNIQUE_NODE_NAME", "true"}
     ],
     Ret = rabbit_ct_helpers:get_config(
             NodeConfig, enabled_feature_flags_list_file),
@@ -1414,7 +1417,8 @@ rabbitmq_diagnostics(Config, Node, Args) ->
       {"RABBITMQ_MNESIA_DIR", ?config(data_dir, NodeConfig)},
       {"RABBITMQ_PLUGINS_DIR", ?config(plugins_dir, NodeConfig)},
       {"RABBITMQ_ENABLED_PLUGINS_FILE",
-        ?config(enabled_plugins_file, NodeConfig)}
+        ?config(enabled_plugins_file, NodeConfig)},
+      {"RABBITMQ_CTL_UNIQUE_NODE_NAME", "true"}
     ],
     Cmd = [Rabbitmqdiagnostics, "-n", Nodename | Args],
     rabbit_ct_helpers:exec(Cmd, [{env, Env}]).
@@ -2252,7 +2256,8 @@ plugin_action(Config, Node, Args) ->
       {"RABBITMQ_MNESIA_DIR", ?config(data_dir, NodeConfig)},
       {"RABBITMQ_PLUGINS_DIR", ?config(plugins_dir, NodeConfig)},
       {"RABBITMQ_ENABLED_PLUGINS_FILE",
-        ?config(enabled_plugins_file, NodeConfig)}
+        ?config(enabled_plugins_file, NodeConfig)},
+      {"RABBITMQ_CTL_UNIQUE_NODE_NAME", "true"}
     ],
     Cmd = [Rabbitmqplugins, "-n", Nodename | Args],
     {ok, _} = rabbit_ct_helpers:exec(Cmd, [{env, Env}]),

deps/rabbitmq_mqtt/test/auth_SUITE.erl

@@ -898,7 +898,10 @@ will_queue_publish_permission_topic_write(Config) ->
 will_queue_delete_permission(Config) ->
     set_permissions(".*", ".*", ".*", Config),
     ClientId = atom_to_binary(?FUNCTION_NAME),
+    Vhost = ?config(mqtt_vhost, Config),
     disconnect_with_delayed_will(ClientId, Config),
+    QName = rabbit_misc:r(Vhost, queue, <<"mqtt-will-", ClientId/binary>>),
+    ?awaitMatch({ok, _}, rpc(Config, 0, rabbit_amqqueue, lookup, [QName]), 5000),
     set_permissions(<<>>, ".*", ".*", Config),
     %% Now we have a Will queue that user doesn't have permission to delete.
     %% Resuming the session should fail.