fixed problem with MNTK false negatives

Author: Horusiath

src/core/Akka.Cluster.Tests.MultiNode/SplitBrainResolverDowningSpec.cs

@@ -12,6 +12,7 @@
 using Akka.Configuration;
 using Akka.Remote.TestKit;
 using Akka.Remote.Transport;
+using FluentAssertions;
 
 namespace Akka.Cluster.Tests.MultiNode
 {
@@ -81,6 +82,13 @@ private void A_Cluster_must_detect_network_partition_and_down_minor_part_of_the_
 
             EnterBarrier("before-split");
 
+            var downed = false;
+
+            RunOn(() =>
+            {
+                Cluster.RegisterOnMemberRemoved(() => downed = true);
+            }, minority);
+
             RunOn(() =>
             {
                 foreach (var a in majority)
@@ -92,26 +100,15 @@ private void A_Cluster_must_detect_network_partition_and_down_minor_part_of_the_
 
             RunOn(() =>
             {
-                foreach (var role in minority)
-                {
-                    MarkNodeAsUnavailable(GetAddress(role));
-                }
-                
                 // side with majority of the nodes must stay up
                 AwaitMembersUp(majority.Length, canNotBePartOfMemberRing: minority.Select(GetAddress).ToImmutableHashSet());
                 AssertLeader(majority);
             }, majority);
-
+            
             RunOn(() =>
             {
-                foreach (var role in majority)
-                {
-                    MarkNodeAsUnavailable(GetAddress(role));
-                }
-
-                // side with majority of the nodes must stay up
-                AwaitMembersUp(majority.Length, canNotBePartOfMemberRing: minority.Select(GetAddress).ToImmutableHashSet());
-                AssertLeader(majority);
+                // side with majority of the nodes must stay up, minority must go down
+                AwaitAssert(() => downed.Should().BeTrue("cluster node on-removed hook has been triggered"));
             }, minority);
 
             EnterBarrier("after-2");

src/core/Akka.Cluster/SplitBrainResolver.cs

@@ -111,7 +111,7 @@ public IEnumerable<Member> Apply(NetworkPartitionContext context)
                 ? context.Remaining
                 : context.Unreachable;
         }
-        public override string ToString() => $"StaticQuorum(quorumSize: {QuorumSize}, role: {Role})";
+        public override string ToString() => $"StaticQuorum(quorumSize: {QuorumSize}, role: '{Role}')";
     }
 
     internal sealed class KeepMajority : ISplitBrainStrategy
@@ -148,7 +148,7 @@ private ImmutableSortedSet<Member> MembersWithRole(ImmutableSortedSet<Member> me
             ? members
             : members.Where(m => m.HasRole(Role)).ToImmutableSortedSet();
 
-        public override string ToString() => $"KeepMajority(role: {Role})";
+        public override string ToString() => $"KeepMajority(role: '{Role}')";
     }
 
     internal sealed class KeepOldest : ISplitBrainStrategy
@@ -190,7 +190,7 @@ private ImmutableSortedSet<Member> MembersWithRole(ImmutableSortedSet<Member> me
             ? members
             : members.Where(m => m.HasRole(Role)).ToImmutableSortedSet();
 
-        public override string ToString() => $"KeepOldest(downIfAlone: {DownIfAlone}, role: {Role})";
+        public override string ToString() => $"KeepOldest(downIfAlone: {DownIfAlone}, role: '{Role})'";
     }
 
     internal sealed class KeepReferee : ISplitBrainStrategy
@@ -307,21 +307,25 @@ protected override void OnReceive(object message)
                     return;
                 case StabilityReached _:
                     HandleStabilityReached();
-
                     return;
             }
         }
 
         private void HandleStabilityReached()
         {
+            if (Log.IsInfoEnabled)
+            {
+                Log.Info("A network partition detected - unreachable nodes: [{0}], remaining: [{1}]", string.Join(", ", _unreachable.Select(m => m.Address)), string.Join(", ", _reachable.Select(m => m.Address)));
+            }
+
             var context = new NetworkPartitionContext(_unreachable, _reachable);
             var nodesToDown = _strategy.Apply(context).ToImmutableArray();
 
             if (nodesToDown.Length > 0)
             {
                 if (Log.IsInfoEnabled)
                 {
-                    Log.Info("A network partition has been detected. {0} decided to down following nodes: [{0}]", _strategy, string.Join(", ", nodesToDown));
+                    Log.Info("A network partition has been detected. {0} decided to down following nodes: [{1}]", _strategy, string.Join(", ", nodesToDown));
                 }
 
                 foreach (var member in nodesToDown)