Purge left nodes along with dead ones (#254)

Keep the state from infinitely expanding

Author: bwaters

util.go

@@ -96,13 +96,13 @@ func pushPullScale(interval time.Duration, n int) time.Duration {
 	return time.Duration(multiplier) * interval
 }
 
-// moveDeadNodes moves nodes that are dead and beyond the gossip to the dead interval
+// moveDeadNodes moves dead and left nodes that that have not changed during the gossipToTheDeadTime interval
 // to the end of the slice and returns the index of the first moved node.
 func moveDeadNodes(nodes []*nodeState, gossipToTheDeadTime time.Duration) int {
 	numDead := 0
 	n := len(nodes)
 	for i := 0; i < n-numDead; i++ {
-		if nodes[i].State != StateDead {
+		if !nodes[i].DeadOrLeft() {
 			continue
 		}
 

util_test.go

@@ -178,6 +178,16 @@ func TestMoveDeadNodes(t *testing.T) {
 			State:       StateDead,
 			StateChange: time.Now().Add(-10 * time.Second),
 		},
+		// This left node should not be moved, as its state changed
+		// less than the specified GossipToTheDead time ago
+		&nodeState{
+			State:       StateLeft,
+			StateChange: time.Now().Add(-10 * time.Second),
+		},
+		&nodeState{
+			State:       StateLeft,
+			StateChange: time.Now().Add(-20 * time.Second),
+		},
 		&nodeState{
 			State:       StateAlive,
 			StateChange: time.Now().Add(-20 * time.Second),
@@ -190,10 +200,14 @@ func TestMoveDeadNodes(t *testing.T) {
 			State:       StateAlive,
 			StateChange: time.Now().Add(-20 * time.Second),
 		},
+		&nodeState{
+			State:       StateLeft,
+			StateChange: time.Now().Add(-20 * time.Second),
+		},
 	}
 
 	idx := moveDeadNodes(nodes, (15 * time.Second))
-	if idx != 4 {
+	if idx != 5 {
 		t.Fatalf("bad index")
 	}
 	for i := 0; i < idx; i++ {
@@ -204,14 +218,19 @@ func TestMoveDeadNodes(t *testing.T) {
 			if nodes[i].State != StateDead {
 				t.Fatalf("Bad state %d", i)
 			}
+		case 3:
+			//Recently left node should remain at 3
+			if nodes[i].State != StateLeft {
+				t.Fatalf("Bad State %d", i)
+			}
 		default:
 			if nodes[i].State != StateAlive {
 				t.Fatalf("Bad state %d", i)
 			}
 		}
 	}
 	for i := idx; i < len(nodes); i++ {
-		if nodes[i].State != StateDead {
+		if !nodes[i].DeadOrLeft() {
 			t.Fatalf("Bad state %d", i)
 		}
 	}