More recovery tests

Author: michaelklishin

projects/client/Unit/src/unit/TestConnectionRecovery.cs

@@ -73,6 +73,35 @@ public void TestBasicConnectionRecovery()
             Assert.IsTrue(Conn.IsOpen);
         }
 
+        [Test]
+        public void TestBasicConnectionRecoveryOnBrokerRestart()
+        {
+            Assert.IsTrue(Conn.IsOpen);
+            RestartServerAndWaitForRecovery();
+            Assert.IsTrue(Conn.IsOpen);
+        }
+
+        [Test]
+        public void TestShutdownEventHandlersRecoveryOnConnectionAfterDelayedServerRestart()
+        {
+            Int32 counter = 0;
+            Conn.ConnectionShutdown += (c, args) =>
+            {
+                Interlocked.Increment(ref counter);
+            };
+
+            Assert.IsTrue(Conn.IsOpen);
+            StopRabbitMQ();
+            Console.WriteLine("About to sleep for 9 seconds...");
+            Thread.Sleep(9000);
+            StartRabbitMQ();
+            WaitForShutdown();
+            WaitForRecovery();
+            Assert.IsTrue(Conn.IsOpen);
+
+            Assert.IsTrue(counter >= 1);
+        }
+
         [Test]
         public void TestShutdownEventHandlersRecoveryOnConnection()
         {
@@ -151,6 +180,14 @@ public void TestBasicModelRecovery()
             Assert.IsTrue(Model.IsOpen);
         }
 
+        [Test]
+        public void TestBasicModelRecoveryOnServerRestart()
+        {
+            Assert.IsTrue(Model.IsOpen);
+            RestartServerAndWaitForRecovery();
+            Assert.IsTrue(Model.IsOpen);
+        }
+
         [Test]
         public void TestShutdownEventHandlersRecoveryOnModel()
         {
@@ -268,6 +305,17 @@ public void TestClientNamedQueueRecovery()
             }, s);
         }
 
+        [Test]
+        public void TestClientNamedQueueRecoveryOnServerRestart()
+        {
+            string s = "dotnet-client.test.recovery.q1";
+            WithTemporaryNonExclusiveQueue(Model, (m, q) => {
+                RestartServerAndWaitForRecovery();
+                AssertQueueRecovery(m, q, false);
+                Model.QueueDelete(q);
+            }, s);
+        }
+
         [Test]
         public void TestClientNamedQueueRecoveryNoWait()
         {
@@ -802,6 +850,35 @@ protected void CloseAndWaitForShutdown(AutorecoveringConnection conn)
             Wait(sl);
         }
 
+        protected void RestartServerAndWaitForRecovery()
+        {
+            RestartServerAndWaitForRecovery((AutorecoveringConnection)this.Conn);
+        }
+
+        protected void RestartServerAndWaitForRecovery(AutorecoveringConnection conn)
+        {
+            var sl = PrepareForShutdown(conn);
+            var rl = PrepareForRecovery(conn);
+            RestartRabbitMQ();
+            Wait(sl);
+            Wait(rl);
+        }
+
+        protected void WaitForRecovery()
+        {
+            Wait(PrepareForRecovery((AutorecoveringConnection)this.Conn));
+        }
+
+        protected void WaitForRecovery(AutorecoveringConnection conn)
+        {
+            Wait(PrepareForRecovery(conn));
+        }
+
+        protected void WaitForShutdown()
+        {
+            Wait(PrepareForShutdown(this.Conn));
+        }
+
         protected void WaitForShutdown(IConnection conn)
         {
             Wait(PrepareForShutdown(conn));