Do not remove FailedMessage documents

Greg Bielleman · Greg Bielleman · commit 05bd383c1881 · 2014-12-03T15:42:08.000+11:00
diff --git a/src/ServiceControl.UnitTests/Expiration/CustomExpirationBundleTests.cs b/src/ServiceControl.UnitTests/Expiration/CustomExpirationBundleTests.cs
@@ -10,7 +10,7 @@
     using NUnit.Framework;
     using Raven.Client;
     using ServiceBus.Management.Infrastructure.Settings;
-    using ServiceControl.CompositeViews.Messages;
+    using ServiceControl.Infrastructure.RavenDB.Expiration;
 
     [TestFixture]
     public class CustomExpirationBundleTests : TestWithRavenDB
@@ -54,13 +54,14 @@ public void Processed_messages_are_being_expired()
         [Test]
         public void Many_processed_messages_are_being_expired()
         {
-            new MessagesViewIndex().Execute(documentStore);
+            new ExpiryProcessedMessageIndex().Execute(documentStore);
 
             var processedMessage = new ProcessedMessage
             {
                 Id = Guid.NewGuid().ToString(),
                 ProcessedAt = DateTime.UtcNow.AddMinutes(-DateTime.UtcNow.Millisecond%30).AddDays(-(Settings.HoursToKeepMessagesBeforeExpiring*3)),
             };
+
             var processedMessage2 = new ProcessedMessage
             {
                 Id = "2",
@@ -69,7 +70,7 @@ public void Many_processed_messages_are_being_expired()
             
             using (var session = documentStore.OpenSession())
             {
-                for (var i = 0; i < 2049; i++)
+                for (var i = 0; i < 100; i++)
                 {
                     processedMessage = new ProcessedMessage
                     {
@@ -85,13 +86,10 @@ public void Many_processed_messages_are_being_expired()
             }
 
             WaitForIndexing(documentStore);
-            Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 4);
-
+            Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 10);
             using (var session = documentStore.OpenSession())
             {
-                var results = session.Query<ProcessedMessage, MessagesViewIndex>()
-                    .Customize(x => x.WaitForNonStaleResults())
-                    .ToArray();
+                var results = session.Query<ProcessedMessage, ExpiryProcessedMessageIndex>().Customize(x => x.WaitForNonStaleResults()).ToArray();
                 Assert.AreEqual(1, results.Length);
 
                 var msg = session.Load<ProcessedMessage>(processedMessage.Id);
@@ -105,6 +103,8 @@ public void Many_processed_messages_are_being_expired()
         [Test]
         public void Only_processed_messages_are_being_expired()
         {
+            new ExpiryProcessedMessageIndex().Execute(documentStore);
+
             var processedMessage = new ProcessedMessage
             {
                 Id = "1",
@@ -141,6 +141,8 @@ public void Only_processed_messages_are_being_expired()
         [Test]
         public void Recent_processed_messages_are_not_being_expired()
         {
+            new ExpiryProcessedMessageIndex().Execute(documentStore);
+
             var processedMessage = new ProcessedMessage
             {
                 Id = "1",
@@ -205,12 +207,8 @@ public void SetUp()
         {
             documentStore = InMemoryStoreBuilder.GetInMemoryStore(withExpiration: true);
 
-            var customIndex = new MessagesViewIndex();
+            var customIndex = new ExpiryProcessedMessageIndex();
             customIndex.Execute(documentStore);
-
-            var transformer = new MessagesViewTransformer();
-
-            transformer.Execute(documentStore);
         }
 
         [TearDown]
diff --git a/src/ServiceControl.UnitTests/Infrastructure/RavenDB/Indexes/InMemoryStoreBuilder.cs b/src/ServiceControl.UnitTests/Infrastructure/RavenDB/Indexes/InMemoryStoreBuilder.cs
@@ -2,7 +2,7 @@
 using System.IO;
 using Raven.Client.Embedded;
 using ServiceBus.Management.Infrastructure.Settings;
-using ServiceControl.CompositeViews.Messages;
+using ServiceControl.Infrastructure.RavenDB.Expiration;
 
 public class InMemoryStoreBuilder
 {
@@ -25,15 +25,15 @@ public static EmbeddableDocumentStore GetInMemoryStore(bool withExpiration = fal
         if (withExpiration)
         {
             Settings.ExpirationProcessTimerInSeconds = 1; // so we don't have to wait too much in tests
-            store.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(ServiceControl.Infrastructure.RavenDB.Expiration.ExpiredDocumentsCleaner).Assembly));
+            store.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(ExpiredDocumentsCleaner).Assembly));
             store.Configuration.Settings.Add("Raven/ActiveBundles", "CustomDocumentExpiration"); // Enable the expiration bundle
         }
 
         store.Initialize();
 
         if (withExpiration)
         {
-            new MessagesViewIndex().Execute(store); // this index is being queried by our expiration bundle
+            new ExpiryProcessedMessageIndex().Execute(store); // this index is being queried by our expiration bundle
         }
 
         return store;
diff --git a/src/ServiceControl/Infrastructure/RavenDB/Expiration/ExpiredDocumentsCleaner.cs b/src/ServiceControl/Infrastructure/RavenDB/Expiration/ExpiredDocumentsCleaner.cs
@@ -7,7 +7,6 @@
     using System.Globalization;
     using System.Linq;
     using System.Threading;
-    using CompositeViews.Messages;
     using Raven.Abstractions;
     using Raven.Abstractions.Commands;
     using Raven.Abstractions.Data;
@@ -31,7 +30,7 @@ public class ExpiredDocumentsCleaner : IStartupTask, IDisposable
         public void Execute(DocumentDatabase database)
         {
             Database = database;
-            indexName = new MessagesViewIndex().IndexName;
+            indexName = new ExpiryProcessedMessageIndex().IndexName;
             
             deletionBatchSize = Settings.ExpirationProcessBatchSize;
             deleteFrequencyInSeconds = Settings.ExpirationProcessTimerInSeconds;
diff --git a/src/ServiceControl/Infrastructure/RavenDB/Expiration/ExpiryProcessedMessageIndex.cs b/src/ServiceControl/Infrastructure/RavenDB/Expiration/ExpiryProcessedMessageIndex.cs
@@ -0,0 +1,23 @@
+namespace ServiceControl.Infrastructure.RavenDB.Expiration
+{
+    using System.Linq;
+    using Contracts.Operations;
+    using MessageAuditing;
+    using Raven.Client.Indexes;
+
+    public class ExpiryProcessedMessageIndex : AbstractIndexCreationTask<ProcessedMessage>
+    {  
+        public ExpiryProcessedMessageIndex()
+        {
+            Map = (messages => from message in messages
+                select new 
+                {
+                    MessageId = (string) message.MessageMetadata["MessageId"],
+                    Status = (bool)message.MessageMetadata["IsRetried"] ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful,
+                    ProcessedAt = message.ProcessedAt,
+                });
+
+            DisableInMemoryIndexing = true;
+        }
+    }
+}
diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj
@@ -271,6 +271,7 @@
       <SubType>Component</SubType>
     </Compile>
     <Compile Include="Hosting\Options.cs" />
+    <Compile Include="Infrastructure\RavenDB\Expiration\ExpiryProcessedMessageIndex.cs" />
     <Compile Include="Infrastructure\Settings\RegistryReader.cs" />
     <Compile Include="Infrastructure\SubscribeToAllEvents.cs" />
     <Compile Include="MessageFailures\FailedMessageViewIndexNotifications.cs" />

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@`
`10`	`10`	`using NUnit.Framework;`
`11`	`11`	`using Raven.Client;`
`12`	`12`	`using ServiceBus.Management.Infrastructure.Settings;`
`13`		`- using ServiceControl.CompositeViews.Messages;`
	`13`	`+ using ServiceControl.Infrastructure.RavenDB.Expiration;`
`14`	`14`
`15`	`15`	`[TestFixture]`
`16`	`16`	`public class CustomExpirationBundleTests : TestWithRavenDB`
`@@ -54,13 +54,14 @@ public void Processed_messages_are_being_expired()`
`54`	`54`	`[Test]`
`55`	`55`	`public void Many_processed_messages_are_being_expired()`
`56`	`56`	`{`
`57`		`- new MessagesViewIndex().Execute(documentStore);`
	`57`	`+ new ExpiryProcessedMessageIndex().Execute(documentStore);`
`58`	`58`
`59`	`59`	`var processedMessage = new ProcessedMessage`
`60`	`60`	`{`
`61`	`61`	`Id = Guid.NewGuid().ToString(),`
`62`	`62`	`ProcessedAt = DateTime.UtcNow.AddMinutes(-DateTime.UtcNow.Millisecond%30).AddDays(-(Settings.HoursToKeepMessagesBeforeExpiring*3)),`
`63`	`63`	`};`
	`64`	`+`
`64`	`65`	`var processedMessage2 = new ProcessedMessage`
`65`	`66`	`{`
`66`	`67`	`Id = "2",`
`@@ -69,7 +70,7 @@ public void Many_processed_messages_are_being_expired()`
`69`	`70`
`70`	`71`	`using (var session = documentStore.OpenSession())`
`71`	`72`	`{`
`72`		`- for (var i = 0; i < 2049; i++)`
	`73`	`+ for (var i = 0; i < 100; i++)`
`73`	`74`	`{`
`74`	`75`	`processedMessage = new ProcessedMessage`
`75`	`76`	`{`
`@@ -85,13 +86,10 @@ public void Many_processed_messages_are_being_expired()`
`85`	`86`	`}`
`86`	`87`
`87`	`88`	`WaitForIndexing(documentStore);`
`88`		`- Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 4);`
`89`		`-`
	`89`	`+ Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 10);`
`90`	`90`	`using (var session = documentStore.OpenSession())`
`91`	`91`	`{`
`92`		`- var results = session.Query<ProcessedMessage, MessagesViewIndex>()`
`93`		`- .Customize(x => x.WaitForNonStaleResults())`
`94`		`- .ToArray();`
	`92`	`+ var results = session.Query<ProcessedMessage, ExpiryProcessedMessageIndex>().Customize(x => x.WaitForNonStaleResults()).ToArray();`
`95`	`93`	`Assert.AreEqual(1, results.Length);`
`96`	`94`
`97`	`95`	`var msg = session.Load<ProcessedMessage>(processedMessage.Id);`
`@@ -105,6 +103,8 @@ public void Many_processed_messages_are_being_expired()`
`105`	`103`	`[Test]`
`106`	`104`	`public void Only_processed_messages_are_being_expired()`
`107`	`105`	`{`
	`106`	`+ new ExpiryProcessedMessageIndex().Execute(documentStore);`
	`107`	`+`
`108`	`108`	`var processedMessage = new ProcessedMessage`
`109`	`109`	`{`
`110`	`110`	`Id = "1",`
`@@ -141,6 +141,8 @@ public void Only_processed_messages_are_being_expired()`
`141`	`141`	`[Test]`
`142`	`142`	`public void Recent_processed_messages_are_not_being_expired()`
`143`	`143`	`{`
	`144`	`+ new ExpiryProcessedMessageIndex().Execute(documentStore);`
	`145`	`+`
`144`	`146`	`var processedMessage = new ProcessedMessage`
`145`	`147`	`{`
`146`	`148`	`Id = "1",`
`@@ -205,12 +207,8 @@ public void SetUp()`
`205`	`207`	`{`
`206`	`208`	`documentStore = InMemoryStoreBuilder.GetInMemoryStore(withExpiration: true);`
`207`	`209`
`208`		`- var customIndex = new MessagesViewIndex();`
	`210`	`+ var customIndex = new ExpiryProcessedMessageIndex();`
`209`	`211`	`customIndex.Execute(documentStore);`
`210`		`-`
`211`		`- var transformer = new MessagesViewTransformer();`
`212`		`-`
`213`		`- transformer.Execute(documentStore);`
`214`	`212`	`}`
`215`	`213`
`216`	`214`	`[TearDown]`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using System.IO;`
`3`	`3`	`using Raven.Client.Embedded;`
`4`	`4`	`using ServiceBus.Management.Infrastructure.Settings;`
`5`		`-using ServiceControl.CompositeViews.Messages;`
	`5`	`+using ServiceControl.Infrastructure.RavenDB.Expiration;`
`6`	`6`
`7`	`7`	`public class InMemoryStoreBuilder`
`8`	`8`	`{`
`@@ -25,15 +25,15 @@ public static EmbeddableDocumentStore GetInMemoryStore(bool withExpiration = fal`
`25`	`25`	`if (withExpiration)`
`26`	`26`	`{`
`27`	`27`	`Settings.ExpirationProcessTimerInSeconds = 1; // so we don't have to wait too much in tests`
`28`		`- store.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(ServiceControl.Infrastructure.RavenDB.Expiration.ExpiredDocumentsCleaner).Assembly));`
	`28`	`+ store.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(ExpiredDocumentsCleaner).Assembly));`
`29`	`29`	`store.Configuration.Settings.Add("Raven/ActiveBundles", "CustomDocumentExpiration"); // Enable the expiration bundle`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`store.Initialize();`
`33`	`33`
`34`	`34`	`if (withExpiration)`
`35`	`35`	`{`
`36`		`- new MessagesViewIndex().Execute(store); // this index is being queried by our expiration bundle`
	`36`	`+ new ExpiryProcessedMessageIndex().Execute(store); // this index is being queried by our expiration bundle`
`37`	`37`	`}`
`38`	`38`
`39`	`39`	`return store;`