fix flapping alerts in Alertmanager

- If startsAt and endsAt are equal, Alertmanager will mark the alert as
  resolved.  This is the case when the grace period is set to 0 for an
  alert condition.

- Alertmanager will prune alerts when it hits the time specified in the
  endsAt field.  Graylog is good at sending notifications on time, but
  a small window is added in case there is a delay will prevent alerts
  going stale.

Author: knackaron

src/main/java/de/gdata/mobilelab/alertmanagercallback/AlertManagerPayloadBuilder.java

@@ -118,7 +118,16 @@ private String extractEndsAt() {
             return new DateTime().plusMinutes(1).toString();
         }
 
-        return checkResult.getTriggeredAt().plusMinutes(checkResult.getTriggeredCondition().getGrace()).toString();
+        int delay = checkResult.getTriggeredCondition().getGrace();
+
+        // when grace is 0, the next alert isn't for another minute
+        if(delay == 0) {
+            delay += 1;
+        }
+
+        // give a small window to avoid alerts expiring due to the notification
+        // not being sent exactly on time
+        return checkResult.getTriggeredAt().plusMinutes(delay).plusSeconds(10).toString();
     }
 
     private String extractStartsAt() {

src/test/java/de/gdata/mobilelab/alertmanagercallback/AlertManagerPayloadBuilderTest.java

@@ -157,7 +157,7 @@ public void buildWithFullInformation() {
         assertEquals(triggeredAt.toString(), alertManagerPayload.getStartsAt());
 
         // - endsAt
-        assertEquals(triggeredAt.plusMinutes(1337).toString(), alertManagerPayload.getEndsAt());
+        assertEquals(triggeredAt.plusMinutes(1337).plusSeconds(10).toString(), alertManagerPayload.getEndsAt());
 
         // - generatorUrl
         assertEquals("aStreamUrl", alertManagerPayload.getGeneratorURL());
@@ -216,6 +216,8 @@ public void buildWithNoInformationForEndsAt() {
         AlertCondition alertCondition = mock(AlertCondition.class);
         when(alertCondition.getGrace()).thenReturn(0);
         when(checkResult.getTriggeredCondition()).thenReturn(alertCondition);
+        DateTime triggeredAt = new DateTime();
+        when(checkResult.getTriggeredAt()).thenReturn(triggeredAt);
 
         // and: instance with set mocks as values
         AlertManagerPayload alertManagerPayload = AlertManagerPayloadBuilder.newInstance()
@@ -226,6 +228,7 @@ public void buildWithNoInformationForEndsAt() {
         // - endsAt
         assertNotNull(alertManagerPayload.getEndsAt());
         assertNotEquals("", alertManagerPayload.getEndsAt());
+        assertEquals(triggeredAt.plusMinutes(1).plusSeconds(10).toString(), alertManagerPayload.getEndsAt());
     }
 
     @Test
@@ -602,4 +605,4 @@ public void buildWithCustomTemplateAnnotationsWithoutMessageBacklog() {
         assertEquals("<No Backlog>", alertManagerPayload.getAnnotations().get("myMessageBacklog"));
         assertEquals(13, alertManagerPayload.getAnnotations().size());
     }
-}
+}