Add retries to apple push notifications (#128)

* Add retries to apple push notifications

* Add retry timeout config

* Add check for retry timeout being greater than send timeout

Author: larkox

config/mattermost-push-proxy.sample.json

@@ -5,6 +5,7 @@
     "ThrottleVaryByHeader":"X-Forwarded-For",
     "EnableMetrics": false,
     "SendTimeoutSec": 30,
+    "RetryTimeoutSec": 8,
     "ApplePushSettings":[
         {
             "Type":"apple",

server/apple_notification_server.go

@@ -25,14 +25,16 @@ type AppleNotificationServer struct {
 	logger            *Logger
 	ApplePushSettings ApplePushSettings
 	sendTimeout       time.Duration
+	retryTimeout      time.Duration
 }
 
-func NewAppleNotificationServer(settings ApplePushSettings, logger *Logger, metrics *metrics, sendTimeoutSecs int) *AppleNotificationServer {
+func NewAppleNotificationServer(settings ApplePushSettings, logger *Logger, metrics *metrics, sendTimeoutSecs int, retryTimeoutSecs int) *AppleNotificationServer {
 	return &AppleNotificationServer{
 		ApplePushSettings: settings,
 		metrics:           metrics,
 		logger:            logger,
 		sendTimeout:       time.Duration(sendTimeoutSecs) * time.Second,
+		retryTimeout:      time.Duration(retryTimeoutSecs) * time.Second,
 	}
 }
 
@@ -227,15 +229,8 @@ func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushR
 
 	if me.AppleClient != nil {
 		me.logger.Infof("Sending apple push notification for device=%v type=%v ackId=%v", me.ApplePushSettings.Type, msg.Type, msg.AckID)
-		start := time.Now()
-
-		ctx, cancel := context.WithTimeout(context.Background(), me.sendTimeout)
-		defer cancel()
 
-		res, err := me.AppleClient.PushWithContext(ctx, notification)
-		if me.metrics != nil {
-			me.metrics.observerNotificationResponse(PushNotifyApple, time.Since(start).Seconds())
-		}
+		res, err := me.SendNotificationWithRetry(notification)
 		if err != nil {
 			me.logger.Errorf("Failed to send apple push sid=%v did=%v err=%v type=%v", msg.ServerID, msg.DeviceID, err, me.ApplePushSettings.Type)
 			if me.metrics != nil {
@@ -269,3 +264,51 @@ func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushR
 	}
 	return NewOkPushResponse()
 }
+
+func (me *AppleNotificationServer) SendNotificationWithRetry(notification *apns.Notification) (*apns.Response, error) {
+	var res *apns.Response
+	var err error
+	waitTime := time.Second
+
+	// Keep a general context to make sure the whole retry
+	// doesn't take longer than the timeout.
+	generalContext, cancelGeneralContext := context.WithTimeout(context.Background(), me.sendTimeout)
+	defer cancelGeneralContext()
+
+	for retries := 0; retries < MAX_RETRIES; retries++ {
+		start := time.Now()
+
+		retryContext, cancelRetryContext := context.WithTimeout(generalContext, me.retryTimeout)
+		defer cancelRetryContext()
+		res, err = me.AppleClient.PushWithContext(retryContext, notification)
+		if me.metrics != nil {
+			me.metrics.observerNotificationResponse(PushNotifyApple, time.Since(start).Seconds())
+		}
+
+		if err == nil {
+			break
+		}
+
+		me.logger.Errorf("Failed to send apple push did=%v retry=%v error=%v", notification.DeviceToken, retries, err)
+
+		if retries == MAX_RETRIES-1 {
+			me.logger.Errorf("Max retries reached did=%v", notification.DeviceToken)
+			break
+		}
+
+		select {
+		case <-generalContext.Done():
+		case <-time.After(waitTime):
+		}
+
+		if generalContext.Err() != nil {
+			me.logger.Infof("Not retrying because context error did=%v retry=%v error=%v", notification.DeviceToken, retries, generalContext.Err())
+			err = generalContext.Err()
+			break
+		}
+
+		waitTime *= 2
+	}
+
+	return res, err
+}

server/config_push_proxy.go

@@ -17,6 +17,7 @@ type ConfigPushProxy struct {
 	ThrottleVaryByHeader    string
 	LogFileLocation         string
 	SendTimeoutSec          int
+	RetryTimeoutSec         int
 	ApplePushSettings       []ApplePushSettings
 	EnableMetrics           bool
 	EnableConsoleLog        bool
@@ -81,6 +82,20 @@ func LoadConfig(fileName string) (*ConfigPushProxy, error) {
 	if !cfg.EnableConsoleLog && !cfg.EnableFileLog {
 		cfg.EnableConsoleLog = true
 	}
+
+	// Set timeout defaults
+	if cfg.SendTimeoutSec == 0 {
+		cfg.SendTimeoutSec = 30
+	}
+
+	if cfg.RetryTimeoutSec == 0 {
+		cfg.RetryTimeoutSec = 8
+	}
+
+	if cfg.RetryTimeoutSec > cfg.SendTimeoutSec {
+		cfg.RetryTimeoutSec = cfg.SendTimeoutSec
+	}
+
 	if cfg.EnableFileLog {
 		if cfg.LogFileLocation == "" {
 			// We just do an mkdir -p equivalent.

server/server.go

@@ -27,6 +27,7 @@ const (
 	HEADER_REAL_IP             = "X-Real-IP"
 	WAIT_FOR_SERVER_SHUTDOWN   = time.Second * 5
 	CONNECTION_TIMEOUT_SECONDS = 60
+	MAX_RETRIES                = 3
 )
 
 type NotificationServer interface {
@@ -69,7 +70,7 @@ func (s *Server) Start() {
 	}
 
 	for _, settings := range s.cfg.ApplePushSettings {
-		server := NewAppleNotificationServer(settings, s.logger, m, s.cfg.SendTimeoutSec)
+		server := NewAppleNotificationServer(settings, s.logger, m, s.cfg.SendTimeoutSec, s.cfg.RetryTimeoutSec)
 		err := server.Initialize()
 		if err != nil {
 			s.logger.Errorf("Failed to initialize client: %v", err)