Merge pull request #7 from munnerz/u-exp-backoff

Add exponential back-off when issuance fails

Author: jetstack-bot

manager/manager.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 	"time"
@@ -460,6 +461,15 @@ func (m *Manager) ManageVolume(volumeID string) error {
 	stopCh := make(chan struct{})
 	m.managedVolumes[volumeID] = stopCh
 
+	// Create a context that will be cancelled when the stopCh is closed
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		select {
+		case <-stopCh:
+			cancel()
+		}
+	}()
+
 	go func() {
 		// check every volume once per second
 		// TODO: optimise this to not check so often
@@ -477,13 +487,24 @@ func (m *Manager) ManageVolume(volumeID string) error {
 				}
 
 				if meta.NextIssuanceTime == nil || m.clock.Now().After(*meta.NextIssuanceTime) {
-					log.Info("Triggering new issuance")
-					if err := m.issue(volumeID); err != nil {
-						log.Error(err, "Failed to issue certificate")
-						// retry the request in 1 second time
-						// TODO: exponentially back-off
-						continue
-					}
+					wait.ExponentialBackoffWithContext(ctx, wait.Backoff{
+						Duration: time.Second * 2,
+						Factor:   2.0,
+						Jitter:   0.5,
+						// Set this to the maximum int value to avoid resetting the exponential backoff
+						// timer early.
+						// This will mean that once the back-off hits 1 minute, we will constantly retry once
+						// per minute rather than resetting back to `Duration` (2s).
+						Steps:    math.MaxInt32,
+						Cap:      time.Minute,
+					}, func() (bool, error) {
+						log.Info("Triggering new issuance")
+						if err := m.issue(volumeID); err != nil {
+							log.Error(err, "Failed to issue certificate, retrying after applying exponential backoff")
+							return false, nil
+						}
+						return true, nil
+					})
 				}
 			}
 		}