feat: exponential backoff for flagd-proxy reconciliation

Signed-off-by: Matthias Riegler <[email protected]>

Author: xvzf

common/utils/utils.go

@@ -3,6 +3,8 @@ package utils
 import (
 	"fmt"
 	"strings"
+	"sync/atomic"
+	"time"
 )
 
 func TrueVal() *bool {
@@ -41,3 +43,23 @@ func FeatureFlagId(namespace, name string) string {
 func FeatureFlagConfigMapKey(namespace, name string) string {
 	return fmt.Sprintf("%s.flagd.json", FeatureFlagId(namespace, name))
 }
+
+type ExponentialBackoff struct {
+	StartDelay time.Duration
+	MaxDelay   time.Duration
+	counter    int64
+}
+
+func (e *ExponentialBackoff) Next() time.Duration {
+	val := atomic.AddInt64(&e.counter, 1)
+
+	delay := e.StartDelay * (1 << (val - 1))
+	if delay > e.MaxDelay {
+		delay = e.MaxDelay
+	}
+	return delay
+}
+
+func (e *ExponentialBackoff) Reset() {
+	e.counter = 0
+}

common/utils/utils_test.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
@@ -39,3 +40,47 @@ func Test_ParseAnnotations(t *testing.T) {
 	require.Equal(t, "default", s1)
 	require.Equal(t, "anno", s2)
 }
+
+func TestExponentialBackoff_Next(t *testing.T) {
+	tests := []struct {
+		name       string
+		startDelay time.Duration
+		maxDelay   time.Duration
+		steps      int
+		expected   time.Duration
+	}{
+		{name: "basic backoff", startDelay: 1 * time.Second, maxDelay: 16 * time.Second, steps: 3, expected: 4 * time.Second},
+		{name: "max delay reached", startDelay: 1 * time.Second, maxDelay: 5 * time.Second, steps: 10, expected: 5 * time.Second},
+		{name: "single step", startDelay: 500 * time.Millisecond, maxDelay: 10 * time.Second, steps: 1, expected: 500 * time.Millisecond},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			backoff := &ExponentialBackoff{StartDelay: tt.startDelay, MaxDelay: tt.maxDelay}
+			var result time.Duration
+			for i := 0; i < tt.steps; i++ {
+				result = backoff.Next()
+			}
+			if result != tt.expected {
+				t.Errorf("Expected delay after %d steps to be %v; got %v", tt.steps, tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestExponentialBackoff_Reset(t *testing.T) {
+	backoff := &ExponentialBackoff{StartDelay: 1 * time.Second, MaxDelay: 10 * time.Second}
+
+	// Increment the backoff a few times
+	backoff.Next()
+	backoff.Next()
+
+	// Reset and check the counter
+	backoff.Reset()
+	if backoff.counter != 0 {
+		t.Errorf("Expected counter to be reset to 0; got %d", backoff.counter)
+	}
+	if backoff.Next() != 1*time.Second {
+		t.Errorf("Expected delay after reset to be %v; got %v", 1*time.Second, backoff.Next())
+	}
+}

controllers/core/featureflagsource/controller.go

@@ -26,22 +26,27 @@ import (
 	api "github.com/open-feature/open-feature-operator/apis/core/v1beta1"
 	"github.com/open-feature/open-feature-operator/common"
 	"github.com/open-feature/open-feature-operator/common/flagdproxy"
+	"github.com/open-feature/open-feature-operator/common/utils"
 	appsV1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 // FeatureFlagSourceReconciler reconciles a FeatureFlagSource object
 type FeatureFlagSourceReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 	// ReqLogger contains the Logger of this controller
-	Log        logr.Logger
-	FlagdProxy *flagdproxy.FlagdProxyHandler
+	Log logr.Logger
+
+	// FlagdProxy is the handler for the flagd-proxy deployment
+	FlagdProxy        *flagdproxy.FlagdProxyHandler
+	FlagdProxyBackoff *utils.ExponentialBackoff
 }
 
 // renovate: datasource=github-tags depName=open-feature/flagd/flagd-proxy
@@ -73,13 +78,21 @@ func (r *FeatureFlagSourceReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		return r.finishReconcile(err, false)
 	}
 
+	needsFlagdProxy := false
 	for _, source := range fsConfig.Spec.Sources {
 		if source.Provider.IsFlagdProxy() {
-			r.Log.Info(fmt.Sprintf("featureflagsource %s uses flagd-proxy, checking deployment", req.NamespacedName))
-			if err := r.FlagdProxy.HandleFlagdProxy(ctx); err != nil {
-				r.Log.Error(err, "error handling the flagd-proxy deployment")
-			}
-			break
+			r.Log.Info(fmt.Sprintf("featureflagsource %s requires flagd-proxy", req.NamespacedName))
+			needsFlagdProxy = true
+		}
+	}
+
+	if needsFlagdProxy {
+		r.Log.Info(fmt.Sprintf("featureflagsource %s uses flagd-proxy, checking deployment", req.NamespacedName))
+		if err := r.FlagdProxy.HandleFlagdProxy(ctx); err != nil {
+			r.Log.Error(err, "error handling the flagd-proxy deployment")
+			return reconcile.Result{RequeueAfter: r.FlagdProxyBackoff.Next()}, err
+		} else {
+			r.FlagdProxyBackoff.Reset()
 		}
 	}
 

controllers/core/featureflagsource/controller_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/open-feature/open-feature-operator/common"
 	"github.com/open-feature/open-feature-operator/common/flagdproxy"
 	commontypes "github.com/open-feature/open-feature-operator/common/types"
+	"github.com/open-feature/open-feature-operator/common/utils"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -113,10 +114,11 @@ func TestFeatureFlagSourceReconciler_Reconcile(t *testing.T) {
 			)
 
 			r := &FeatureFlagSourceReconciler{
-				Client:     fakeClient,
-				Log:        ctrl.Log.WithName("featureflagsource-controller"),
-				Scheme:     fakeClient.Scheme(),
-				FlagdProxy: kph,
+				Client:            fakeClient,
+				Log:               ctrl.Log.WithName("featureflagsource-controller"),
+				Scheme:            fakeClient.Scheme(),
+				FlagdProxy:        kph,
+				FlagdProxyBackoff: &utils.ExponentialBackoff{StartDelay: time.Duration(0), MaxDelay: time.Duration(0)},
 			}
 
 			if tt.deployment != nil {

controllers/core/flagd/controller_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	api "github.com/open-feature/open-feature-operator/apis/core/v1beta1"
-	"github.com/open-feature/open-feature-operator/controllers/core/flagd/common"
+	resources "github.com/open-feature/open-feature-operator/controllers/core/flagd/common"
 	commonmock "github.com/open-feature/open-feature-operator/controllers/core/flagd/mock"
 	resourcemock "github.com/open-feature/open-feature-operator/controllers/core/flagd/resources/mock"
 	"github.com/stretchr/testify/require"

main.go

@@ -23,13 +23,15 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/kelseyhightower/envconfig"
 	corev1beta1 "github.com/open-feature/open-feature-operator/apis/core/v1beta1"
 	"github.com/open-feature/open-feature-operator/common"
 	"github.com/open-feature/open-feature-operator/common/flagdinjector"
 	"github.com/open-feature/open-feature-operator/common/flagdproxy"
 	"github.com/open-feature/open-feature-operator/common/types"
+	"github.com/open-feature/open-feature-operator/common/utils"
 	"github.com/open-feature/open-feature-operator/controllers/core/featureflagsource"
 	"github.com/open-feature/open-feature-operator/controllers/core/flagd"
 	flagdresources "github.com/open-feature/open-feature-operator/controllers/core/flagd/resources"
@@ -228,6 +230,10 @@ func main() {
 		Scheme:     mgr.GetScheme(),
 		Log:        ctrl.Log.WithName("FeatureFlagSource Controller"),
 		FlagdProxy: kph,
+		FlagdProxyBackoff: &utils.ExponentialBackoff{
+			StartDelay: time.Second,
+			MaxDelay:   time.Minute,
+		},
 	}
 	if err = flagSourceController.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FeatureFlagSource")