merge

Signed-off-by: James Milligan <[email protected]>

Author: james-milligan

Makefile

@@ -4,7 +4,7 @@ IMG ?= controller:latest
 # customize overlay to be used in the build, DEFAULT or HELM
 KUSTOMIZE_OVERLAY ?= DEFAULT
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-FLAGD_VERSION=v0.3.0
+FLAGD_VERSION=v0.3.1
 CHART_VERSION=v0.2.23# x-release-please-version
 ENVTEST_K8S_VERSION = 1.25
 

docs/getting_started.md

@@ -72,7 +72,7 @@ kubectl describe pod busybox-curl-7bd5767999-spf7v
 ```
 ```yaml
   flagd:
-    Image:         ghcr.io/open-feature/flagd:vX.X.X
+    Image:         ghcr.io/open-feature/flagd:v0.3.1
     Port:          8014/TCP
     Host Port:     0/TCP
     Args:

docs/permissions.md

@@ -44,7 +44,8 @@ The `proxy-role` definition can be found [here](../config/rbac/auth_proxy_role.y
 ### Flagd Kubernetes Sync
 
 The `flagd-kubernetes-sync` role providers the permission to get, watch and list all `core.openfeature.dev` resources, permitting the kubernetes sync feature in injected `flagd` containers.
-Its definition can be found [here](../config/rbac/flagd_kubernetes_sync_clusterrole.yaml)
+Its definition can be found [here](../config/rbac/flagd_kubernetes_sync_clusterrole.yaml). 
+During startup the operator will backfill permissions to the `flagd-kubernetes-sync` cluster role binding from the current state of the cluster, adding all service accounts from pods with the `core.openfeature.dev/enabled` annotation set to `"true"`, preventing unexpected behavior during upgrades.
 
 | API Group      | Resource | Verbs |
 | ----------- | ----------- | ----------- |

main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
@@ -131,6 +132,17 @@ func main() {
 		os.Exit(1)
 	}
 
+	// setup indexer for backfilling permissions on the flagd-kubernetes-sync role binding
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&corev1.Pod{},
+		webhooks.OpenFeatureEnabledAnnotationPath,
+		webhooks.OpenFeatureEnabledAnnotationIndex,
+	); err != nil {
+		setupLog.Error(err, "unable to create indexer", "webhook", webhooks.OpenFeatureEnabledAnnotationPath)
+		os.Exit(1)
+	}
+
 	if err = (&controllers.FeatureFlagConfigurationReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
@@ -181,10 +193,7 @@ func main() {
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("mutating-pod-webhook"),
 	}
-	podMutator.BackfillPermissions()
-	hookServer.Register("/mutate-v1-pod", &webhook.Admission{
-		Handler: podMutator,
-	})
+	hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: podMutator})
 	hookServer.Register("/validate-v1alpha1-featureflagconfiguration", &webhook.Admission{Handler: &webhooks.FeatureFlagConfigurationValidator{
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("validating-featureflagconfiguration-webhook"),
@@ -200,7 +209,19 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	ctx := ctrl.SetupSignalHandler()
+	errChan := make(chan error, 1)
+	go func(chan error) {
+		if err := mgr.Start(ctx); err != nil {
+			errChan <- err
+		}
+	}(errChan)
+
+	setupLog.Info("restoring flagd-kubernetes-sync cluster role binding subjects from current cluster state")
+	// backfill can be handled asynchronously, so we do not need to block via the channel
+	go podMutator.BackfillPermissions(ctx, make(chan struct{}, 1))
+
+	if err := <-errChan; err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

renovate.json

@@ -2,5 +2,19 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
     "config:base"
+  ],
+  "regexManagers": [
+    {
+      "fileMatch": ["^Makefile$"],
+      "matchStrings": ["FLAGD_VERSION=(?<currentValue>.*?)\\n"],
+      "depNameTemplate": "open-feature/flagd",
+      "datasourceTemplate": "github-releases"
+    },
+    {
+      "fileMatch": ["^docs/getting_started.md$"],
+      "matchStrings": ["ghcr\\.io\\/open-feature\\/flagd:(?<currentValue>.*?)\\n"],
+      "depNameTemplate": "open-feature/flagd",
+      "datasourceTemplate": "github-releases"
+    }
   ]
 }

webhooks/pod_webhook.go

@@ -7,6 +7,9 @@ import (
 	"net/http"
 	"reflect"
 	"strings"
+	"time"
+
+	goErr "errors"
 
 	"github.com/go-logr/logr"
 	corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1"
@@ -15,15 +18,18 @@ import (
 	v1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
 // we likely want these to be configurable, eventually
 const (
-	FlagDImagePullPolicy   corev1.PullPolicy = "Always"
-	clusterRoleBindingName string            = "open-feature-operator-flagd-kubernetes-sync"
-	fileSyncMountPath      string            = "/etc/flagd/"
+	FlagDImagePullPolicy             corev1.PullPolicy = "Always"
+	clusterRoleBindingName           string            = "open-feature-operator-flagd-kubernetes-sync"
+	flagdMetricPortEnvVar            string            = "FLAGD_METRICS_PORT"
+	fileSyncMountPath                string            = "/etc/flagd/"
+	OpenFeatureEnabledAnnotationPath                   = "metadata.annotations.openfeature.dev/enabled"
 )
 
 // NOTE: RBAC not needed here.
@@ -42,9 +48,45 @@ type PodMutator struct {
 	Log                       logr.Logger
 }
 
-func (m *PodMutator) BackfillPermissions() {
-	podList := &corev1.PodList{}
-	err := m.Client.List(context.Background(), podList, client.MatchingLabels{"somelabel": "someval"})
+// BackfillPermissions recovers the state of the flagd-kubernetes-sync role binding in the event of upgrade
+func (m *PodMutator) BackfillPermissions(ctx context.Context, backfillComplete chan struct{}) {
+	defer func() {
+		backfillComplete <- struct{}{}
+	}()
+
+	for i := 0; i < 5; i++ {
+		// fetch all pods with the "openfeature.dev/enabled" annotation set to "true"
+		podList := &corev1.PodList{}
+		err := m.Client.List(ctx, podList, client.MatchingFields{OpenFeatureEnabledAnnotationPath: "true"})
+		if err != nil {
+			if !goErr.Is(err, &cache.ErrCacheNotStarted{}) {
+				m.Log.Error(err, "unable to list annotated pods", "webhook", OpenFeatureEnabledAnnotationPath)
+				return
+			}
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		// add each new service account to the flagd-kubernetes-sync role binding
+		for _, pod := range podList.Items {
+			m.Log.V(1).Info(fmt.Sprintf("backfilling permissions for pod %s/%s", pod.Namespace, pod.Name))
+			if err := m.enableClusterRoleBinding(ctx, &pod); err != nil {
+				m.Log.Error(
+					err,
+					fmt.Sprintf("unable backfill permissions for pod %s/%s", pod.Namespace, pod.Name),
+					"webhook",
+					OpenFeatureEnabledAnnotationPath,
+				)
+			}
+		}
+		return
+	}
+	err := goErr.New("unable to backfill permissions for the flagd-kubernetes-sync role binding: timeout")
+	m.Log.Error(
+		err,
+		"webhook",
+		OpenFeatureEnabledAnnotationPath,
+	)
 }
 
 // Handle injects the flagd sidecar (if the prerequisites are all met)
@@ -429,3 +471,27 @@ func setSecurityContext() *corev1.SecurityContext {
 		},
 	}
 }
+
+func OpenFeatureEnabledAnnotationIndex(o client.Object) []string {
+	pod := o.(*corev1.Pod)
+	if pod.ObjectMeta.Annotations == nil {
+		return []string{
+			"false",
+		}
+	}
+	val, ok := pod.ObjectMeta.Annotations["openfeature.dev/enabled"]
+	if ok && val == "true" {
+		return []string{
+			"true",
+		}
+	}
+	val, ok = pod.ObjectMeta.Annotations["openfeature.dev"]
+	if ok && val == "enabled" {
+		return []string{
+			"true",
+		}
+	}
+	return []string{
+		"false",
+	}
+}