fix: handle multiple imagePullSecrets (#666)

toddbaert · web-flow · commit df3d6d922a26 · 2024-06-06T08:52:20.000-04:00
Signed-off-by: Todd Baert &lt;todd.baert@dynatrace.com&gt;
diff --git a/.github/scripts/strip-kustomize-helm.sh b/.github/scripts/strip-kustomize-helm.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# This script is a hack to support helm flow control in kustomize overlays, which would otherwise break them.
+# It allows us to render helm template bindings and add newlines.
+# For instance, it transforms "___{{ .Value.myValue }}___" to {{ .Value.myValue }}.
+# It also adds newlines wherever ___newline___ is found.
+
+CHARTS_DIR='./chart/open-feature-operator/templates';
+
+echo 'Running strip-kustomize-helm.sh script'
+filenames=`find $CHARTS_DIR -name "*.yaml"`
+for file in $filenames; do
+    sed -i "s/___newline___/\\n/g" $file
+    sed -i "s/\"___//g" $file
+    sed -i "s/___\"//g" $file
+    sed -i "s/___//g" $file
+done
+echo 'Done running strip-kustomize-helm.sh script'
diff --git a/Makefile b/Makefile
@@ -252,6 +252,7 @@ set-helm-overlay:
 helm-package: set-helm-overlay generate release-manifests helm
 	mkdir -p chart/open-feature-operator/templates/crds
 	mv chart/open-feature-operator/templates/*customresourcedefinition* chart/open-feature-operator/templates/crds
+	sh .github/scripts/strip-kustomize-helm.sh
 	$(HELM) package --version $(CHART_VERSION) chart/open-feature-operator
 	mkdir -p charts && mv open-feature-operator-*.tgz charts
 	$(HELM) repo index --url https://open-feature.github.io/open-feature-operator/charts charts
diff --git a/chart/open-feature-operator/README.md b/chart/open-feature-operator/README.md
@@ -93,10 +93,10 @@ The command removes all the Kubernetes components associated with the chart and
 
 ### Global
 
-| Name               | Description                                                                                                                                                                                                  | Value                          |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------ |
-| `defaultNamespace` | To override the namespace use the `--namespace` flag. This default is provided to ensure that the kustomize build charts in `/templates` deploy correctly when no `namespace` is provided via the `-n` flag. | `open-feature-operator-system` |
-| `imagePullSecret`  | Secret containing credentials for images pulled by the operator (flagdProxyConfiguration.image, flagdConfiguration.image, controllerManager.manager.image, controllerManager.kubeRbacProxy.image).           | `""`                           |
+| Name               | Description                                                                                                                                                                                                                                                                      | Value                          |
+| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ |
+| `defaultNamespace` | To override the namespace use the `--namespace` flag. This default is provided to ensure that the kustomize build charts in `/templates` deploy correctly when no `namespace` is provided via the `-n` flag.                                                                     | `open-feature-operator-system` |
+| `imagePullSecrets` | Array of ImagePullSecret objects containing credentials for images pulled by the operator (flagdProxyConfiguration.image, flagdConfiguration.image, controllerManager.manager.image, controllerManager.kubeRbacProxy.image). Example: imagePullSecrets: [{"name": "my-secret"}]  | `[]`                           |
 
 ### Sidecar configuration
 
diff --git a/chart/open-feature-operator/values.yaml b/chart/open-feature-operator/values.yaml
@@ -2,8 +2,8 @@
 ## @section Global
 ## @param  defaultNamespace To override the namespace use the `--namespace` flag. This default is provided to ensure that the kustomize build charts in `/templates` deploy correctly when no `namespace` is provided via the `-n` flag.
 defaultNamespace: open-feature-operator-system
-## @param imagePullSecret Secret containing credentials for images pulled by the operator (flagdProxyConfiguration.image, flagdConfiguration.image, controllerManager.manager.image, controllerManager.kubeRbacProxy.image).
-imagePullSecret: ""
+## @param imagePullSecrets Array of ImagePullSecret objects containing credentials for images pulled by the operator (flagdProxyConfiguration.image, flagdConfiguration.image, controllerManager.manager.image, controllerManager.kubeRbacProxy.image). Example: imagePullSecrets: [{"name": "my-secret"}] 
+imagePullSecrets: []
 
 ## @section Sidecar configuration
 sidecarConfiguration:
diff --git a/common/flagdproxy/flagdproxy.go b/common/flagdproxy/flagdproxy.go
@@ -38,10 +38,10 @@ type FlagdProxyConfiguration struct {
 	Tag                    string
 	Namespace              string
 	OperatorDeploymentName string
-	ImagePullSecret        string
+	ImagePullSecrets       []string
 }
 
-func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecret string) *FlagdProxyConfiguration {
+func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecrets []string) *FlagdProxyConfiguration {
 	return &FlagdProxyConfiguration{
 		Image:                  env.FlagdProxyImage,
 		Tag:                    env.FlagdProxyTag,
@@ -50,7 +50,7 @@ func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecret string) *Fl
 		Port:                   env.FlagdProxyPort,
 		ManagementPort:         env.FlagdProxyManagementPort,
 		DebugLogging:           env.FlagdProxyDebugLogging,
-		ImagePullSecret:        imagePullSecret,
+		ImagePullSecrets:       imagePullSecrets,
 	}
 }
 
@@ -146,9 +146,9 @@ func (f *FlagdProxyHandler) newFlagdProxyManifest(ownerReference *metav1.OwnerRe
 		args = append(args, "--debug")
 	}
 	imagePullSecrets := []corev1.LocalObjectReference{}
-	if f.config.ImagePullSecret != "" {
+	for _, secret := range f.config.ImagePullSecrets {
 		imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{
-			Name: f.config.ImagePullSecret,
+			Name: secret,
 		})
 	}
 
diff --git a/common/flagdproxy/flagdproxy_test.go b/common/flagdproxy/flagdproxy_test.go
@@ -19,22 +19,22 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
-const pullSecret = "test-pullSecret"
+var pullSecrets = []string{"test-pullSecret"}
 
 func TestNewFlagdProxyConfiguration(t *testing.T) {
 
 	kpConfig := NewFlagdProxyConfiguration(types.EnvConfig{
 		FlagdProxyPort:           8015,
 		FlagdProxyManagementPort: 8016,
-	}, pullSecret)
+	}, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 	require.Equal(t, &FlagdProxyConfiguration{
 		Port:                   8015,
 		ManagementPort:         8016,
 		DebugLogging:           false,
 		OperatorDeploymentName: common.OperatorDeploymentName,
-		ImagePullSecret:        pullSecret,
+		ImagePullSecrets:       pullSecrets,
 	}, kpConfig)
 }
 
@@ -48,7 +48,7 @@ func TestNewFlagdProxyConfiguration_OverrideEnvVars(t *testing.T) {
 		FlagdProxyDebugLogging:   true,
 	}
 
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 	require.Equal(t, &FlagdProxyConfiguration{
@@ -59,12 +59,12 @@ func TestNewFlagdProxyConfiguration_OverrideEnvVars(t *testing.T) {
 		Tag:                    "my-tag",
 		Namespace:              "my-namespace",
 		OperatorDeploymentName: common.OperatorDeploymentName,
-		ImagePullSecret:        pullSecret,
+		ImagePullSecrets:       pullSecrets,
 	}, kpConfig)
 }
 
 func TestNewFlagdProxyHandler(t *testing.T) {
-	kpConfig := NewFlagdProxyConfiguration(types.EnvConfig{}, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(types.EnvConfig{}, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -100,7 +100,7 @@ func TestDoesFlagdProxyExist(t *testing.T) {
 		},
 	}
 
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -128,7 +128,7 @@ func TestFlagdProxyHandler_HandleFlagdProxy_ProxyExistsWithBadVersion(t *testing
 	env := types.EnvConfig{
 		PodNamespace: "ns",
 	}
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -187,7 +187,7 @@ func TestFlagdProxyHandler_HandleFlagdProxy_ProxyExistsWithoutLabel(t *testing.T
 	env := types.EnvConfig{
 		PodNamespace: "ns",
 	}
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -236,7 +236,7 @@ func TestFlagdProxyHandler_HandleFlagdProxy_ProxyExistsWithNewestVersion(t *test
 	env := types.EnvConfig{
 		PodNamespace: "ns",
 	}
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -280,7 +280,7 @@ func TestFlagdProxyHandler_HandleFlagdProxy_CreateProxy(t *testing.T) {
 		FlagdProxyManagementPort: 90,
 		FlagdProxyDebugLogging:   true,
 	}
-	kpConfig := NewFlagdProxyConfiguration(env, pullSecret)
+	kpConfig := NewFlagdProxyConfiguration(env, pullSecrets)
 
 	require.NotNil(t, kpConfig)
 
@@ -362,7 +362,7 @@ func TestFlagdProxyHandler_HandleFlagdProxy_CreateProxy(t *testing.T) {
 				Spec: corev1.PodSpec{
 					ServiceAccountName: FlagdProxyServiceAccountName,
 					ImagePullSecrets: []corev1.LocalObjectReference{
-						{Name: pullSecret},
+						{Name: pullSecrets[0]},
 					},
 					Containers: []corev1.Container{
 						{
diff --git a/config/overlays/helm/manager.yaml b/config/overlays/helm/manager.yaml
@@ -7,8 +7,8 @@ spec:
   replicas: 0{{ .Values.controllerManager.replicas }}
   template:
     spec:
-      imagePullSecrets:
-        - name: "{{ .Values.imagePullSecret }}"
+      # this is transformed by .github/scripts/strip-kustomize-helm.sh
+      ___imagePullSecrets___: "___ ___newline___{{ toYaml .Values.imagePullSecrets | indent 8 }}___"
       containers:
         - name: manager
           image: "{{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }}"
@@ -92,7 +92,7 @@ spec:
             - --sidecar-ram-limit={{ .Values.sidecarConfiguration.resources.limits.memory }}
             - --sidecar-cpu-request={{ .Values.sidecarConfiguration.resources.requests.cpu }}
             - --sidecar-ram-request={{ .Values.sidecarConfiguration.resources.requests.memory }}
-            - --image-pull-secret={{ .Values.imagePullSecret }}
+            - --image-pull-secrets={{ range .Values.imagePullSecrets }}{{ .name }},{{- end }}
         - name: kube-rbac-proxy
           image: "{{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag }}"
           resources:
diff --git a/controllers/core/featureflagsource/controller_test.go b/controllers/core/featureflagsource/controller_test.go
@@ -27,8 +27,8 @@ func TestFeatureFlagSourceReconciler_Reconcile(t *testing.T) {
 		testNamespace  = "test-namespace"
 		fsConfigName   = "test-config"
 		deploymentName = "test-deploy"
-		pullSecret     = "test-pullsecret"
 	)
+	var pullSecrets = []string{"test-pullsecret"}
 
 	tests := []struct {
 		name                            string
@@ -93,7 +93,7 @@ func TestFeatureFlagSourceReconciler_Reconcile(t *testing.T) {
 			kpConfig := flagdproxy.NewFlagdProxyConfiguration(commontypes.EnvConfig{
 				FlagdProxyImage: "ghcr.io/open-feature/flagd-proxy",
 				FlagdProxyTag:   flagdProxyTag,
-			}, pullSecret)
+			}, pullSecrets)
 
 			kpConfig.Namespace = testNamespace
 			kph := flagdproxy.NewFlagdProxyHandler(
diff --git a/controllers/core/flagd/common/common.go b/controllers/core/flagd/common/common.go
@@ -1,14 +1,14 @@
 package resources
 
 type FlagdConfiguration struct {
-	FlagdPort       int
-	OFREPPort       int
-	SyncPort        int
-	ManagementPort  int
-	DebugLogging    bool
-	Image           string
-	Tag             string
-	ImagePullSecret string
+	FlagdPort        int
+	OFREPPort        int
+	SyncPort         int
+	ManagementPort   int
+	DebugLogging     bool
+	Image            string
+	Tag              string
+	ImagePullSecrets []string
 
 	OperatorNamespace      string
 	OperatorDeploymentName string
diff --git a/controllers/core/flagd/config.go b/controllers/core/flagd/config.go
@@ -6,7 +6,7 @@ import (
 	resources "github.com/open-feature/open-feature-operator/controllers/core/flagd/common"
 )
 
-func NewFlagdConfiguration(env types.EnvConfig, imagePullSecret string) resources.FlagdConfiguration {
+func NewFlagdConfiguration(env types.EnvConfig, imagePullSecrets []string) resources.FlagdConfiguration {
 	return resources.FlagdConfiguration{
 		Image:                  env.FlagdImage,
 		Tag:                    env.FlagdTag,
@@ -16,6 +16,6 @@ func NewFlagdConfiguration(env types.EnvConfig, imagePullSecret string) resource
 		SyncPort:               env.FlagdSyncPort,
 		ManagementPort:         env.FlagdManagementPort,
 		DebugLogging:           env.FlagdDebugLogging,
-		ImagePullSecret:        imagePullSecret,
+		ImagePullSecrets:       imagePullSecrets,
 	}
 }
diff --git a/controllers/core/flagd/resources/deployment.go b/controllers/core/flagd/resources/deployment.go
@@ -78,9 +78,9 @@ func (r *FlagdDeployment) GetResource(ctx context.Context, flagd *api.Flagd) (cl
 
 	featureFlagSource := &api.FeatureFlagSource{}
 	imagePullSecrets := []corev1.LocalObjectReference{}
-	if r.FlagdConfig.ImagePullSecret != "" {
+	for _, secret := range r.FlagdConfig.ImagePullSecrets {
 		imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{
-			Name: r.FlagdConfig.ImagePullSecret,
+			Name: secret,
 		})
 	}
 
diff --git a/main.go b/main.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/kelseyhightower/envconfig"
 	corev1beta1 "github.com/open-feature/open-feature-operator/apis/core/v1beta1"
@@ -65,8 +66,8 @@ const (
 	sidecarRamLimitDefault         = "64M"
 	sidecarCpuRequestDefault       = "0.2"
 	sidecarRamRequestDefault       = "32M"
-	imagePullSecretFlagName        = "image-pull-secret"
-	imagePullSecretDefault         = ""
+	imagePullSecretFlagName        = "image-pull-secrets"
+	imagePullSecretFlagDefault     = ""
 )
 
 var (
@@ -77,7 +78,7 @@ var (
 	probeAddr                                                              string
 	verbose                                                                bool
 	sidecarCpuLimit, sidecarRamLimit, sidecarCpuRequest, sidecarRamRequest string
-	imagePullSecret                                                        string
+	imagePullSecrets                                                       string
 )
 
 func init() {
@@ -105,8 +106,7 @@ func main() {
 	flag.StringVar(&sidecarRamLimit, sidecarRamLimitFlagName, sidecarRamLimitDefault, "sidecar memory limit, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024)")
 	flag.StringVar(&sidecarCpuRequest, sidecarCpuRequestFlagName, sidecarCpuRequestDefault, "sidecar CPU minimum, in cores. (500m = .5 cores)")
 	flag.StringVar(&sidecarRamRequest, sidecarRamRequestFlagName, sidecarRamRequestDefault, "sidecar memory minimum, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024)")
-
-	flag.StringVar(&imagePullSecret, imagePullSecretFlagName, imagePullSecretDefault, "secret containing credentials to pull images.")
+	flag.StringVar(&imagePullSecrets, imagePullSecretFlagName, imagePullSecretFlagDefault, "Comma-delimited list of secrets containing credentials to pull images.")
 
 	flag.Parse()
 
@@ -183,7 +183,7 @@ func main() {
 	}
 
 	kph := flagdproxy.NewFlagdProxyHandler(
-		flagdproxy.NewFlagdProxyConfiguration(env, imagePullSecret),
+		flagdproxy.NewFlagdProxyConfiguration(env, strings.Split(imagePullSecrets, ",")),
 		mgr.GetClient(),
 		ctrl.Log.WithName("FeatureFlagSource FlagdProxyHandler"),
 	)
@@ -215,7 +215,7 @@ func main() {
 		Scheme: mgr.GetScheme(),
 		Log:    flagdControllerLogger,
 	}
-	flagdConfig := flagd.NewFlagdConfiguration(env, imagePullSecret)
+	flagdConfig := flagd.NewFlagdConfiguration(env, strings.Split(imagePullSecrets, ","))
 
 	if err = (&flagd.FlagdReconciler{
 		Client:             mgr.GetClient(),

Original file line number	Diff line number	Diff line change
`@@ -38,10 +38,10 @@ type FlagdProxyConfiguration struct {`
`38`	`38`	`Tag string`
`39`	`39`	`Namespace string`
`40`	`40`	`OperatorDeploymentName string`
`41`		`- ImagePullSecret string`
	`41`	`+ ImagePullSecrets []string`
`42`	`42`	`}`
`43`	`43`
`44`		`-func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecret string) *FlagdProxyConfiguration {`
	`44`	`+func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecrets []string) *FlagdProxyConfiguration {`
`45`	`45`	`return &FlagdProxyConfiguration{`
`46`	`46`	`Image: env.FlagdProxyImage,`
`47`	`47`	`Tag: env.FlagdProxyTag,`
`@@ -50,7 +50,7 @@ func NewFlagdProxyConfiguration(env types.EnvConfig, imagePullSecret string) *Fl`
`50`	`50`	`Port: env.FlagdProxyPort,`
`51`	`51`	`ManagementPort: env.FlagdProxyManagementPort,`
`52`	`52`	`DebugLogging: env.FlagdProxyDebugLogging,`
`53`		`- ImagePullSecret: imagePullSecret,`
	`53`	`+ ImagePullSecrets: imagePullSecrets,`
`54`	`54`	`}`
`55`	`55`	`}`
`56`	`56`
`@@ -146,9 +146,9 @@ func (f FlagdProxyHandler) newFlagdProxyManifest(ownerReference metav1.OwnerRe`
`146`	`146`	`args = append(args, "--debug")`
`147`	`147`	`}`
`148`	`148`	`imagePullSecrets := []corev1.LocalObjectReference{}`
`149`		`- if f.config.ImagePullSecret != "" {`
	`149`	`+ for _, secret := range f.config.ImagePullSecrets {`
`150`	`150`	`imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{`
`151`		`- Name: f.config.ImagePullSecret,`
	`151`	`+ Name: secret,`
`152`	`152`	`})`
`153`	`153`	`}`
`154`	`154`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ import (`
`6`	`6`	`resources "github.com/open-feature/open-feature-operator/controllers/core/flagd/common"`
`7`	`7`	`)`
`8`	`8`
`9`		`-func NewFlagdConfiguration(env types.EnvConfig, imagePullSecret string) resources.FlagdConfiguration {`
	`9`	`+func NewFlagdConfiguration(env types.EnvConfig, imagePullSecrets []string) resources.FlagdConfiguration {`
`10`	`10`	`return resources.FlagdConfiguration{`
`11`	`11`	`Image: env.FlagdImage,`
`12`	`12`	`Tag: env.FlagdTag,`
`@@ -16,6 +16,6 @@ func NewFlagdConfiguration(env types.EnvConfig, imagePullSecret string) resource`
`16`	`16`	`SyncPort: env.FlagdSyncPort,`
`17`	`17`	`ManagementPort: env.FlagdManagementPort,`
`18`	`18`	`DebugLogging: env.FlagdDebugLogging,`
`19`		`- ImagePullSecret: imagePullSecret,`
	`19`	`+ ImagePullSecrets: imagePullSecrets,`
`20`	`20`	`}`
`21`	`21`	`}`