fix: added wait functionality for portal secret (#123)

* refactor(initializer): improve kubeconfig handling

• Load kubeconfig from file to enhance flexibility
• Update manager initialization to use the new configuration

Signed-off-by: Bastian Echterhölter <[email protected]>
On-behalf-of: @SAP <[email protected]>

* refactor(config): enhance configuration structure for KCP

• Introduces KCP kubeconfig handling in the config
• Simplifies lifecycle manager import in the initializer

Signed-off-by: Bastian Echterhölter <[email protected]>
On-behalf-of: @SAP <[email protected]>

* refactor(generator and operator): separated kcp and runtime kubeconfigs

On-behalf-of: SAP [email protected]

* fix: fixed getting config function

On-behalf-of: SAP [email protected]

* fix: removed extra error handler

On-behalf-of: SAP [email protected]

* fix: added wait functionaly for portal secret

On-behalf-of: SAP [email protected]

* chore: removed log

On-behalf-of: SAP [email protected]

* improved test coverage

On-behalf-of: SAP [email protected]

* chore: fixed typos

On-behalf-of: SAP [email protected]

* chore: refactored tests

On-behalf-of: SAP [email protected]

* feat: added timeout for secret waiting

On-behalf-of: SAP [email protected]

* chore: made timeout configurable by config

On-behalf-of: SAP [email protected]

* fix: fixed merge errors

On-behalf-of: SAP [email protected]

---------

Signed-off-by: Bastian Echterhölter <[email protected]>
Co-authored-by: Bastian Echterhölter <[email protected]>

Author: OlegErshov

cmd/initializer.go

@@ -31,7 +31,11 @@ var initializerCmd = &cobra.Command{
 		ctx, _, shutdown := pmcontext.StartContext(log, initializerCfg, defaultCfg.ShutdownTimeout)
 		defer shutdown()
 
-		mgrCfg := ctrl.GetConfigOrDie()
+		restCfg, err := getKubeconfigFromPath(initializerCfg.KCP.Kubeconfig)
+		if err != nil {
+			log.Error().Err(err).Msg("unable to get KCP kubeconfig")
+			os.Exit(1)
+		}
 
 		mgrOpts := ctrl.Options{
 			Scheme:                 scheme,
@@ -57,7 +61,7 @@ var initializerCmd = &cobra.Command{
 			mgrOpts.LeaderElectionConfig = inClusterCfg
 		}
 
-		provider, err := initializingworkspaces.New(mgrCfg, initializingworkspaces.Options{
+		provider, err := initializingworkspaces.New(restCfg, initializingworkspaces.Options{
 			InitializerName: initializerCfg.InitializerName,
 			Scheme:          mgrOpts.Scheme,
 		})
@@ -66,7 +70,7 @@ var initializerCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		mgr, err := mcmanager.New(mgrCfg, provider, mgrOpts)
+		mgr, err := mcmanager.New(restCfg, provider, mgrOpts)
 		if err != nil {
 			setupLog.Error(err, "Failed to create manager")
 			os.Exit(1)
@@ -82,13 +86,9 @@ var initializerCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		inClusterConfig, err := rest.InClusterConfig()
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to create in cluster config")
-			os.Exit(1)
-		}
+		k8sCfg := ctrl.GetConfigOrDie()
 
-		inClusterClient, err := client.New(inClusterConfig, client.Options{Scheme: scheme})
+		runtimeClient, err := client.New(k8sCfg, client.Options{Scheme: scheme})
 		if err != nil {
 			log.Error().Err(err).Msg("Failed to create in cluster client")
 			os.Exit(1)
@@ -98,7 +98,7 @@ var initializerCmd = &cobra.Command{
 			initializerCfg.IDP.AdditionalRedirectURLs = []string{}
 		}
 
-		if err := controller.NewLogicalClusterReconciler(log, orgClient, initializerCfg, inClusterClient, mgr).
+		if err := controller.NewLogicalClusterReconciler(log, orgClient, initializerCfg, runtimeClient, mgr).
 			SetupWithManager(mgr, defaultCfg); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "LogicalCluster")
 			os.Exit(1)

cmd/model_generator.go

@@ -26,13 +26,16 @@ import (
 var modelGeneratorCmd = &cobra.Command{
 	Use: "model-generator",
 	RunE: func(cmd *cobra.Command, args []string) error {
-
 		ctrl.SetLogger(log.ComponentLogger("controller-runtime").Logr())
 
 		ctx, _, shutdown := platformeshcontext.StartContext(log, defaultCfg, defaultCfg.ShutdownTimeout)
 		defer shutdown()
 
-		cfg := ctrl.GetConfigOrDie()
+		restCfg, err := getKubeconfigFromPath(generatorCfg.KCP.Kubeconfig)
+		if err != nil {
+			log.Error().Err(err).Msg("unable to get KCP kubeconfig")
+			return err
+		}
 
 		mgrOpts := manager.Options{
 			Scheme: scheme,
@@ -67,15 +70,15 @@ var modelGeneratorCmd = &cobra.Command{
 			return fmt.Errorf("scheme should not be nil")
 		}
 
-		provider, err := apiexport.New(cfg, apiexport.Options{
+		provider, err := apiexport.New(restCfg, apiexport.Options{
 			Scheme: mgrOpts.Scheme,
 		})
 		if err != nil {
 			log.Error().Err(err).Msg("Failed to create apiexport provider")
 			return err
 		}
 
-		mgr, err := mcmanager.New(cfg, provider, mgrOpts)
+		mgr, err := mcmanager.New(restCfg, provider, mgrOpts)
 		if err != nil {
 			log.Error().Err(err).Msg("Failed to create manager")
 			return err

cmd/operator.go

@@ -77,6 +77,12 @@ var operatorCmd = &cobra.Command{
 		ctx, _, shutdown := platformeshcontext.StartContext(log, defaultCfg, defaultCfg.ShutdownTimeout)
 		defer shutdown()
 
+		restCfg, err := getKubeconfigFromPath(operatorCfg.KCP.Kubeconfig)
+		if err != nil {
+			log.Error().Err(err).Msg("unable to get KCP kubeconfig")
+			return err
+		}
+
 		if defaultCfg.Sentry.Dsn != "" {
 			err := sentry.Start(ctx,
 				defaultCfg.Sentry.Dsn, defaultCfg.Environment, defaultCfg.Region,
@@ -89,8 +95,6 @@ var operatorCmd = &cobra.Command{
 			defer platformeshcontext.Recover(log)
 		}
 
-		cfg := ctrl.GetConfigOrDie()
-
 		mgrOpts := ctrl.Options{
 			Scheme: scheme,
 			Metrics: metricsserver.Options{
@@ -121,15 +125,15 @@ var operatorCmd = &cobra.Command{
 			return fmt.Errorf("scheme should not be nil")
 		}
 
-		provider, err := apiexport.New(cfg, apiexport.Options{
+		provider, err := apiexport.New(restCfg, apiexport.Options{
 			Scheme: mgrOpts.Scheme,
 		})
 		if err != nil {
 			setupLog.Error(err, "unable to construct cluster provider")
 			return err
 		}
 
-		mgr, err := mcmanager.New(cfg, provider, mgrOpts)
+		mgr, err := mcmanager.New(restCfg, provider, mgrOpts)
 		if err != nil {
 			setupLog.Error(err, "Failed to create manager")
 			return err

cmd/root.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"errors"
 	"flag"
 	"strings"
 
@@ -9,6 +10,8 @@ import (
 	"github.com/platform-mesh/golang-commons/logger"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/platform-mesh/security-operator/internal/config"
@@ -18,6 +21,7 @@ var (
 	defaultCfg     *platformeshconfig.CommonServiceConfig
 	initializerCfg config.Config
 	operatorCfg    config.Config
+	generatorCfg   config.Config
 	log            *logger.Logger
 	setupLog       logr.Logger
 )
@@ -43,6 +47,10 @@ func init() {
 	if err := platformeshconfig.BindConfigToFlags(operatorV, operatorCmd, &operatorCfg); err != nil {
 		panic(err)
 	}
+	generatorV := newViper()
+	if err := platformeshconfig.BindConfigToFlags(generatorV, modelGeneratorCmd, &generatorCfg); err != nil {
+		panic(err)
+	}
 	initializerV := newViper()
 	if err := platformeshconfig.BindConfigToFlags(initializerV, initializerCmd, &initializerCfg); err != nil {
 		panic(err)
@@ -51,6 +59,21 @@ func init() {
 	cobra.OnInitialize(initLog)
 }
 
+func getKubeconfigFromPath(kubeconfigPath string) (*rest.Config, error) {
+	if kubeconfigPath == "" {
+		return nil, errors.New("missing value for required flag --kcp-kubeconfig")
+	}
+	cfg, err := clientcmd.LoadFromFile(kubeconfigPath)
+	if err != nil {
+		return nil, err
+	}
+	restCfg, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return restCfg, err
+	}
+	return restCfg, nil
+}
+
 func newViper() *viper.Viper {
 	v := viper.NewWithOptions(
 		viper.EnvKeyReplacer(strings.NewReplacer("-", "_")),

internal/config/config.go

@@ -12,15 +12,19 @@ type Config struct {
 	FGA struct {
 		Target string `mapstructure:"fga-target"`
 	} `mapstructure:",squash"`
-	APIExportEndpointSliceName string `mapstructure:"api-export-endpoint-slice-name"`
-	CoreModulePath             string `mapstructure:"core-module-path"`
-	WorkspaceDir               string `mapstructure:"workspace-dir" default:"/operator/"`
-	BaseDomain                 string `mapstructure:"base-domain" default:"portal.dev.local:8443"`
-	GroupClaim                 string `mapstructure:"group-claim" default:"groups"`
-	UserClaim                  string `mapstructure:"user-claim" default:"email"`
-	InitializerName            string `mapstructure:"initializer-name" default:"root:security"`
-	DomainCALookup             bool   `mapstructure:"domain-ca-lookup" default:"false"`
-	IDP                        struct {
+	KCP struct {
+		Kubeconfig string `mapstructure:"kcp-kubeconfig" default:"/api-kubeconfig/kubeconfig"`
+	} `mapstructure:",squash"`
+	APIExportEndpointSliceName    string `mapstructure:"api-export-endpoint-slice-name"`
+	CoreModulePath                string `mapstructure:"core-module-path"`
+	WorkspaceDir                  string `mapstructure:"workspace-dir" default:"/operator/"`
+	BaseDomain                    string `mapstructure:"base-domain" default:"portal.dev.local:8443"`
+	GroupClaim                    string `mapstructure:"group-claim" default:"groups"`
+	UserClaim                     string `mapstructure:"user-claim" default:"email"`
+	InitializerName               string `mapstructure:"initializer-name" default:"root:security"`
+	DomainCALookup                bool   `mapstructure:"domain-ca-lookup" default:"false"`
+	SecretWaitingTimeoutInSeconds int    `mapstructure:"secret-waiting-timeout-seconds" default:"60"`
+	IDP                           struct {
 		// SMTP settings
 		SMTPServer  string `mapstructure:"idp-smtp-server"`
 		SMTPPort    int    `mapstructure:"idp-smtp-port"`

internal/controller/apibinding_controller.go

@@ -11,7 +11,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
-	lifecyclecontrollerruntime "github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
+	"github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
 	"github.com/platform-mesh/security-operator/internal/subroutine"
 	mccontext "sigs.k8s.io/multicluster-runtime/pkg/context"
 	mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
@@ -21,23 +21,23 @@ import (
 func NewAPIBindingReconciler(logger *logger.Logger, mcMgr mcmanager.Manager) *APIBindingReconciler {
 	return &APIBindingReconciler{
 		log: logger,
-		lifecycle: builder.NewBuilder("apibinding", "apibinding-controller", []lifecyclesubroutine.Subroutine{
+		mclifecycle: builder.NewBuilder("apibinding", "apibinding-controller", []lifecyclesubroutine.Subroutine{
 			subroutine.NewAuthorizationModelGenerationSubroutine(mcMgr),
 		}, logger).
 			BuildMultiCluster(mcMgr),
 	}
 }
 
 type APIBindingReconciler struct {
-	log       *logger.Logger
-	lifecycle *lifecyclecontrollerruntime.LifecycleManager
+	log         *logger.Logger
+	mclifecycle *multicluster.LifecycleManager
 }
 
 func (r *APIBindingReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
 	ctxWithCluster := mccontext.WithCluster(ctx, req.ClusterName)
-	return r.lifecycle.Reconcile(ctxWithCluster, req, &kcpv1alpha1.APIBinding{})
+	return r.mclifecycle.Reconcile(ctxWithCluster, req, &kcpv1alpha1.APIBinding{})
 }
 
 func (r *APIBindingReconciler) SetupWithManager(mgr mcmanager.Manager, cfg *platformeshconfig.CommonServiceConfig, evp ...predicate.Predicate) error {
-	return r.lifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "apibinding-controller", &kcpv1alpha1.APIBinding{}, cfg.DebugLabelValue, r, r.log, evp...)
+	return r.mclifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "apibinding-controller", &kcpv1alpha1.APIBinding{}, cfg.DebugLabelValue, r, r.log, evp...)
 }

internal/controller/authorization_model_controller.go

@@ -6,7 +6,7 @@ import (
 	openfgav1 "github.com/openfga/api/proto/openfga/v1"
 	platformeshconfig "github.com/platform-mesh/golang-commons/config"
 	"github.com/platform-mesh/golang-commons/controller/lifecycle/builder"
-	lifecyclecontrollerruntime "github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
+	"github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
 	lifecyclesubroutine "github.com/platform-mesh/golang-commons/controller/lifecycle/subroutine"
 	"github.com/platform-mesh/golang-commons/logger"
 	corev1alpha1 "github.com/platform-mesh/security-operator/api/v1alpha1"
@@ -19,14 +19,14 @@ import (
 )
 
 type AuthorizationModelReconciler struct {
-	log       *logger.Logger
-	lifecycle *lifecyclecontrollerruntime.LifecycleManager
+	log         *logger.Logger
+	mclifecycle *multicluster.LifecycleManager
 }
 
 func NewAuthorizationModelReconciler(log *logger.Logger, fga openfgav1.OpenFGAServiceClient, mcMgr mcmanager.Manager) *AuthorizationModelReconciler {
 	return &AuthorizationModelReconciler{
 		log: log,
-		lifecycle: builder.NewBuilder("authorizationmodel", "AuthorizationModelReconciler", []lifecyclesubroutine.Subroutine{
+		mclifecycle: builder.NewBuilder("authorizationmodel", "AuthorizationModelReconciler", []lifecyclesubroutine.Subroutine{
 			subroutine.NewTupleSubroutine(fga, mcMgr),
 		}, log).
 			BuildMultiCluster(mcMgr),
@@ -35,9 +35,9 @@ func NewAuthorizationModelReconciler(log *logger.Logger, fga openfgav1.OpenFGASe
 
 func (r *AuthorizationModelReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
 	ctxWithCluster := mccontext.WithCluster(ctx, req.ClusterName)
-	return r.lifecycle.Reconcile(ctxWithCluster, req, &corev1alpha1.AuthorizationModel{})
+	return r.mclifecycle.Reconcile(ctxWithCluster, req, &corev1alpha1.AuthorizationModel{})
 }
 
 func (r *AuthorizationModelReconciler) SetupWithManager(mgr mcmanager.Manager, cfg *platformeshconfig.CommonServiceConfig, evp ...predicate.Predicate) error { // coverage-ignore
-	return r.lifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "authorizationmodel", &corev1alpha1.AuthorizationModel{}, cfg.DebugLabelValue, r, r.log, evp...)
+	return r.mclifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "authorizationmodel", &corev1alpha1.AuthorizationModel{}, cfg.DebugLabelValue, r, r.log, evp...)
 }

internal/controller/initializer_controller.go

@@ -6,7 +6,7 @@ import (
 	kcpcorev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1"
 	platformeshconfig "github.com/platform-mesh/golang-commons/config"
 	"github.com/platform-mesh/golang-commons/controller/lifecycle/builder"
-	lifecyclecontrollerruntime "github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
+	"github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
 	lifecyclesubroutine "github.com/platform-mesh/golang-commons/controller/lifecycle/subroutine"
 	"github.com/platform-mesh/golang-commons/logger"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -23,18 +23,18 @@ import (
 type LogicalClusterReconciler struct {
 	log *logger.Logger
 
-	lifecycle *lifecyclecontrollerruntime.LifecycleManager
+	mclifecycle *multicluster.LifecycleManager
 }
 
 func NewLogicalClusterReconciler(log *logger.Logger, orgClient client.Client, cfg config.Config, inClusterClient client.Client, mgr mcmanager.Manager) *LogicalClusterReconciler {
 	return &LogicalClusterReconciler{
 		log: log,
-		lifecycle: builder.NewBuilder("logicalcluster", "LogicalClusterReconciler", []lifecyclesubroutine.Subroutine{
+		mclifecycle: builder.NewBuilder("logicalcluster", "LogicalClusterReconciler", []lifecyclesubroutine.Subroutine{
 			subroutine.NewWorkspaceInitializer(orgClient, cfg, mgr),
 			subroutine.NewWorkspaceAuthConfigurationSubroutine(orgClient, inClusterClient, cfg),
 			subroutine.NewRealmSubroutine(inClusterClient, &cfg, cfg.BaseDomain),
 			subroutine.NewInviteSubroutine(orgClient, mgr),
-			subroutine.NewRemoveInitializer(mgr, cfg.InitializerName),
+			subroutine.NewRemoveInitializer(mgr, cfg, inClusterClient),
 		}, log).
 			WithReadOnly().
 			BuildMultiCluster(mgr),
@@ -43,9 +43,9 @@ func NewLogicalClusterReconciler(log *logger.Logger, orgClient client.Client, cf
 
 func (r *LogicalClusterReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
 	ctxWithCluster := mccontext.WithCluster(ctx, req.ClusterName)
-	return r.lifecycle.Reconcile(ctxWithCluster, req, &kcpcorev1alpha1.LogicalCluster{})
+	return r.mclifecycle.Reconcile(ctxWithCluster, req, &kcpcorev1alpha1.LogicalCluster{})
 }
 
 func (r *LogicalClusterReconciler) SetupWithManager(mgr mcmanager.Manager, cfg *platformeshconfig.CommonServiceConfig, evp ...predicate.Predicate) error {
-	return r.lifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "LogicalCluster", &kcpcorev1alpha1.LogicalCluster{}, cfg.DebugLabelValue, r, r.log, evp...)
+	return r.mclifecycle.SetupWithManager(mgr, cfg.MaxConcurrentReconciles, "LogicalCluster", &kcpcorev1alpha1.LogicalCluster{}, cfg.DebugLabelValue, r, r.log, evp...)
 }

internal/controller/invite_controller.go

@@ -6,7 +6,7 @@ import (
 
 	platformeshconfig "github.com/platform-mesh/golang-commons/config"
 	"github.com/platform-mesh/golang-commons/controller/lifecycle/builder"
-	lifecycle "github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
+	"github.com/platform-mesh/golang-commons/controller/lifecycle/multicluster"
 	lifecyclesubroutine "github.com/platform-mesh/golang-commons/controller/lifecycle/subroutine"
 	"github.com/platform-mesh/golang-commons/logger"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -20,7 +20,7 @@ import (
 )
 
 type InviteReconciler struct {
-	lifecycle *lifecycle.LifecycleManager
+	mclifecycle *multicluster.LifecycleManager
 }
 
 func NewInviteReconciler(ctx context.Context, mgr mcmanager.Manager, cfg *config.Config, log *logger.Logger) *InviteReconciler {
@@ -35,7 +35,7 @@ func NewInviteReconciler(ctx context.Context, mgr mcmanager.Manager, cfg *config
 	}
 
 	return &InviteReconciler{
-		lifecycle: builder.NewBuilder(
+		mclifecycle: builder.NewBuilder(
 			"invite",
 			"InviteReconciler",
 			[]lifecyclesubroutine.Subroutine{
@@ -46,11 +46,11 @@ func NewInviteReconciler(ctx context.Context, mgr mcmanager.Manager, cfg *config
 }
 
 func (r *InviteReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
-	return r.lifecycle.Reconcile(mccontext.WithCluster(ctx, req.ClusterName), req, &v1alpha1.Invite{})
+	return r.mclifecycle.Reconcile(mccontext.WithCluster(ctx, req.ClusterName), req, &v1alpha1.Invite{})
 }
 
 func (r *InviteReconciler) SetupWithManager(mgr mcmanager.Manager, cfg *platformeshconfig.CommonServiceConfig, log *logger.Logger) error { // coverage-ignore
-	return r.lifecycle.
+	return r.mclifecycle.
 		SetupWithManager(
 			mgr,
 			cfg.MaxConcurrentReconciles,