operator v1: scheduled requeue for ghost broker decommissioner

Sometimes, it does not get triggered. Only after up to 10h, the
reconciler gets called (syncPeriod).

The recommended way to do a schedule is RequeueAfter:
kubernetes-sigs/kubebuilder#1015 (comment)

(cherry picked from commit dc8996c)

Author: birdayz

operator/cmd/run/run.go

@@ -129,37 +129,38 @@ func (s *LabelSelectorValue) Type() string {
 
 func Command() *cobra.Command {
 	var (
-		clusterDomain                   string
-		metricsAddr                     string
-		secureMetrics                   bool
-		enableHTTP2                     bool
-		probeAddr                       string
-		pprofAddr                       string
-		enableLeaderElection            bool
-		webhookEnabled                  bool
-		configuratorBaseImage           string
-		configuratorTag                 string
-		configuratorImagePullPolicy     string
-		decommissionWaitInterval        time.Duration
-		metricsTimeout                  time.Duration
-		restrictToRedpandaVersion       string
-		namespace                       string
-		additionalControllers           []string
-		operatorMode                    bool
-		enableHelmControllers           bool
-		ghostbuster                     bool
-		unbindPVCsAfter                 time.Duration
-		unbinderSelector                LabelSelectorValue
-		autoDeletePVCs                  bool
-		forceDefluxedMode               bool
-		helmRepositoryURL               string
-		webhookCertPath                 string
-		webhookCertName                 string
-		webhookCertKey                  string
-		metricsCertPath                 string
-		metricsCertName                 string
-		metricsCertKey                  string
-		enableGhostBrokerDecommissioner bool
+		clusterDomain                       string
+		metricsAddr                         string
+		secureMetrics                       bool
+		enableHTTP2                         bool
+		probeAddr                           string
+		pprofAddr                           string
+		enableLeaderElection                bool
+		webhookEnabled                      bool
+		configuratorBaseImage               string
+		configuratorTag                     string
+		configuratorImagePullPolicy         string
+		decommissionWaitInterval            time.Duration
+		metricsTimeout                      time.Duration
+		restrictToRedpandaVersion           string
+		namespace                           string
+		additionalControllers               []string
+		operatorMode                        bool
+		enableHelmControllers               bool
+		ghostbuster                         bool
+		unbindPVCsAfter                     time.Duration
+		unbinderSelector                    LabelSelectorValue
+		autoDeletePVCs                      bool
+		forceDefluxedMode                   bool
+		helmRepositoryURL                   string
+		webhookCertPath                     string
+		webhookCertName                     string
+		webhookCertKey                      string
+		metricsCertPath                     string
+		metricsCertName                     string
+		metricsCertKey                      string
+		enableGhostBrokerDecommissioner     bool
+		ghostBrokerDecommissionerSyncPeriod time.Duration
 	)
 
 	cmd := &cobra.Command{
@@ -201,6 +202,7 @@ func Command() *cobra.Command {
 				metricsCertName,
 				metricsCertKey,
 				enableGhostBrokerDecommissioner,
+				ghostBrokerDecommissionerSyncPeriod,
 			)
 		},
 	}
@@ -246,6 +248,7 @@ func Command() *cobra.Command {
 	cmd.Flags().StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
 	cmd.Flags().StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	cmd.Flags().BoolVar(&enableGhostBrokerDecommissioner, "enable-ghost-broker-decommissioner", false, "Enable ghost broker decommissioner.")
+	cmd.Flags().DurationVar(&ghostBrokerDecommissionerSyncPeriod, "ghost-broker-decommissioner-sync-period", time.Minute*5, "Ghost broker sync period. The Ghost Broker Decommissioner is guaranteed to be called after this period.")
 
 	// 3rd party flags.
 	clientOptions.BindFlags(cmd.Flags())
@@ -307,6 +310,7 @@ func Run(
 	metricsCertName string,
 	metricsCertKey string,
 	enableGhostBrokerDecommissioner bool,
+	ghostBrokerDecommissionerSyncPeriod time.Duration,
 ) error {
 	setupLog := ctrl.LoggerFrom(ctx).WithName("setup")
 
@@ -671,6 +675,7 @@ func Run(
 
 	if enableGhostBrokerDecommissioner {
 		d := decommissioning.NewStatefulSetDecommissioner(mgr, &v1Fetcher{client: mgr.GetClient()},
+			decommissioning.WithSyncPeriod(ghostBrokerDecommissionerSyncPeriod),
 			decommissioning.WithCleanupPVCs(false),
 			decommissioning.WithFactory(internalclient.NewFactory(mgr.GetConfig(), mgr.GetClient())),
 			decommissioning.WithFilter(func(ctx context.Context, sts *appsv1.StatefulSet) (bool, error) {

operator/internal/controller/decommissioning/statefulset_decomissioner.go

@@ -117,6 +117,12 @@ func WithCleanupPVCs(cleanup bool) Option {
 	}
 }
 
+func WithSyncPeriod(d time.Duration) Option {
+	return func(ssd *StatefulSetDecomissioner) {
+		ssd.syncPeriod = d
+	}
+}
+
 type StatefulSetDecomissioner struct {
 	client               client.Client
 	factory              internalclient.ClientFactory
@@ -129,6 +135,7 @@ type StatefulSetDecomissioner struct {
 	delayedVolumeCache   *CategorizedDelayedCache[types.NamespacedName, types.NamespacedName]
 	filter               func(ctx context.Context, set *appsv1.StatefulSet) (bool, error)
 	cleanupPVCs          bool
+	syncPeriod           time.Duration
 }
 
 func NewStatefulSetDecommissioner(mgr ctrl.Manager, fetcher Fetcher, options ...Option) *StatefulSetDecomissioner {
@@ -260,7 +267,10 @@ func (s *StatefulSetDecomissioner) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{RequeueAfter: timeout}, nil
 	}
 
-	return ctrl.Result{}, nil
+	// If a schedule is configured, just requeue with this period.
+	// Sometimes, the node disappears and no event about sts/pod is fired, or fired before we can actually decommission the ghost broker.
+	// Running the reconciler regularly ensures we're not missing any ghost brokers.
+	return ctrl.Result{RequeueAfter: s.syncPeriod}, nil
 }
 
 // Decommission decommissions any stray resources for a StatefulSet. This includes: