Allow the Operator lease duration and renewal timeout to be configured (oracle#773)

Author: Jonathan Knight

helm-charts/coherence-operator/templates/deployment.yaml

@@ -189,6 +189,12 @@ spec:
 {{- end }}
 {{- range .Values.cipherDenyList }}
         - --cipher-deny-list={{ . }}
+{{- end }}
+{{- if .Values.leaderElectionDuration }}
+        - --leader-election-duration={{ .Values.leaderElectionDuration | quote }}
+{{- end }}
+{{- if .Values.leaderElectionRenewTimeout }}
+        - --leader-election-renew-timeout={{ .Values.leaderElectionRenewTimeout | quote }}
 {{- end }}
         command:
         - "/files/runner"

helm-charts/coherence-operator/values.yaml

@@ -225,8 +225,34 @@ allowCoherenceJobs: true
 # The CRDs must be manually installed before the Operator can be installed.
 installCrd: true
 
+# The list of allowed TLS cipher suite names.
 cipherAllowList: []
 
+# The list of disallowed TLS cipher suite names.
 cipherDenyList: []
 
+# This value is used to set the `GODEBUG` environment variables.
+# The `fips` value is unset by default, if set it must be one of the values, "off", "on" or "only".
+# If `fips` is set to any other value, the chart will fail to install.
 fips:
+
+# The value that the Operator will use for the leadership lease duration.
+# This is a string value that should be a valid Go Duration string.
+#
+# The default value is 30 seconds. The only reason to change this is in some environments
+# that may be particularly slow and would need a larger value due to loss of leadership issues
+#
+# Normally this will be a number of seconds. For example, 30 seconds is "30s" and
+# there would not be any reason to have values in minutes or hours.
+leaderElectionDuration:
+
+# The value that the Operator will use for the leadership lease renewal timeout.
+# This is a string value that should be a valid Go Duration string.
+#
+# The default value is 20 seconds. The only reason to change this is in some environments
+# that may be particularly slow and would need a larger value due to loss of leadership issues
+#
+# Normally this will be a number of seconds. For example, 30 seconds is "30s" and
+# there would not be any reason to have values in minutes or hours.
+leaderElectionRenewTimeout:
+

pkg/operator/operator.go

@@ -49,45 +49,47 @@ const (
 	DefaultMutatingWebhookName   = "coherence-operator-mutating-webhook-configuration"
 	DefaultValidatingWebhookName = "coherence-operator-validating-webhook-configuration"
 
-	FlagCACertRotateBefore    = "ca-cert-rotate-before"
-	FlagCACertValidity        = "ca-cert-validity"
-	FlagCertType              = "cert-type"
-	FlagCertIssuer            = "cert-issuer"
-	FlagCoherenceImage        = "coherence-image"
-	FlagCRD                   = "install-crd"
-	FlagJobCRD                = "install-job-crd"
-	FlagEnableCoherenceJobs   = "enable-jobs"
-	FlagDevMode               = "coherence-dev-mode"
-	FlagCipherDenyList        = "cipher-deny-list"
-	FlagCipherAllowList       = "cipher-allow-list"
-	FlagConfig                = "config"
-	FlagConfigType            = "config-type"
-	FlagDryRun                = "dry-run"
-	FlagEnableWebhook         = "enable-webhook"
-	FlagEnableHttp2           = "enable-http2"
-	FlagGlobalAnnotation      = "global-annotation"
-	FlagGlobalLabel           = "global-label"
-	FlagHealthAddress         = "health-addr"
-	FlagLeaderElection        = "enable-leader-election"
-	FlagMetricsAddress        = "metrics-addr"
-	FlagMutatingWebhookName   = "mutating-webhook-name"
-	FlagOperatorNamespace     = "operator-namespace"
-	FlagNodeLookupEnabled     = "node-lookup-enabled"
-	FlagRackLabel             = "rack-label"
-	FlagRestHost              = "rest-host"
-	FlagRestPort              = "rest-port"
-	FlagSecureMetrics         = "metrics-secure"
-	FlagServiceName           = "service-name"
-	FlagServicePort           = "service-port"
-	FlagSiteLabel             = "site-label"
-	FlagSkipServiceSuspend    = "skip-service-suspend"
-	FlagOperatorImage         = "operator-image"
-	FlagValidatingWebhookName = "validating-webhook-name"
-	FlagWebhookCertDir        = "webhook-cert-dir"
-	FlagWebhookSecret         = "webhook-secret"
-	FlagWebhookService        = "webhook-service"
-	FlagEnvVar                = "env"
-	FlagJvmArg                = "jvm"
+	FlagCACertRotateBefore     = "ca-cert-rotate-before"
+	FlagCACertValidity         = "ca-cert-validity"
+	FlagCertType               = "cert-type"
+	FlagCertIssuer             = "cert-issuer"
+	FlagCoherenceImage         = "coherence-image"
+	FlagCRD                    = "install-crd"
+	FlagJobCRD                 = "install-job-crd"
+	FlagEnableCoherenceJobs    = "enable-jobs"
+	FlagDevMode                = "coherence-dev-mode"
+	FlagCipherDenyList         = "cipher-deny-list"
+	FlagCipherAllowList        = "cipher-allow-list"
+	FlagConfig                 = "config"
+	FlagConfigType             = "config-type"
+	FlagDryRun                 = "dry-run"
+	FlagEnableWebhook          = "enable-webhook"
+	FlagEnableHttp2            = "enable-http2"
+	FlagGlobalAnnotation       = "global-annotation"
+	FlagGlobalLabel            = "global-label"
+	FlagHealthAddress          = "health-addr"
+	FlagLeaderElection         = "enable-leader-election"
+	FlagLeaderElectionDuration = "leader-election-duration"
+	FlagLeaderElectionRenew    = "leader-election-renew-timeout"
+	FlagMetricsAddress         = "metrics-addr"
+	FlagMutatingWebhookName    = "mutating-webhook-name"
+	FlagOperatorNamespace      = "operator-namespace"
+	FlagNodeLookupEnabled      = "node-lookup-enabled"
+	FlagRackLabel              = "rack-label"
+	FlagRestHost               = "rest-host"
+	FlagRestPort               = "rest-port"
+	FlagSecureMetrics          = "metrics-secure"
+	FlagServiceName            = "service-name"
+	FlagServicePort            = "service-port"
+	FlagSiteLabel              = "site-label"
+	FlagSkipServiceSuspend     = "skip-service-suspend"
+	FlagOperatorImage          = "operator-image"
+	FlagValidatingWebhookName  = "validating-webhook-name"
+	FlagWebhookCertDir         = "webhook-cert-dir"
+	FlagWebhookSecret          = "webhook-secret"
+	FlagWebhookService         = "webhook-service"
+	FlagEnvVar                 = "env"
+	FlagJvmArg                 = "jvm"
 
 	// EnvVarWatchNamespace is the environment variable to use to set the watch namespace(s)
 	EnvVarWatchNamespace = "WATCH_NAMESPACE"
@@ -304,6 +306,17 @@ func SetupFlags(cmd *cobra.Command, v *viper.Viper) {
 		FlagCipherAllowList,
 		nil,
 		"A list of TLS cipher names to be enabled (if a cipher appears in this list and the deny list it will be disabled)")
+	cmd.Flags().Duration(
+		FlagLeaderElectionDuration,
+		time.Second*30,
+		"The value the Operator uses for the leadership lease duration. "+
+			"Setting this value too low can cause Pod restarts as the leader may lose leadership. "+
+			"If the value entered is less than 10s, then 10s will be used")
+	cmd.Flags().Duration(
+		FlagLeaderElectionRenew,
+		time.Second*20,
+		"The duration the Operator uses for the leadership lease renewal timeout. "+
+			"If the value entered is less than 10s, then 10s will be used")
 
 	// enable using dashed notation in flags and underscores in env
 	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))

pkg/runner/cmd_operator.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	hooks "sigs.k8s.io/controller-runtime/pkg/webhook"
+	"time"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -148,13 +149,24 @@ func execute(v *viper.Viper) error {
 		TLSOpts: tlsOpts,
 	})
 
+	duration := viper.GetDuration(operator.FlagLeaderElectionDuration)
+	if duration < time.Second*10 {
+		duration = time.Second * 10
+	}
+	renew := viper.GetDuration(operator.FlagLeaderElectionRenew)
+	if renew < time.Second*10 {
+		renew = time.Second * 10
+	}
+
 	options := ctrl.Options{
 		Scheme:                 scheme,
 		HealthProbeBindAddress: viper.GetString(operator.FlagHealthAddress),
 		Metrics:                metricsServerOptions,
 		WebhookServer:          webhookServer,
 		LeaderElection:         viper.GetBool(operator.FlagLeaderElection),
 		LeaderElectionID:       lockName,
+		LeaseDuration:          &duration,
+		RenewDeadline:          &renew,
 		Controller: config.Controller{
 			SkipNameValidation: ptr.To(dryRun),
 		},