[Bugfix] Fix restart procedure in case of failing members (#884)

Author: ajanikow

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Add metrics for the plan actions
 - Add ArangoClusterSynchronization Operator
 - Update licenses
+- Fix restart procedure in case of failing members
 
 ## [1.2.6](https://github.com/arangodb/kube-arangodb/tree/1.2.6) (2021-12-15)
 - Add ArangoBackup backoff functionality

pkg/apis/deployment/v1/server_group_spec.go

@@ -23,6 +23,7 @@ package v1
 import (
 	"math"
 	"strings"
+	"time"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 
@@ -148,6 +149,8 @@ type ServerGroupSpec struct {
 	InternalPort *int `json:"internalPort,omitempty"`
 	// AllowMemberRecreation allows to recreate member. Value is used only for Coordinator and DBServer with default to True, for all other groups set to false.
 	AllowMemberRecreation *bool `json:"allowMemberRecreation,omitempty"`
+	// TerminationGracePeriodSeconds override default TerminationGracePeriodSeconds for pods - via silent rotation
+	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 }
 
 // ServerGroupSpecSecurityContext contains specification for pod security context
@@ -700,3 +703,12 @@ func (s ServerGroupSpec) GetShutdownDelay(group ServerGroup) int {
 	}
 	return *s.ShutdownDelay
 }
+
+// GetTerminationGracePeriod returns termination grace period as Duration
+func (s ServerGroupSpec) GetTerminationGracePeriod(group ServerGroup) time.Duration {
+	if v := s.TerminationGracePeriodSeconds; v == nil {
+		return group.DefaultTerminationGracePeriod()
+	} else {
+		return time.Second * time.Duration(*v)
+	}
+}

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -23,6 +23,7 @@ package v2alpha1
 import (
 	"math"
 	"strings"
+	"time"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 
@@ -148,6 +149,8 @@ type ServerGroupSpec struct {
 	InternalPort *int `json:"internalPort,omitempty"`
 	// AllowMemberRecreation allows to recreate member. Value is used only for Coordinator and DBServer with default to True, for all other groups set to false.
 	AllowMemberRecreation *bool `json:"allowMemberRecreation,omitempty"`
+	// TerminationGracePeriodSeconds override default TerminationGracePeriodSeconds for pods - via silent rotation
+	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 }
 
 // ServerGroupSpecSecurityContext contains specification for pod security context
@@ -700,3 +703,12 @@ func (s ServerGroupSpec) GetShutdownDelay(group ServerGroup) int {
 	}
 	return *s.ShutdownDelay
 }
+
+// GetTerminationGracePeriod returns termination grace period as Duration
+func (s ServerGroupSpec) GetTerminationGracePeriod(group ServerGroup) time.Duration {
+	if v := s.TerminationGracePeriodSeconds; v == nil {
+		return group.DefaultTerminationGracePeriod()
+	} else {
+		return time.Second * time.Duration(*v)
+	}
+}

pkg/apis/deployment/v2alpha1/server_group_spec.go

@@ -23,8 +23,6 @@ package reconcile
 import (
 	"context"
 
-	core "k8s.io/api/core/v1"
-
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
@@ -35,12 +33,11 @@ func createBootstrapPlan(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan {
-
 	if !status.Conditions.IsTrue(api.ConditionTypeReady) {
 		return nil
 	}
 
-	if condition, hasBootstrap := status.Conditions.Get(api.ConditionTypeBootstrapCompleted); !hasBootstrap || condition.Status == core.ConditionTrue {
+	if status.Conditions.IsTrue(api.ConditionTypeBootstrapCompleted) {
 		return nil
 	}
 

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -62,6 +62,7 @@ func createNormalPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil
 		// Check for members to be removed
 		ApplyIfEmpty(createReplaceMemberPlan).
 		// Check for the need to rotate one or more members
+		ApplyIfEmpty(createMarkToRemovePlan).
 		ApplyIfEmpty(createRotateOrUpgradePlan).
 		// Disable maintenance if upgrade process was done. Upgrade task throw IDLE Action if upgrade is pending
 		ApplyIfEmpty(createMaintenanceManagementPlan).