[Bugfix] Adjust pod finalizers (#749)

Author: ajanikow

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Switch K8S CRD API to V1
 - Deprecate Alpine image usage
 - Use persistent name and namespace in ArangoDeployment reconcilation loop
+- Remove finalizers when Server container is already terminated and reduce initial reconciliation delay
 
 ## [1.1.9](https://github.com/arangodb/kube-arangodb/tree/1.1.9) (2021-05-28)
 - Add IP, DNS, ShortDNS, HeadlessService (Default) communication methods

pkg/deployment/deployment.go

@@ -253,7 +253,11 @@ func (d *Deployment) run() {
 
 	d.lookForServiceMonitorCRD()
 
-	inspectionInterval := maxInspectionInterval
+	// Execute inspection for first time without delay of 10s
+	log.Debug().Msg("Initially inspect deployment...")
+	inspectionInterval := d.inspectDeployment(minInspectionInterval)
+	log.Debug().Str("interval", inspectionInterval.String()).Msg("...deployment inspect started")
+
 	for {
 		select {
 		case <-d.stopCh:

pkg/deployment/resources/pod_finalizers.go

@@ -49,23 +49,42 @@ const (
 func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) (util.Interval, error) {
 	log := r.log.With().Str("pod-name", p.GetName()).Logger()
 	var removalList []string
+
+	isServerContainerDead := !k8sutil.IsPodServerContainerRunning(p)
+
 	for _, f := range p.ObjectMeta.GetFinalizers() {
 		switch f {
 		case constants.FinalizerPodAgencyServing:
 			log.Debug().Msg("Inspecting agency-serving finalizer")
+			if isServerContainerDead {
+				log.Debug().Msg("Server Container is dead, removing finalizer")
+				removalList = append(removalList, f)
+				break
+			}
 			if err := r.inspectFinalizerPodAgencyServing(ctx, log, p, memberStatus, updateMember); err == nil {
 				removalList = append(removalList, f)
 			} else {
 				log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet")
 			}
 		case constants.FinalizerPodDrainDBServer:
 			log.Debug().Msg("Inspecting drain dbserver finalizer")
+			if isServerContainerDead {
+				log.Debug().Msg("Server Container is dead, removing finalizer")
+				removalList = append(removalList, f)
+				break
+			}
 			if err := r.inspectFinalizerPodDrainDBServer(ctx, log, p, memberStatus, updateMember); err == nil {
 				removalList = append(removalList, f)
 			} else {
 				log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove Pod finalizer yet")
 			}
 		case constants.FinalizerDelayPodTermination:
+			if isServerContainerDead {
+				log.Debug().Msg("Server Container is dead, removing finalizer")
+				removalList = append(removalList, f)
+				break
+			}
+
 			s, _ := r.context.GetStatus()
 			_, group, ok := s.Members.ElementByID(memberStatus.ID)
 			if !ok {

pkg/util/k8sutil/pods.go

@@ -82,6 +82,22 @@ func GetPodByName(pods []core.Pod, podName string) (core.Pod, bool) {
 	return core.Pod{}, false
 }
 
+// IsPodServerContainerRunning returns true if the arangodb container of the pod is still running
+func IsPodServerContainerRunning(pod *core.Pod) bool {
+	for _, c := range pod.Status.ContainerStatuses {
+		if c.Name != ServerContainerName {
+			continue
+		}
+
+		if c.State.Terminated != nil {
+			return false
+		} else {
+			return true
+		}
+	}
+	return false
+}
+
 // IsPodSucceeded returns true if the arangodb container of the pod
 // has terminated with exit code 0.
 func IsPodSucceeded(pod *core.Pod) bool {