[Fix] Cache and DBServer member removal (#689)

Author: ajanikow

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - Fix AKS Volume Resize mode
+- Use cached status in member client creation
+- Remove failed DBServers
 
 ## [1.1.4](https://github.com/arangodb/kube-arangodb/tree/1.1.4) (2021-02-15)
 - Add support for spec.ClusterDomain to be able to use FQDN in ArangoDB cluster communication

pkg/deployment/context_impl.go

@@ -263,7 +263,10 @@ func (d *Deployment) getAuth() (driver.Authentication, error) {
 		return nil, nil
 	}
 
-	secrets := d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace())
+	var secrets inspector.SecretReadInterface = d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace())
+	if currentState := d.currentState; currentState != nil {
+		secrets = currentState.SecretReadInterface()
+	}
 
 	var secret string
 	if i := d.apiObject.Status.CurrentImage; i == nil || !features.JWTRotation().Supported(i.ArangoDBVersion, i.Enterprise) {
@@ -589,3 +592,11 @@ func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) {
 
 	return podList, nil
 }
+
+func (d *Deployment) GetCachedStatus() inspector.Inspector {
+	return d.currentState
+}
+
+func (d *Deployment) SetCachedStatus(i inspector.Inspector) {
+	d.currentState = i
+}

pkg/deployment/deployment.go

@@ -121,6 +121,7 @@ type Deployment struct {
 	inspectCRDTrigger         trigger.Trigger
 	updateDeploymentTrigger   trigger.Trigger
 	clientCache               deploymentClient.Cache
+	currentState              inspector.Inspector
 	recentInspectionErrors    int
 	clusterScalingIntegration *clusterScalingIntegration
 	reconciler                *reconcile.Reconciler

pkg/deployment/deployment_inspector.go

@@ -130,6 +130,9 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, cachedStatus inspector.Inspector) (nextInterval util.Interval, inspectError error) {
 	t := time.Now()
 
+	d.SetCachedStatus(cachedStatus)
+	defer d.SetCachedStatus(nil)
+
 	defer func() {
 		d.deps.Log.Info().Msgf("Reconciliation loop took %s", time.Since(t))
 	}()

pkg/deployment/reconcile/plan_builder_cluster.go

@@ -88,6 +88,18 @@ func createClusterOperationPlan(ctx context.Context,
 					api.NewAction(api.ActionTypeClusterMemberCleanup, api.ServerGroupCoordinators, string(id)),
 				}
 			}
+		case driver.ServerRoleDBServer:
+			if member.Status != driver.ServerStatusFailed {
+				continue
+			}
+
+			if !member.CanBeDeleted {
+				continue
+			}
+
+			return api.Plan{
+				api.NewAction(api.ActionTypeClusterMemberCleanup, api.ServerGroupDBServers, string(id)),
+			}
 		}
 	}
 

pkg/deployment/resources/context.go

@@ -25,6 +25,8 @@ package resources
 import (
 	"context"
 
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+
 	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
 
 	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
@@ -103,4 +105,7 @@ type Context interface {
 	// GetBackup receives information about a backup resource
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 	GetScope() scope.Scope
+
+	GetCachedStatus() inspector.Inspector
+	SetCachedStatus(i inspector.Inspector)
 }

pkg/deployment/resources/inspector/inspector.go

@@ -25,6 +25,8 @@ package inspector
 import (
 	"sync"
 
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
 
@@ -33,6 +35,11 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+// SecretReadInterface has methods to work with Secret resources with ReadOnly mode.
+type SecretReadInterface interface {
+	Get(name string, options meta.GetOptions) (*core.Secret, error)
+}
+
 func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, namespace string) (Inspector, error) {
 	pods, err := podsToMap(k, namespace)
 	if err != nil {
@@ -102,6 +109,7 @@ type Inspector interface {
 
 	Secret(name string) (*core.Secret, bool)
 	IterateSecrets(action SecretAction, filters ...SecretFilter) error
+	SecretReadInterface() SecretReadInterface
 
 	PersistentVolumeClaim(name string) (*core.PersistentVolumeClaim, bool)
 	IteratePersistentVolumeClaims(action PersistentVolumeClaimAction, filters ...PersistentVolumeClaimFilter) error

pkg/deployment/resources/inspector/secrets.go

@@ -25,7 +25,9 @@ package inspector
 import (
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	core "k8s.io/api/core/v1"
+	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -66,6 +68,25 @@ func (i *inspector) Secret(name string) (*core.Secret, bool) {
 	return secret, true
 }
 
+func (i *inspector) SecretReadInterface() SecretReadInterface {
+	return &secretReadInterface{i: i}
+}
+
+type secretReadInterface struct {
+	i *inspector
+}
+
+func (s secretReadInterface) Get(name string, options meta.GetOptions) (*core.Secret, error) {
+	if s, ok := s.i.Secret(name); !ok {
+		return nil, apiErrors.NewNotFound(schema.GroupResource{
+			Group:    core.GroupName,
+			Resource: "Secret",
+		}, name)
+	} else {
+		return s, nil
+	}
+}
+
 func secretsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) {
 	secrets, err := getSecrets(k, namespace, "")
 	if err != nil {

pkg/util/k8sutil/secrets.go

@@ -34,9 +34,9 @@ import (
 
 // SecretInterface has methods to work with Secret resources.
 type SecretInterface interface {
+	Get(name string, options meta.GetOptions) (*core.Secret, error)
 	Create(*core.Secret) (*core.Secret, error)
 	Update(*core.Secret) (*core.Secret, error)
-	Get(name string, options meta.GetOptions) (*core.Secret, error)
 	Delete(name string, options *meta.DeleteOptions) error
 	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*core.Secret, error)
 }