[Feature] Namespaced mode (#615)

ajanikow · web-flow · commit 145d71dde6d6 · 2020-08-14T09:35:11.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add Operator Namespaced mode (Alpha)
 
 ## [1.0.5](https://github.com/arangodb/kube-arangodb/tree/1.0.5) (2020-08-05)
 - Add Labels and Annotations to ServiceMonitor
diff --git a/chart/kube-arangodb/README.md b/chart/kube-arangodb/README.md
@@ -75,6 +75,7 @@ Default: `legacy`
 
 Supported modes:
 - `legacy` - mode with limited cluster scope access
+- `namespaced` - mode with namespace access only
 
 ### `operator.service.type`
 
diff --git a/chart/kube-arangodb/templates/backup-operator/cluster-role-binding.yaml b/chart/kube-arangodb/templates/backup-operator/cluster-role-binding.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.backup -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -20,6 +21,6 @@ subjects:
       name: {{ template "kube-arangodb.operatorName" . }}
       namespace: {{ .Release.Namespace }}
 
-
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/backup-operator/cluster-role.yaml b/chart/kube-arangodb/templates/backup-operator/cluster-role.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.backup -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -15,5 +16,7 @@ rules:
     - apiGroups: ["apiextensions.k8s.io"]
       resources: ["customresourcedefinitions"]
       verbs: ["get", "list", "watch"]
+
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/deployment-operator/cluster-role-binding.yaml b/chart/kube-arangodb/templates/deployment-operator/cluster-role-binding.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.deployment -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -20,6 +21,6 @@ subjects:
       name: {{ template "kube-arangodb.operatorName" . }}
       namespace: {{ .Release.Namespace }}
 
-
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/deployment-operator/cluster-role.yaml b/chart/kube-arangodb/templates/deployment-operator/cluster-role.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.deployment -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -18,9 +19,7 @@ rules:
     - apiGroups: [""]
       resources: ["namespaces", "nodes", "persistentvolumes"]
       verbs: ["get", "list"]
-    - apiGroups: ["storage.k8s.io"]
-      resources: ["storageclasses"]
-      verbs: ["get", "list"]
 
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/deployment-replications-operator/cluster-role-binding.yaml b/chart/kube-arangodb/templates/deployment-replications-operator/cluster-role-binding.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.deploymentReplications -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -20,6 +21,6 @@ subjects:
       name: {{ template "kube-arangodb.operatorName" . }}
       namespace: {{ .Release.Namespace }}
 
-
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/deployment-replications-operator/cluster-role.yaml b/chart/kube-arangodb/templates/deployment-replications-operator/cluster-role.yaml
@@ -1,4 +1,5 @@
 {{ if .Values.rbac.enabled -}}
+{{ if not (eq .Values.operator.scope "namespaced") -}}
 {{ if .Values.operator.features.deploymentReplications -}}
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -14,10 +15,11 @@ metadata:
 rules:
     - apiGroups: ["apiextensions.k8s.io"]
       resources: ["customresourcedefinitions"]
-      verbs: ["get"]
+      verbs: ["get", "list", "watch"]
     - apiGroups: [""]
       resources: ["namespaces", "nodes"]
       verbs: ["get", "list"]
 
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/deployment.yaml b/chart/kube-arangodb/templates/deployment.yaml
@@ -1,8 +1,13 @@
-{{- if eq .Values.operator.scope "legacy" -}}
+{{ if eq .Values.operator.scope "legacy" -}}
 # Scope "legacy" selected
+{{ else if eq .Values.operator.scope "namespaced" -}}
+# Scope "namespaced" selected
+{{ if .Values.operator.features.storage -}}
+{{ fail (printf "Storage Operator not supported in %s scope!" .Values.operator.scope) -}}
+{{ end -}}
 {{ else -}}
-{{ fail (printf "Operator Scope %s is not supported!" .Values.operator.scope) }}
-{{- end -}}
+{{ fail (printf "Operator Scope %s is not supported!" .Values.operator.scope) -}}
+{{ end -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
diff --git a/chart/kube-arangodb/templates/storage-operator/cluster-role-binding.yaml b/chart/kube-arangodb/templates/storage-operator/cluster-role-binding.yaml
@@ -20,6 +20,5 @@ subjects:
       name: {{ template "kube-arangodb.operatorName" . }}
       namespace: {{ .Release.Namespace }}
 
-
 {{- end }}
 {{- end }}
diff --git a/chart/kube-arangodb/templates/storage-operator/cluster-role.yaml b/chart/kube-arangodb/templates/storage-operator/cluster-role.yaml
@@ -17,7 +17,7 @@ rules:
       verbs: ["*"]
     - apiGroups: ["apiextensions.k8s.io"]
       resources: ["customresourcedefinitions"]
-      verbs: ["get"]
+      verbs: ["get", "list", "watch"]
     - apiGroups: [""]
       resources: ["namespaces", "nodes"]
       verbs: ["get", "list"]
@@ -27,5 +27,6 @@ rules:
     - apiGroups: ["storage.arangodb.com"]
       resources: ["arangolocalstorages"]
       verbs: ["*"]
+
 {{- end }}
 {{- end }}
diff --git a/main.go b/main.go
@@ -31,6 +31,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 
 	"github.com/rs/zerolog/log"
@@ -139,7 +141,7 @@ func init() {
 	f.StringVar(&operatorOptions.arangoImage, "operator.arango-image", ArangoImageEnv.GetOrDefault(defaultArangoImage), "Docker image used for arango by default")
 	f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment")
 	f.BoolVar(&operatorOptions.singleMode, "mode.single", false, "Enable single mode in Operator. WARNING: There should be only one replica of Operator, otherwise Operator can take unexpected actions")
-	f.StringVar(&operatorOptions.scope, "scope", operator.DefaultScope.String(), "Define scope on which Operator works. Legacy - pre 1.1.0 scope with limited cluster access")
+	f.StringVar(&operatorOptions.scope, "scope", scope.DefaultScope.String(), "Define scope on which Operator works. Legacy - pre 1.1.0 scope with limited cluster access")
 
 	features.Init(&cmdMain)
 }
@@ -299,7 +301,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 	}
 	eventRecorder := createRecorder(cliLog, kubecli, name, namespace)
 
-	scope, ok := operator.AsScope(operatorOptions.scope)
+	scope, ok := scope.AsScope(operatorOptions.scope)
 	if !ok {
 		return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Scope %s is not known by Operator", operatorOptions.scope))
 	}
diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go
@@ -31,6 +31,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
+
 	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
@@ -84,6 +86,10 @@ func (d *Deployment) GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface
 	return d.deps.KubeMonitoringCli
 }
 
+func (d *Deployment) GetScope() scope.Scope {
+	return d.config.Scope
+}
+
 // GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 func (d *Deployment) GetLifecycleImage() string {
 	return d.config.LifecycleImage
@@ -425,16 +431,6 @@ func (d *Deployment) UpdatePvc(pvc *v1.PersistentVolumeClaim) error {
 	return maskAny(err)
 }
 
-// GetPv returns PV info about PV with given name.
-func (d *Deployment) GetPv(pvName string) (*v1.PersistentVolume, error) {
-	pv, err := d.GetKubeCli().CoreV1().PersistentVolumes().Get(pvName, meta.GetOptions{})
-	if err == nil {
-		return pv, nil
-	}
-
-	return nil, maskAny(err)
-}
-
 // GetOwnedPVCs returns a list of all PVCs owned by the deployment.
 func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) {
 	// Get all current PVCs
diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go
@@ -29,6 +29,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
+
 	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
@@ -63,6 +65,7 @@ type Config struct {
 	OperatorUUIDInitImage string
 	MetricsExporterImage  string
 	ArangoImage           string
+	Scope                 scope.Scope
 }
 
 // Dependencies holds dependent services for a Deployment
@@ -466,7 +469,12 @@ func (d *Deployment) isOwnerOf(obj metav1.Object) bool {
 // once at creation time of the deployment and then always if the CRD
 // informer is triggered.
 func (d *Deployment) lookForServiceMonitorCRD() {
-	_, err := d.deps.KubeExtCli.ApiextensionsV1beta1().CustomResourceDefinitions().Get("servicemonitors.monitoring.coreos.com", metav1.GetOptions{})
+	var err error
+	if d.GetScope().IsNamespaced() {
+		_, err = d.deps.KubeMonitoringCli.ServiceMonitors(d.GetNamespace()).List(metav1.ListOptions{})
+	} else {
+		_, err = d.deps.KubeExtCli.ApiextensionsV1beta1().CustomResourceDefinitions().Get("servicemonitors.monitoring.coreos.com", metav1.GetOptions{})
+	}
 	log := d.deps.Log
 	log.Debug().Msgf("Looking for ServiceMonitor CRD...")
 	if err == nil {
diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go
@@ -84,8 +84,6 @@ type ActionContext interface {
 	// GetPvc returns PVC info about PVC with given name in the namespace
 	// of the deployment.
 	GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error)
-	// GetPv returns PV info about PV with given name.
-	GetPv(pvName string) (*v1.PersistentVolume, error)
 	// UpdatePvc update PVC with given name in the namespace
 	// of the deployment.
 	UpdatePvc(pvc *v1.PersistentVolumeClaim) error
@@ -187,10 +185,6 @@ func (ac *actionContext) UpdateClusterCondition(conditionType api.ConditionType,
 	})
 }
 
-func (ac *actionContext) GetPv(pvName string) (*v1.PersistentVolume, error) {
-	return ac.context.GetPv(pvName)
-}
-
 func (ac *actionContext) GetAPIObject() k8sutil.APIObject {
 	return ac.context.GetAPIObject()
 }
diff --git a/pkg/deployment/reconcile/action_pvc_resize.go b/pkg/deployment/reconcile/action_pvc_resize.go
@@ -129,17 +129,8 @@ func (a *actionPVCResize) CheckProgress(ctx context.Context) (bool, bool, error)
 		return false, true, err
 	}
 
-	pv, err := a.actionCtx.GetPv(pvc.Spec.VolumeName)
-	if err != nil {
-		if errors.IsNotFound(err) {
-			return true, false, nil
-		}
-
-		return false, true, err
-	}
-
 	if requestedSize, ok := pvc.Spec.Resources.Requests[core.ResourceStorage]; ok {
-		if volumeSize, ok := pv.Spec.Capacity[core.ResourceStorage]; ok {
+		if volumeSize, ok := pvc.Status.Capacity[core.ResourceStorage]; ok {
 			cmp := volumeSize.Cmp(requestedSize)
 			if cmp >= 0 {
 				return true, false, nil
diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go
@@ -83,8 +83,6 @@ type Context interface {
 	UpdatePvc(pvc *v1.PersistentVolumeClaim) error
 	// GetPvc gets a PVC by the given name, in the samespace of the deployment.
 	GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error)
-	// GetPv returns PV info about PV with given name.
-	GetPv(pvName string) (*v1.PersistentVolume, error)
 	// GetTLSKeyfile returns the keyfile encoded TLS certificate+key for
 	// the given member.
 	GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error)
diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go
@@ -25,6 +25,8 @@ package resources
 import (
 	"context"
 
+	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
+
 	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
 
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
@@ -100,4 +102,5 @@ type Context interface {
 	WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error
 	// GetBackup receives information about a backup resource
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
+	GetScope() scope.Scope
 }
diff --git a/pkg/deployment/resources/pod_termination.go b/pkg/deployment/resources/pod_termination.go
@@ -54,9 +54,9 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog
 		return nil
 	}
 
-	// Check node the pod is scheduled on
+	// Check node the pod is scheduled on. Only if not in namespaced scope
 	agentDataWillBeGone := false
-	if p.Spec.NodeName != "" {
+	if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" {
 		node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(p.Spec.NodeName, metav1.GetOptions{})
 		if k8sutil.IsNotFound(err) {
 			log.Warn().Msg("Node not found")
@@ -165,7 +165,7 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 
 	// Check node the pod is scheduled on
 	dbserverDataWillBeGone := false
-	if p.Spec.NodeName != "" {
+	if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" {
 		node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(p.Spec.NodeName, metav1.GetOptions{})
 		if k8sutil.IsNotFound(err) {
 			log.Warn().Msg("Node not found")
diff --git a/pkg/operator/crd.go b/pkg/operator/crd.go
diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go
diff --git a/pkg/operator/operator_deployment.go b/pkg/operator/operator_deployment.go
diff --git a/pkg/operator/scope/scope.go b/pkg/operator/scope/scope.go
diff --git a/pkg/operator/server_discovery_api.go b/pkg/operator/server_discovery_api.go
diff --git a/pkg/util/crd/crd.go b/pkg/util/crd/crd.go
