CON-3113 - Implementation of probes for NFS deployments

pkg/flavor/kubernetes/nfs.go

@@ -18,6 +18,7 @@ import (
 	"google.golang.org/grpc/status"
 	apps_v1 "k8s.io/api/apps/v1"
 	core_v1 "k8s.io/api/core/v1"
+	rbac_v1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,38 +37,42 @@ const (
 	defaultRLimitMemory   = "2Gi"
 	defaultRRequestMemory = "512Mi"
 
-	creationInterval             = 60 // 300s with sleep interval of 5s
-	creationDelay                = 5 * time.Second
-	defaultExportPath            = "/export"
-	nfsResourceLimitsCPUKey      = "nfsResourceLimitsCpuM"
-	nfsResourceRequestsCPUKey    = "nfsResourceRequestsCpuM"
-	nfsResourceLimitsMemoryKey   = "nfsResourceLimitsMemoryMi"
-	nfsResourceRequestsMemoryKey = "nfsResourceRequestsMemoryMi"
-	nfsMountOptionsKey           = "nfsMountOptions"
-	nfsResourceLabelKey          = "nfsResourceLabel"
-	nfsNodeSelectorKey           = "csi.hpe.com/hpe-nfs"
-	nfsNodeSelectorDefaultValue  = "true"
-	nfsNodeSelectorParamKey      = "nfsNodeSelector"
-	nfsParentVolumeIDKey         = "nfs-parent-volume-id"
-	nfsNamespaceKey              = "nfsNamespace"
-	nfsSourceNamespaceKey        = "csi.storage.k8s.io/pvc/namespace"
-	nfsSourcePVCNameKey          = "csi.storage.k8s.io/pvc/name"
-	nfsProvisionerImageKey       = "nfsProvisionerImage"
-	pvcKind                      = "PersistentVolumeClaim"
-	nfsConfigFile                = "ganesha.conf"
-	nfsConfigMap                 = "hpe-nfs-config"
-	nfsServiceAccount            = "hpe-csi-nfs-sa"
-	defaultPodLabelKey           = "monitored-by"
-	defaultPodLabelValue         = "hpe-csi"
-	nfsAffinityLabelKey          = "spread-by"
-	nfsAffinityLabelValue        = "hpe-nfs"
-	nfsDedicatedTolerationKey    = "csi.hpe.com/hpe-nfs"
-	nfsProvisionedByKey          = "provisioned-by"
-	nfsProvisionedFromKey        = "provisioned-from"
-	nfsForeignStorageClassKey    = "nfsForeignStorageClass"
-	nfsResourcesKey              = "nfsResources"
-	nfsTolerationSecScKey        = "nfsTolerationSeconds"
-	defaultNfsTolerationSeconds  = 30
+	creationInterval               = 60 // 300s with sleep interval of 5s
+	creationDelay                  = 5 * time.Second
+	defaultExportPath              = "/export"
+	nfsResourceLimitsCPUKey        = "nfsResourceLimitsCpuM"
+	nfsResourceRequestsCPUKey      = "nfsResourceRequestsCpuM"
+	nfsResourceLimitsMemoryKey     = "nfsResourceLimitsMemoryMi"
+	nfsResourceRequestsMemoryKey   = "nfsResourceRequestsMemoryMi"
+	nfsMountOptionsKey             = "nfsMountOptions"
+	nfsResourceLabelKey            = "nfsResourceLabel"
+	nfsNodeSelectorKey             = "csi.hpe.com/hpe-nfs"
+	nfsNodeSelectorDefaultValue    = "true"
+	nfsNodeSelectorParamKey        = "nfsNodeSelector"
+	nfsParentVolumeIDKey           = "nfs-parent-volume-id"
+	nfsNamespaceKey                = "nfsNamespace"
+	nfsSourceNamespaceKey          = "csi.storage.k8s.io/pvc/namespace"
+	nfsSourcePVCNameKey            = "csi.storage.k8s.io/pvc/name"
+	nfsProvisionerImageKey         = "nfsProvisionerImage"
+	pvcKind                        = "PersistentVolumeClaim"
+	nfsConfigFile                  = "ganesha.conf"
+	nfsConfigMap                   = "hpe-nfs-config"
+	nfsServiceAccount              = "hpe-csi-nfs-sa"
+	defaultPodLabelKey             = "monitored-by"
+	defaultPodLabelValue           = "hpe-csi"
+	nfsAffinityLabelKey            = "spread-by"
+	nfsAffinityLabelValue          = "hpe-nfs"
+	nfsDedicatedTolerationKey      = "csi.hpe.com/hpe-nfs"
+	nfsProvisionedByKey            = "provisioned-by"
+	nfsProvisionedFromKey          = "provisioned-from"
+	nfsForeignStorageClassKey      = "nfsForeignStorageClass"
+	nfsResourcesKey                = "nfsResources"
+	nfsTolerationSecScKey          = "nfsTolerationSeconds"
+	defaultNfsTolerationSeconds    = 30
+	nfsProbeInitialDelaySeconds    = 10
+	nfsProbePeriodSeconds	       = 5
+	nfsProbeTimeoutSeconds         = 2
+	nfsLivenessProbeTimeoutSeconds = 4
 )
 
 // NFSSpec for creating NFS resources
@@ -152,6 +157,13 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter
 		return nil, true, err
 	}
 
+	log.Tracef("Create a role and role binding for the service account %s", nfsServiceAccount)
+	err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace)
+	if err != nil {
+		log.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error())
+		return nil, true, fmt.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error())
+	}
+
 	// create deployment with name hpe-nfs-<originalclaim-uid>
 	deploymentName := fmt.Sprintf("%s%s", nfsPrefix, claim.ObjectMeta.UID)
 	err = flavor.createNFSDeployment(deploymentName, nfsSpec, nfsResourceNamespace)
@@ -221,6 +233,70 @@ func (flavor *Flavor) createServiceAccount(nfsNamespace string) error {
 	return nil
 }
 
+func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace string) error {
+	log.Tracef(">>>>> createRoleAndRoleBinding for ServiceAccount %s under namespace %s", nfsServiceAccount, nfsNamespace)
+	defer log.Tracef("<<<<< createRoleAndRoleBinding")
+
+	roleName := nfsServiceAccount + "-deployment-rollout"
+	role := &rbac_v1.Role{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      roleName,
+			Namespace: nfsNamespace,
+		},
+		Rules: []rbac_v1.PolicyRule{
+			{
+				APIGroups: []string{"apps"},
+				Resources: []string{"deployments"},
+				Verbs:     []string{"update", "patch", "list", "get"},
+			},
+		},
+	}
+
+	_, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.Background(), role, meta_v1.CreateOptions{})
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			log.Infof("Role %s already exists.", roleName)
+		} else {
+			log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error())
+			return err
+		}
+	} else {
+		log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount)
+	}
+
+	roleBindingName := nfsServiceAccount + "deployment-rollout-binding"
+	roleBinding := &rbac_v1.RoleBinding{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      roleBindingName,
+			Namespace: nfsNamespace,
+		},
+		Subjects: []rbac_v1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      nfsServiceAccount,
+				Namespace: nfsNamespace,
+			},
+		},
+		RoleRef: rbac_v1.RoleRef{
+			Kind:     "Role",
+			Name:     roleName,
+			APIGroup: "rbac.authorization.k8s.io",
+		},
+	}
+	_, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.Background(), roleBinding, meta_v1.CreateOptions{})
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			log.Infof("RoleBinding %s already exists.", roleBinding)
+			return nil
+		} else {
+			log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error())
+			return err
+		}
+	}
+	log.Infof(" RoleBinding '%s for the ServiceAccount %s created successfully.", roleBindingName, nfsServiceAccount)
+	return nil
+}
+
 func (flavor *Flavor) createNFSConfigMap(nfsNamespace, hostDomain string) error {
 	log.Tracef(">>>>> createNFSConfigMap with namespace %s, domain %s", nfsNamespace, hostDomain)
 	defer log.Tracef("<<<<< createNFSConfigMap")
@@ -907,6 +983,44 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp
 		LabelSelector:     &podLabelSelector,
 	}
 
+	startupProbe := &core_v1.Probe{
+		ProbeHandler: core_v1.ProbeHandler{
+			Exec: &core_v1.ExecAction{
+				Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "1", name, nfsNamespace},
+			},
+		},
+		InitialDelaySeconds: nfsProbeInitialDelaySeconds,
+		PeriodSeconds:       nfsProbePeriodSeconds,
+		TimeoutSeconds:      nfsProbeTimeoutSeconds,
+	}
+
+	readinessProbe := &core_v1.Probe{
+		ProbeHandler: core_v1.ProbeHandler{
+			Exec: &core_v1.ExecAction{
+				Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "2", name, nfsNamespace},
+			},
+		},
+		InitialDelaySeconds: nfsProbeInitialDelaySeconds,
+		PeriodSeconds:       nfsProbePeriodSeconds,
+		TimeoutSeconds:      nfsProbeTimeoutSeconds,
+	}
+
+	livenessProbe := &core_v1.Probe{
+		ProbeHandler: core_v1.ProbeHandler{
+			Exec: &core_v1.ExecAction{
+				Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "3", name, nfsNamespace},
+			},
+		},
+		InitialDelaySeconds: nfsProbeInitialDelaySeconds,
+		PeriodSeconds:       nfsProbePeriodSeconds,
+		TimeoutSeconds:      nfsLivenessProbeTimeoutSeconds,
+	}
+
+	containers := []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)}
+	containers[0].StartupProbe = startupProbe
+	containers[0].ReadinessProbe = readinessProbe
+	containers[0].LivenessProbe = livenessProbe
+
 	podSpec := core_v1.PodTemplateSpec{
 		ObjectMeta: meta_v1.ObjectMeta{
 			Name:        name,
@@ -915,7 +1029,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp
 		},
 		Spec: core_v1.PodSpec{
 			ServiceAccountName:        nfsServiceAccount,
-			Containers:                []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)},
+			Containers:                containers,
 			RestartPolicy:             core_v1.RestartPolicyAlways,
 			Volumes:                   volumes,
 			HostIPC:                   false,