From bebf254c5e1b032403738a65a1e4634659ae7d2f Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Sun, 9 Feb 2025 23:08:21 +0530 Subject: [PATCH 01/10] nfs probes addition Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 679afaa6..b01bbdd0 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -907,6 +907,44 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp LabelSelector: &podLabelSelector, } + startupProbe := &core_v1.Probe{ + ProbeHandler: core_v1.ProbeHandler{ + Exec: &core_v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "1", name, nfsNamespace}, + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 2, + } + + readinessProbe := &core_v1.Probe{ + ProbeHandler: core_v1.ProbeHandler{ + Exec: &core_v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "2", name, nfsNamespace}, + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 2, + } + + livenessProbe := &core_v1.Probe{ + ProbeHandler: core_v1.ProbeHandler{ + Exec: &core_v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "3", name, nfsNamespace}, + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 2, + } + + containers := []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)} + containers[0].StartupProbe = startupProbe + containers[0].ReadinessProbe = readinessProbe + containers[0].LivenessProbe = livenessProbe + podSpec := core_v1.PodTemplateSpec{ ObjectMeta: meta_v1.ObjectMeta{ Name: name, @@ -915,7 +953,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp }, Spec: core_v1.PodSpec{ ServiceAccountName: nfsServiceAccount, - Containers: []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)}, + Containers: containers, RestartPolicy: core_v1.RestartPolicyAlways, Volumes: volumes, HostIPC: false, From 0a9cf2d9fca37e65bc6623c300684b80dc3b44a8 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Wed, 12 Feb 2025 01:31:38 -0800 Subject: [PATCH 02/10] Minor improvements Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index b01bbdd0..e83858c6 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -910,7 +910,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp startupProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "1", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "1", name, nfsNamespace}, }, }, InitialDelaySeconds: 10, @@ -921,7 +921,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp readinessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "2", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "2", name, nfsNamespace}, }, }, InitialDelaySeconds: 10, @@ -932,12 +932,12 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp livenessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "-c", "/nfsHealthCheck.sh", "3", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "3", name, nfsNamespace}, }, }, InitialDelaySeconds: 10, PeriodSeconds: 5, - TimeoutSeconds: 2, + TimeoutSeconds: 4, } containers := []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)} From 3f31303d0dca969373d474f6738677ff7f7dcb7a Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Mon, 24 Feb 2025 22:19:12 +0530 Subject: [PATCH 03/10] Added logic to add role and role binding for the nfs ServiceAccount Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index e83858c6..d56b7757 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -17,7 +17,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" apps_v1 "k8s.io/api/apps/v1" + auth_v1 "k8s.io/api/authorization/v1" core_v1 "k8s.io/api/core/v1" + rbac_v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -152,6 +154,17 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter return nil, true, err } + //check whether the service account can update and list the deployments + allowed := flavor.canRollOutDeployment(nfsServiceAccount, nfsResourceNamespace) + if !allowed { + log.Tracef("create a role and role binding for the service account %s", nfsServiceAccount) + err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace) + if err != nil { + log.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return nil, true, fmt.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + } + } + // create deployment with name hpe-nfs- deploymentName := fmt.Sprintf("%s%s", nfsPrefix, claim.ObjectMeta.UID) err = flavor.createNFSDeployment(deploymentName, nfsSpec, nfsResourceNamespace) @@ -221,6 +234,83 @@ func (flavor *Flavor) createServiceAccount(nfsNamespace string) error { return nil } +func (flavor *Flavor) canRollOutDeployment(nfsServiceAccount, nfsNamespace string) bool { + log.Tracef(">>>>> Service Account %s canRollOutDeployment with namespace %s", nfsServiceAccount, nfsNamespace) + defer log.Tracef("<<<<< canRollOutDeployment") + sar := &auth_v1.SubjectAccessReview{ + Spec: auth_v1.SubjectAccessReviewSpec{ + User: fmt.Sprintf("system:serviceaccount:%s:%s", nfsNamespace, nfsServiceAccount), + ResourceAttributes: &auth_v1.ResourceAttributes{ + Namespace: nfsNamespace, + Verb: "update", + Group: "apps", + Resource: "deployments", + }, + }, + } + + // Perform SubjectAccessReview + response, err := flavor.kubeClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, meta_v1.CreateOptions{}) + if err != nil { + return false + } + return response.Status.Allowed +} + +func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace string) error { + log.Tracef(">>>>> createRoleAndRoleBinding for ServiceAccount %s under namespace %s", nfsServiceAccount, nfsNamespace) + defer log.Tracef("<<<<< createRoleAndRoleBinding") + + roleName := nfsServiceAccount + "-deployment-rollout" + role := &rbac_v1.Role{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: roleName, + Namespace: nfsNamespace, + }, + Rules: []rbac_v1.PolicyRule{ + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"update", "patch"}, + }, + }, + } + + _, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.TODO(), role, meta_v1.CreateOptions{}) + if err != nil { + log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return err + } + + roleBindingName := nfsServiceAccount + "deployment-rollout-binding" + log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount) + roleBinding := &rbac_v1.RoleBinding{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: roleBindingName, + Namespace: nfsNamespace, + }, + Subjects: []rbac_v1.Subject{ + { + Kind: "ServiceAccount", + Name: nfsServiceAccount, + Namespace: nfsNamespace, + }, + }, + RoleRef: rbac_v1.RoleRef{ + Kind: "Role", + Name: roleName, + APIGroup: "rbac.authorization.k8s.io", + }, + } + _, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.TODO(), roleBinding, meta_v1.CreateOptions{}) + if err != nil { + log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return err + } + fmt.Println(" RoleBinding '%s for the ServiceAccount %s created successfully.", roleBindingName, nfsServiceAccount) + return nil +} + func (flavor *Flavor) createNFSConfigMap(nfsNamespace, hostDomain string) error { log.Tracef(">>>>> createNFSConfigMap with namespace %s, domain %s", nfsNamespace, hostDomain) defer log.Tracef("<<<<< createNFSConfigMap") From d2a08758cf269bef11747923525f65cc5ce303c6 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Thu, 27 Feb 2025 23:58:35 -0800 Subject: [PATCH 04/10] Improvements to the role and rolebinding creation Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 57 +++++++++++------------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index d56b7757..62df5ae9 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -154,15 +154,11 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter return nil, true, err } - //check whether the service account can update and list the deployments - allowed := flavor.canRollOutDeployment(nfsServiceAccount, nfsResourceNamespace) - if !allowed { - log.Tracef("create a role and role binding for the service account %s", nfsServiceAccount) - err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace) - if err != nil { - log.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) - return nil, true, fmt.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) - } + log.Tracef("Create a role and role binding for the service account %s", nfsServiceAccount) + err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace) + if err != nil { + log.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return nil, true, fmt.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) } // create deployment with name hpe-nfs- @@ -234,29 +230,6 @@ func (flavor *Flavor) createServiceAccount(nfsNamespace string) error { return nil } -func (flavor *Flavor) canRollOutDeployment(nfsServiceAccount, nfsNamespace string) bool { - log.Tracef(">>>>> Service Account %s canRollOutDeployment with namespace %s", nfsServiceAccount, nfsNamespace) - defer log.Tracef("<<<<< canRollOutDeployment") - sar := &auth_v1.SubjectAccessReview{ - Spec: auth_v1.SubjectAccessReviewSpec{ - User: fmt.Sprintf("system:serviceaccount:%s:%s", nfsNamespace, nfsServiceAccount), - ResourceAttributes: &auth_v1.ResourceAttributes{ - Namespace: nfsNamespace, - Verb: "update", - Group: "apps", - Resource: "deployments", - }, - }, - } - - // Perform SubjectAccessReview - response, err := flavor.kubeClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, meta_v1.CreateOptions{}) - if err != nil { - return false - } - return response.Status.Allowed -} - func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace string) error { log.Tracef(">>>>> createRoleAndRoleBinding for ServiceAccount %s under namespace %s", nfsServiceAccount, nfsNamespace) defer log.Tracef("<<<<< createRoleAndRoleBinding") @@ -271,15 +244,17 @@ func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace s { APIGroups: []string{"apps"}, Resources: []string{"deployments"}, - Verbs: []string{"update", "patch"}, + Verbs: []string{"update", "patch", "list", "get"}, }, }, } - _, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.TODO(), role, meta_v1.CreateOptions{}) + _, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.Background(), role, meta_v1.CreateOptions{}) if err != nil { - log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) - return err + if !errors.IsAlreadyExists(err) { + log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return err + } } roleBindingName := nfsServiceAccount + "deployment-rollout-binding" @@ -302,12 +277,14 @@ func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace s APIGroup: "rbac.authorization.k8s.io", }, } - _, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.TODO(), roleBinding, meta_v1.CreateOptions{}) + _, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.Background(), roleBinding, meta_v1.CreateOptions{}) if err != nil { - log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) - return err + if !errors.IsAlreadyExists(err) { + log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + return err + } } - fmt.Println(" RoleBinding '%s for the ServiceAccount %s created successfully.", roleBindingName, nfsServiceAccount) + log.Infof(" RoleBinding '%s for the ServiceAccount %s created successfully.", roleBindingName, nfsServiceAccount) return nil } From 28df14a737ea8a6548930a573b35e70c0d297202 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Fri, 28 Feb 2025 13:36:30 +0530 Subject: [PATCH 05/10] Made improvements for role and role binding functionality Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 62df5ae9..b7d3a7e1 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -17,7 +17,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" apps_v1 "k8s.io/api/apps/v1" - auth_v1 "k8s.io/api/authorization/v1" core_v1 "k8s.io/api/core/v1" rbac_v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -251,14 +250,17 @@ func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace s _, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.Background(), role, meta_v1.CreateOptions{}) if err != nil { - if !errors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) { + log.Infof("Role %s already exists.", roleName) + } else { log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) return err } + } else { + log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount) } roleBindingName := nfsServiceAccount + "deployment-rollout-binding" - log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount) roleBinding := &rbac_v1.RoleBinding{ ObjectMeta: meta_v1.ObjectMeta{ Name: roleBindingName, @@ -279,7 +281,10 @@ func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace s } _, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.Background(), roleBinding, meta_v1.CreateOptions{}) if err != nil { - if !errors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) { + log.Infof("RoleBinding %s already exists.", roleBinding) + return nil + } else { log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error()) return err } From e77f720e518abaa47aede4757463acebf52be2d0 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Mon, 24 Mar 2025 03:11:16 -0700 Subject: [PATCH 06/10] Incorporated review comments Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 90 +++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index b7d3a7e1..98b75be0 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -37,38 +37,42 @@ const ( defaultRLimitMemory = "2Gi" defaultRRequestMemory = "512Mi" - creationInterval = 60 // 300s with sleep interval of 5s - creationDelay = 5 * time.Second - defaultExportPath = "/export" - nfsResourceLimitsCPUKey = "nfsResourceLimitsCpuM" - nfsResourceRequestsCPUKey = "nfsResourceRequestsCpuM" - nfsResourceLimitsMemoryKey = "nfsResourceLimitsMemoryMi" - nfsResourceRequestsMemoryKey = "nfsResourceRequestsMemoryMi" - nfsMountOptionsKey = "nfsMountOptions" - nfsResourceLabelKey = "nfsResourceLabel" - nfsNodeSelectorKey = "csi.hpe.com/hpe-nfs" - nfsNodeSelectorDefaultValue = "true" - nfsNodeSelectorParamKey = "nfsNodeSelector" - nfsParentVolumeIDKey = "nfs-parent-volume-id" - nfsNamespaceKey = "nfsNamespace" - nfsSourceNamespaceKey = "csi.storage.k8s.io/pvc/namespace" - nfsSourcePVCNameKey = "csi.storage.k8s.io/pvc/name" - nfsProvisionerImageKey = "nfsProvisionerImage" - pvcKind = "PersistentVolumeClaim" - nfsConfigFile = "ganesha.conf" - nfsConfigMap = "hpe-nfs-config" - nfsServiceAccount = "hpe-csi-nfs-sa" - defaultPodLabelKey = "monitored-by" - defaultPodLabelValue = "hpe-csi" - nfsAffinityLabelKey = "spread-by" - nfsAffinityLabelValue = "hpe-nfs" - nfsDedicatedTolerationKey = "csi.hpe.com/hpe-nfs" - nfsProvisionedByKey = "provisioned-by" - nfsProvisionedFromKey = "provisioned-from" - nfsForeignStorageClassKey = "nfsForeignStorageClass" - nfsResourcesKey = "nfsResources" - nfsTolerationSecScKey = "nfsTolerationSeconds" - defaultNfsTolerationSeconds = 30 + creationInterval = 60 // 300s with sleep interval of 5s + creationDelay = 5 * time.Second + defaultExportPath = "/export" + nfsResourceLimitsCPUKey = "nfsResourceLimitsCpuM" + nfsResourceRequestsCPUKey = "nfsResourceRequestsCpuM" + nfsResourceLimitsMemoryKey = "nfsResourceLimitsMemoryMi" + nfsResourceRequestsMemoryKey = "nfsResourceRequestsMemoryMi" + nfsMountOptionsKey = "nfsMountOptions" + nfsResourceLabelKey = "nfsResourceLabel" + nfsNodeSelectorKey = "csi.hpe.com/hpe-nfs" + nfsNodeSelectorDefaultValue = "true" + nfsNodeSelectorParamKey = "nfsNodeSelector" + nfsParentVolumeIDKey = "nfs-parent-volume-id" + nfsNamespaceKey = "nfsNamespace" + nfsSourceNamespaceKey = "csi.storage.k8s.io/pvc/namespace" + nfsSourcePVCNameKey = "csi.storage.k8s.io/pvc/name" + nfsProvisionerImageKey = "nfsProvisionerImage" + pvcKind = "PersistentVolumeClaim" + nfsConfigFile = "ganesha.conf" + nfsConfigMap = "hpe-nfs-config" + nfsServiceAccount = "hpe-csi-nfs-sa" + defaultPodLabelKey = "monitored-by" + defaultPodLabelValue = "hpe-csi" + nfsAffinityLabelKey = "spread-by" + nfsAffinityLabelValue = "hpe-nfs" + nfsDedicatedTolerationKey = "csi.hpe.com/hpe-nfs" + nfsProvisionedByKey = "provisioned-by" + nfsProvisionedFromKey = "provisioned-from" + nfsForeignStorageClassKey = "nfsForeignStorageClass" + nfsResourcesKey = "nfsResources" + nfsTolerationSecScKey = "nfsTolerationSeconds" + defaultNfsTolerationSeconds = 30 + nfsProbeInitialDelaySeconds = 10 + nfsProbePeriodSeconds = 5 + nfsProbeTimeoutSeconds = 2 + nfsLivenessProbeTimeoutSeconds = 4 ) // NFSSpec for creating NFS resources @@ -156,8 +160,8 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter log.Tracef("Create a role and role binding for the service account %s", nfsServiceAccount) err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace) if err != nil { - log.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) - return nil, true, fmt.Errorf("Error occured while creating the role and rolebinding for the ServiceAccount %s:%s", nfsServiceAccount, err.Error()) + log.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error()) + return nil, true, fmt.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error()) } // create deployment with name hpe-nfs- @@ -985,9 +989,9 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "1", name, nfsNamespace}, }, }, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - TimeoutSeconds: 2, + InitialDelaySeconds: nfsProbeInitialDelaySeconds, + PeriodSeconds: nfsProbePeriodSeconds, + TimeoutSeconds: nfsProbeTimeoutSeconds, } readinessProbe := &core_v1.Probe{ @@ -996,9 +1000,9 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "2", name, nfsNamespace}, }, }, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - TimeoutSeconds: 2, + InitialDelaySeconds: nfsProbeInitialDelaySeconds, + PeriodSeconds: nfsProbePeriodSeconds, + TimeoutSeconds: nfsProbeTimeoutSeconds, } livenessProbe := &core_v1.Probe{ @@ -1007,9 +1011,9 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "3", name, nfsNamespace}, }, }, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - TimeoutSeconds: 4, + InitialDelaySeconds: nfsProbeInitialDelaySeconds, + PeriodSeconds: nfsProbePeriodSeconds, + TimeoutSeconds: nfsLivenessProbeTimeoutSeconds, } containers := []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)} From ac39b04ac20e4d6f0575560938db26ed990f4178 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Wed, 16 Apr 2025 00:17:47 +0530 Subject: [PATCH 07/10] Added deleteion of roles and role binding logic Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 59 ++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 98b75be0..9b38aa6c 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -70,9 +70,11 @@ const ( nfsTolerationSecScKey = "nfsTolerationSeconds" defaultNfsTolerationSeconds = 30 nfsProbeInitialDelaySeconds = 10 - nfsProbePeriodSeconds = 5 + nfsProbePeriodSeconds = 5 nfsProbeTimeoutSeconds = 2 nfsLivenessProbeTimeoutSeconds = 4 + nfsRoleBindingSuffix = "-deployment-rollout-binding" + nfsRoleSuffix = "-deployment-rollout-role" ) // NFSSpec for creating NFS resources @@ -157,8 +159,8 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter return nil, true, err } - log.Tracef("Create a role and role binding for the service account %s", nfsServiceAccount) - err = flavor.createRoleAndRoleBinding(nfsServiceAccount, nfsResourceNamespace) + log.Tracef("Create a role and role binding for the pv %s and service account %s", pvName, nfsServiceAccount) + err = flavor.createRoleAndRoleBinding(pvName, nfsServiceAccount, nfsResourceNamespace) if err != nil { log.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error()) return nil, true, fmt.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error()) @@ -233,11 +235,12 @@ func (flavor *Flavor) createServiceAccount(nfsNamespace string) error { return nil } -func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace string) error { - log.Tracef(">>>>> createRoleAndRoleBinding for ServiceAccount %s under namespace %s", nfsServiceAccount, nfsNamespace) +func (flavor *Flavor) createRoleAndRoleBinding(pvName, nfsServiceAccount, nfsNamespace string) error { + log.Tracef(">>>>> createRoleAndRoleBinding for PV %s and ServiceAccount %s under namespace %s", pvName, nfsServiceAccount, nfsNamespace) defer log.Tracef("<<<<< createRoleAndRoleBinding") - roleName := nfsServiceAccount + "-deployment-rollout" + pvName = strings.TrimPrefix(pvName, "pvc-") + roleName := nfsPrefix + pvName + nfsRoleSuffix role := &rbac_v1.Role{ ObjectMeta: meta_v1.ObjectMeta{ Name: roleName, @@ -264,7 +267,7 @@ func (flavor *Flavor) createRoleAndRoleBinding(nfsServiceAccount, nfsNamespace s log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount) } - roleBindingName := nfsServiceAccount + "deployment-rollout-binding" + roleBindingName := nfsPrefix + pvName + nfsRoleBindingSuffix roleBinding := &rbac_v1.RoleBinding{ ObjectMeta: meta_v1.ObjectMeta{ Name: roleBindingName, @@ -355,7 +358,7 @@ EXPORT func (flavor *Flavor) RollbackNFSResources(nfsResourceName string, nfsNamespace string) error { log.Tracef(">>>>> RollbackNFSResources with name %s namespace %s", nfsResourceName, nfsNamespace) defer log.Tracef("<<<<< RollbackNFSResources") - err := flavor.deleteNFSResources(nfsResourceName, nfsNamespace) + err := flavor.deleteNFSResources("", nfsResourceName, nfsNamespace) if err != nil { return err } @@ -375,7 +378,7 @@ func (flavor *Flavor) DeleteNFSVolume(volumeID string) error { if err != nil { return err } - err = flavor.deleteNFSResources(nfsResourceName, nfsNamespace) + err = flavor.deleteNFSResources(volumeID, nfsResourceName, nfsNamespace) if err != nil { return err } @@ -383,7 +386,7 @@ func (flavor *Flavor) DeleteNFSVolume(volumeID string) error { return err } -func (flavor *Flavor) deleteNFSResources(nfsResourceName, nfsNamespace string) (err error) { +func (flavor *Flavor) deleteNFSResources(volumeID, nfsResourceName, nfsNamespace string) (err error) { // delete deployment deployment/hpe-nfs- err = flavor.deleteNFSDeployment(nfsResourceName, nfsNamespace) if err != nil { @@ -402,9 +405,45 @@ func (flavor *Flavor) deleteNFSResources(nfsResourceName, nfsNamespace string) ( if err != nil { log.Errorf("unable to delete nfs service %s as part of cleanup, err %s", nfsResourceName, err.Error()) } + + roleName := nfsPrefix + volumeID + nfsRoleSuffix + err = flavor.deleteNFSRole(volumeID, roleName, nfsNamespace) + if err != nil { + log.Errorf("unable to delete role %s as part of cleanup, err %s", roleName, err.Error()) + } + + roleBindingName := nfsPrefix + volumeID + nfsRoleBindingSuffix + err = flavor.deleteNFSRoleBinding(volumeID, roleBindingName, nfsNamespace) + if err != nil { + log.Errorf("unable to delete role binding %s as part of cleanup, err %s", roleBindingName, err.Error()) + } return err } +func (flavor *Flavor) deleteNFSRole(volumeID, roleName, nfsNamespace string) error { + log.Tracef(">>>>> deleteNFSRole for the volume %s", volumeID) + defer log.Tracef("<<<<< deleteNFSRole") + err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Delete(context.Background(), roleName, meta_v1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + log.Errorf("failed to delete the role %s for volume %s, err %+v", roleName, volumeID, err) + return err + } + log.Infof("Triggered deletion of role %s", roleName) + return nil +} + +func (flavor *Flavor) deleteNFSRoleBinding(volumeID, roleBindingName, nfsNamespace string) error { + log.Tracef(">>>>> deleteNFSRoleBinding for the volume %s", volumeID) + defer log.Tracef("<<<<< deleteNFSRoleBinding") + err := flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Delete(context.Background(), roleBindingName, meta_v1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + log.Errorf("failed to delete the role binding %s for volume %s, err %+v", roleBindingName, volumeID, err) + return err + } + log.Infof("Triggered deletion of role binding %s", roleBindingName) + return nil +} + func (flavor *Flavor) getNFSResourceNameByVolumeID(volumeID string) (string, error) { // get underlying by NFS(RWX) PV volume-id pv, err := flavor.getPVByNFSLabel(nfsParentVolumeIDKey, volumeID) From a393eb799aa68179ae349b7bab7570e2df87b9b5 Mon Sep 17 00:00:00 2001 From: AnushaY1916 Date: Wed, 16 Apr 2025 17:57:08 +0530 Subject: [PATCH 08/10] Added some constants for the probes input Signed-off-by: AnushaY1916 --- pkg/flavor/kubernetes/nfs.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 9b38aa6c..983ade52 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -75,6 +75,9 @@ const ( nfsLivenessProbeTimeoutSeconds = 4 nfsRoleBindingSuffix = "-deployment-rollout-binding" nfsRoleSuffix = "-deployment-rollout-role" + READINESS = "READINESS" + STARTUP = "STARTUP" + LIVENESS = "LIVENESS" ) // NFSSpec for creating NFS resources @@ -1025,7 +1028,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp startupProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "1", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", STARTUP, name, nfsNamespace}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, @@ -1036,7 +1039,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp readinessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "2", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", READINESS, name, nfsNamespace}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, @@ -1047,7 +1050,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp livenessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", "3", name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", LIVENESS, name, nfsNamespace}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, From 91ab461f5e5f5efe92d7d11192e1aa8a498bd59d Mon Sep 17 00:00:00 2001 From: Michael Mattsson Date: Thu, 17 Apr 2025 23:47:47 -0700 Subject: [PATCH 09/10] Tweaks Signed-off-by: Michael Mattsson --- pkg/flavor/kubernetes/nfs.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 983ade52..901e79a2 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -69,15 +69,15 @@ const ( nfsResourcesKey = "nfsResources" nfsTolerationSecScKey = "nfsTolerationSeconds" defaultNfsTolerationSeconds = 30 - nfsProbeInitialDelaySeconds = 10 - nfsProbePeriodSeconds = 5 + nfsProbeInitialDelaySeconds = 3 + nfsProbePeriodSeconds = 30 nfsProbeTimeoutSeconds = 2 - nfsLivenessProbeTimeoutSeconds = 4 + nfsLivenessProbeTimeoutSeconds = 90 + nfsProbeReadinessKey = "READINESS" + nfsProbeStartupKey = "STARTUP" + nfsProbeLivenessKey = "LIVENESS" nfsRoleBindingSuffix = "-deployment-rollout-binding" nfsRoleSuffix = "-deployment-rollout-role" - READINESS = "READINESS" - STARTUP = "STARTUP" - LIVENESS = "LIVENESS" ) // NFSSpec for creating NFS resources @@ -1028,7 +1028,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp startupProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", STARTUP, name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeStartupKey}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, @@ -1039,7 +1039,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp readinessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", READINESS, name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeReadinessKey}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, @@ -1050,7 +1050,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp livenessProbe := &core_v1.Probe{ ProbeHandler: core_v1.ProbeHandler{ Exec: &core_v1.ExecAction{ - Command: []string{"/bin/sh", "/nfsHealthCheck.sh", LIVENESS, name, nfsNamespace}, + Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeLivenessKey}, }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, From bf3c607ba81deed2805e74a57a8aecd3d691d565 Mon Sep 17 00:00:00 2001 From: Michael Mattsson Date: Fri, 18 Apr 2025 11:58:37 -0700 Subject: [PATCH 10/10] Post e2e analysis Signed-off-by: Michael Mattsson --- pkg/flavor/kubernetes/nfs.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/flavor/kubernetes/nfs.go b/pkg/flavor/kubernetes/nfs.go index 901e79a2..d570a231 100644 --- a/pkg/flavor/kubernetes/nfs.go +++ b/pkg/flavor/kubernetes/nfs.go @@ -69,9 +69,10 @@ const ( nfsResourcesKey = "nfsResources" nfsTolerationSecScKey = "nfsTolerationSeconds" defaultNfsTolerationSeconds = 30 - nfsProbeInitialDelaySeconds = 3 - nfsProbePeriodSeconds = 30 - nfsProbeTimeoutSeconds = 2 + nfsProbeInitialDelaySeconds = 0 + nfsProbePeriodSeconds = 10 + nfsProbeTimeoutSeconds = 5 + nfsLivenessProbePeriodSeconds = 30 nfsLivenessProbeTimeoutSeconds = 90 nfsProbeReadinessKey = "READINESS" nfsProbeStartupKey = "STARTUP" @@ -1054,7 +1055,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp }, }, InitialDelaySeconds: nfsProbeInitialDelaySeconds, - PeriodSeconds: nfsProbePeriodSeconds, + PeriodSeconds: nfsLivenessProbePeriodSeconds, TimeoutSeconds: nfsLivenessProbeTimeoutSeconds, }