Skip to content

Commit e21ffa7

Browse files
AnushaY1916datamattsson
AnushaY1916
and
datamattsson
authored
CON-3113 - Implementation of probes for NFS deployments (#439)
* nfs probes addition Signed-off-by: AnushaY1916 <[email protected]> * Minor improvements Signed-off-by: AnushaY1916 <[email protected]> * Added logic to add role and role binding for the nfs ServiceAccount Signed-off-by: AnushaY1916 <[email protected]> * Improvements to the role and rolebinding creation Signed-off-by: AnushaY1916 <[email protected]> * Made improvements for role and role binding functionality Signed-off-by: AnushaY1916 <[email protected]> * Incorporated review comments Signed-off-by: AnushaY1916 <[email protected]> * Added deleteion of roles and role binding logic Signed-off-by: AnushaY1916 <[email protected]> * Added some constants for the probes input Signed-off-by: AnushaY1916 <[email protected]> * Tweaks Signed-off-by: Michael Mattsson <[email protected]> * Post e2e analysis Signed-off-by: Michael Mattsson <[email protected]> --------- Signed-off-by: AnushaY1916 <[email protected]> Signed-off-by: Michael Mattsson <[email protected]> Co-authored-by: Michael Mattsson <[email protected]>
1 parent 9a5b25b commit e21ffa7

File tree

1 file changed

+193
-36
lines changed

1 file changed

+193
-36
lines changed

‎pkg/flavor/kubernetes/nfs.go‎

Lines changed: 193 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"google.golang.org/grpc/status"
1919
apps_v1 "k8s.io/api/apps/v1"
2020
core_v1 "k8s.io/api/core/v1"
21+
rbac_v1 "k8s.io/api/rbac/v1"
2122
"k8s.io/apimachinery/pkg/api/errors"
2223
"k8s.io/apimachinery/pkg/api/resource"
2324
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,38 +37,48 @@ const (
3637
defaultRLimitMemory = "2Gi"
3738
defaultRRequestMemory = "512Mi"
3839

39-
creationInterval = 60 // 300s with sleep interval of 5s
40-
creationDelay = 5 * time.Second
41-
defaultExportPath = "/export"
42-
nfsResourceLimitsCPUKey = "nfsResourceLimitsCpuM"
43-
nfsResourceRequestsCPUKey = "nfsResourceRequestsCpuM"
44-
nfsResourceLimitsMemoryKey = "nfsResourceLimitsMemoryMi"
45-
nfsResourceRequestsMemoryKey = "nfsResourceRequestsMemoryMi"
46-
nfsMountOptionsKey = "nfsMountOptions"
47-
nfsResourceLabelKey = "nfsResourceLabel"
48-
nfsNodeSelectorKey = "csi.hpe.com/hpe-nfs"
49-
nfsNodeSelectorDefaultValue = "true"
50-
nfsNodeSelectorParamKey = "nfsNodeSelector"
51-
nfsParentVolumeIDKey = "nfs-parent-volume-id"
52-
nfsNamespaceKey = "nfsNamespace"
53-
nfsSourceNamespaceKey = "csi.storage.k8s.io/pvc/namespace"
54-
nfsSourcePVCNameKey = "csi.storage.k8s.io/pvc/name"
55-
nfsProvisionerImageKey = "nfsProvisionerImage"
56-
pvcKind = "PersistentVolumeClaim"
57-
nfsConfigFile = "ganesha.conf"
58-
nfsConfigMap = "hpe-nfs-config"
59-
nfsServiceAccount = "hpe-csi-nfs-sa"
60-
defaultPodLabelKey = "monitored-by"
61-
defaultPodLabelValue = "hpe-csi"
62-
nfsAffinityLabelKey = "spread-by"
63-
nfsAffinityLabelValue = "hpe-nfs"
64-
nfsDedicatedTolerationKey = "csi.hpe.com/hpe-nfs"
65-
nfsProvisionedByKey = "provisioned-by"
66-
nfsProvisionedFromKey = "provisioned-from"
67-
nfsForeignStorageClassKey = "nfsForeignStorageClass"
68-
nfsResourcesKey = "nfsResources"
69-
nfsTolerationSecScKey = "nfsTolerationSeconds"
70-
defaultNfsTolerationSeconds = 30
40+
creationInterval = 60 // 300s with sleep interval of 5s
41+
creationDelay = 5 * time.Second
42+
defaultExportPath = "/export"
43+
nfsResourceLimitsCPUKey = "nfsResourceLimitsCpuM"
44+
nfsResourceRequestsCPUKey = "nfsResourceRequestsCpuM"
45+
nfsResourceLimitsMemoryKey = "nfsResourceLimitsMemoryMi"
46+
nfsResourceRequestsMemoryKey = "nfsResourceRequestsMemoryMi"
47+
nfsMountOptionsKey = "nfsMountOptions"
48+
nfsResourceLabelKey = "nfsResourceLabel"
49+
nfsNodeSelectorKey = "csi.hpe.com/hpe-nfs"
50+
nfsNodeSelectorDefaultValue = "true"
51+
nfsNodeSelectorParamKey = "nfsNodeSelector"
52+
nfsParentVolumeIDKey = "nfs-parent-volume-id"
53+
nfsNamespaceKey = "nfsNamespace"
54+
nfsSourceNamespaceKey = "csi.storage.k8s.io/pvc/namespace"
55+
nfsSourcePVCNameKey = "csi.storage.k8s.io/pvc/name"
56+
nfsProvisionerImageKey = "nfsProvisionerImage"
57+
pvcKind = "PersistentVolumeClaim"
58+
nfsConfigFile = "ganesha.conf"
59+
nfsConfigMap = "hpe-nfs-config"
60+
nfsServiceAccount = "hpe-csi-nfs-sa"
61+
defaultPodLabelKey = "monitored-by"
62+
defaultPodLabelValue = "hpe-csi"
63+
nfsAffinityLabelKey = "spread-by"
64+
nfsAffinityLabelValue = "hpe-nfs"
65+
nfsDedicatedTolerationKey = "csi.hpe.com/hpe-nfs"
66+
nfsProvisionedByKey = "provisioned-by"
67+
nfsProvisionedFromKey = "provisioned-from"
68+
nfsForeignStorageClassKey = "nfsForeignStorageClass"
69+
nfsResourcesKey = "nfsResources"
70+
nfsTolerationSecScKey = "nfsTolerationSeconds"
71+
defaultNfsTolerationSeconds = 30
72+
nfsProbeInitialDelaySeconds = 0
73+
nfsProbePeriodSeconds = 10
74+
nfsProbeTimeoutSeconds = 5
75+
nfsLivenessProbePeriodSeconds = 30
76+
nfsLivenessProbeTimeoutSeconds = 90
77+
nfsProbeReadinessKey = "READINESS"
78+
nfsProbeStartupKey = "STARTUP"
79+
nfsProbeLivenessKey = "LIVENESS"
80+
nfsRoleBindingSuffix = "-deployment-rollout-binding"
81+
nfsRoleSuffix = "-deployment-rollout-role"
7182
)
7283

7384
// NFSSpec for creating NFS resources
@@ -152,6 +163,13 @@ func (flavor *Flavor) CreateNFSVolume(pvName string, reqVolSize int64, parameter
152163
return nil, true, err
153164
}
154165

166+
log.Tracef("Create a role and role binding for the pv %s and service account %s", pvName, nfsServiceAccount)
167+
err = flavor.createRoleAndRoleBinding(pvName, nfsServiceAccount, nfsResourceNamespace)
168+
if err != nil {
169+
log.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error())
170+
return nil, true, fmt.Errorf("error occured while creating the role and rolebinding for the service account %s:%s", nfsServiceAccount, err.Error())
171+
}
172+
155173
// create deployment with name hpe-nfs-<originalclaim-uid>
156174
deploymentName := fmt.Sprintf("%s%s", nfsPrefix, claim.ObjectMeta.UID)
157175
err = flavor.createNFSDeployment(deploymentName, nfsSpec, nfsResourceNamespace)
@@ -221,6 +239,71 @@ func (flavor *Flavor) createServiceAccount(nfsNamespace string) error {
221239
return nil
222240
}
223241

242+
func (flavor *Flavor) createRoleAndRoleBinding(pvName, nfsServiceAccount, nfsNamespace string) error {
243+
log.Tracef(">>>>> createRoleAndRoleBinding for PV %s and ServiceAccount %s under namespace %s", pvName, nfsServiceAccount, nfsNamespace)
244+
defer log.Tracef("<<<<< createRoleAndRoleBinding")
245+
246+
pvName = strings.TrimPrefix(pvName, "pvc-")
247+
roleName := nfsPrefix + pvName + nfsRoleSuffix
248+
role := &rbac_v1.Role{
249+
ObjectMeta: meta_v1.ObjectMeta{
250+
Name: roleName,
251+
Namespace: nfsNamespace,
252+
},
253+
Rules: []rbac_v1.PolicyRule{
254+
{
255+
APIGroups: []string{"apps"},
256+
Resources: []string{"deployments"},
257+
Verbs: []string{"update", "patch", "list", "get"},
258+
},
259+
},
260+
}
261+
262+
_, err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Create(context.Background(), role, meta_v1.CreateOptions{})
263+
if err != nil {
264+
if errors.IsAlreadyExists(err) {
265+
log.Infof("Role %s already exists.", roleName)
266+
} else {
267+
log.Errorf("Error occured while creating the role for ServiceAccount %s:%s", nfsServiceAccount, err.Error())
268+
return err
269+
}
270+
} else {
271+
log.Infof("Role %s for the the ServiceAccount %s created successfully", roleName, nfsServiceAccount)
272+
}
273+
274+
roleBindingName := nfsPrefix + pvName + nfsRoleBindingSuffix
275+
roleBinding := &rbac_v1.RoleBinding{
276+
ObjectMeta: meta_v1.ObjectMeta{
277+
Name: roleBindingName,
278+
Namespace: nfsNamespace,
279+
},
280+
Subjects: []rbac_v1.Subject{
281+
{
282+
Kind: "ServiceAccount",
283+
Name: nfsServiceAccount,
284+
Namespace: nfsNamespace,
285+
},
286+
},
287+
RoleRef: rbac_v1.RoleRef{
288+
Kind: "Role",
289+
Name: roleName,
290+
APIGroup: "rbac.authorization.k8s.io",
291+
},
292+
}
293+
_, err = flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Create(context.Background(), roleBinding, meta_v1.CreateOptions{})
294+
if err != nil {
295+
if errors.IsAlreadyExists(err) {
296+
log.Infof("RoleBinding %s already exists.", roleBinding)
297+
return nil
298+
} else {
299+
log.Errorf("Error occured while creating the role binding for ServiceAccount %s:%s", nfsServiceAccount, err.Error())
300+
return err
301+
}
302+
}
303+
log.Infof(" RoleBinding '%s for the ServiceAccount %s created successfully.", roleBindingName, nfsServiceAccount)
304+
return nil
305+
}
306+
224307
func (flavor *Flavor) createNFSConfigMap(nfsNamespace, hostDomain string) error {
225308
log.Tracef(">>>>> createNFSConfigMap with namespace %s, domain %s", nfsNamespace, hostDomain)
226309
defer log.Tracef("<<<<< createNFSConfigMap")
@@ -279,7 +362,7 @@ EXPORT
279362
func (flavor *Flavor) RollbackNFSResources(nfsResourceName string, nfsNamespace string) error {
280363
log.Tracef(">>>>> RollbackNFSResources with name %s namespace %s", nfsResourceName, nfsNamespace)
281364
defer log.Tracef("<<<<< RollbackNFSResources")
282-
err := flavor.deleteNFSResources(nfsResourceName, nfsNamespace)
365+
err := flavor.deleteNFSResources("", nfsResourceName, nfsNamespace)
283366
if err != nil {
284367
return err
285368
}
@@ -299,15 +382,15 @@ func (flavor *Flavor) DeleteNFSVolume(volumeID string) error {
299382
if err != nil {
300383
return err
301384
}
302-
err = flavor.deleteNFSResources(nfsResourceName, nfsNamespace)
385+
err = flavor.deleteNFSResources(volumeID, nfsResourceName, nfsNamespace)
303386
if err != nil {
304387
return err
305388
}
306389

307390
return err
308391
}
309392

310-
func (flavor *Flavor) deleteNFSResources(nfsResourceName, nfsNamespace string) (err error) {
393+
func (flavor *Flavor) deleteNFSResources(volumeID, nfsResourceName, nfsNamespace string) (err error) {
311394
// delete deployment deployment/hpe-nfs-<originalclaim-uid>
312395
err = flavor.deleteNFSDeployment(nfsResourceName, nfsNamespace)
313396
if err != nil {
@@ -326,9 +409,45 @@ func (flavor *Flavor) deleteNFSResources(nfsResourceName, nfsNamespace string) (
326409
if err != nil {
327410
log.Errorf("unable to delete nfs service %s as part of cleanup, err %s", nfsResourceName, err.Error())
328411
}
412+
413+
roleName := nfsPrefix + volumeID + nfsRoleSuffix
414+
err = flavor.deleteNFSRole(volumeID, roleName, nfsNamespace)
415+
if err != nil {
416+
log.Errorf("unable to delete role %s as part of cleanup, err %s", roleName, err.Error())
417+
}
418+
419+
roleBindingName := nfsPrefix + volumeID + nfsRoleBindingSuffix
420+
err = flavor.deleteNFSRoleBinding(volumeID, roleBindingName, nfsNamespace)
421+
if err != nil {
422+
log.Errorf("unable to delete role binding %s as part of cleanup, err %s", roleBindingName, err.Error())
423+
}
329424
return err
330425
}
331426

427+
func (flavor *Flavor) deleteNFSRole(volumeID, roleName, nfsNamespace string) error {
428+
log.Tracef(">>>>> deleteNFSRole for the volume %s", volumeID)
429+
defer log.Tracef("<<<<< deleteNFSRole")
430+
err := flavor.kubeClient.RbacV1().Roles(nfsNamespace).Delete(context.Background(), roleName, meta_v1.DeleteOptions{})
431+
if err != nil && !errors.IsNotFound(err) {
432+
log.Errorf("failed to delete the role %s for volume %s, err %+v", roleName, volumeID, err)
433+
return err
434+
}
435+
log.Infof("Triggered deletion of role %s", roleName)
436+
return nil
437+
}
438+
439+
func (flavor *Flavor) deleteNFSRoleBinding(volumeID, roleBindingName, nfsNamespace string) error {
440+
log.Tracef(">>>>> deleteNFSRoleBinding for the volume %s", volumeID)
441+
defer log.Tracef("<<<<< deleteNFSRoleBinding")
442+
err := flavor.kubeClient.RbacV1().RoleBindings(nfsNamespace).Delete(context.Background(), roleBindingName, meta_v1.DeleteOptions{})
443+
if err != nil && !errors.IsNotFound(err) {
444+
log.Errorf("failed to delete the role binding %s for volume %s, err %+v", roleBindingName, volumeID, err)
445+
return err
446+
}
447+
log.Infof("Triggered deletion of role binding %s", roleBindingName)
448+
return nil
449+
}
450+
332451
func (flavor *Flavor) getNFSResourceNameByVolumeID(volumeID string) (string, error) {
333452
// get underlying by NFS(RWX) PV volume-id
334453
pv, err := flavor.getPVByNFSLabel(nfsParentVolumeIDKey, volumeID)
@@ -907,6 +1026,44 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp
9071026
LabelSelector: &podLabelSelector,
9081027
}
9091028

1029+
startupProbe := &core_v1.Probe{
1030+
ProbeHandler: core_v1.ProbeHandler{
1031+
Exec: &core_v1.ExecAction{
1032+
Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeStartupKey},
1033+
},
1034+
},
1035+
InitialDelaySeconds: nfsProbeInitialDelaySeconds,
1036+
PeriodSeconds: nfsProbePeriodSeconds,
1037+
TimeoutSeconds: nfsProbeTimeoutSeconds,
1038+
}
1039+
1040+
readinessProbe := &core_v1.Probe{
1041+
ProbeHandler: core_v1.ProbeHandler{
1042+
Exec: &core_v1.ExecAction{
1043+
Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeReadinessKey},
1044+
},
1045+
},
1046+
InitialDelaySeconds: nfsProbeInitialDelaySeconds,
1047+
PeriodSeconds: nfsProbePeriodSeconds,
1048+
TimeoutSeconds: nfsProbeTimeoutSeconds,
1049+
}
1050+
1051+
livenessProbe := &core_v1.Probe{
1052+
ProbeHandler: core_v1.ProbeHandler{
1053+
Exec: &core_v1.ExecAction{
1054+
Command: []string{"/bin/sh", "/nfsHealthCheck.sh", nfsProbeLivenessKey},
1055+
},
1056+
},
1057+
InitialDelaySeconds: nfsProbeInitialDelaySeconds,
1058+
PeriodSeconds: nfsLivenessProbePeriodSeconds,
1059+
TimeoutSeconds: nfsLivenessProbeTimeoutSeconds,
1060+
}
1061+
1062+
containers := []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)}
1063+
containers[0].StartupProbe = startupProbe
1064+
containers[0].ReadinessProbe = readinessProbe
1065+
containers[0].LivenessProbe = livenessProbe
1066+
9101067
podSpec := core_v1.PodTemplateSpec{
9111068
ObjectMeta: meta_v1.ObjectMeta{
9121069
Name: name,
@@ -915,7 +1072,7 @@ func (flavor *Flavor) makeNFSDeployment(name string, nfsSpec *NFSSpec, nfsNamesp
9151072
},
9161073
Spec: core_v1.PodSpec{
9171074
ServiceAccountName: nfsServiceAccount,
918-
Containers: []core_v1.Container{flavor.makeContainer("hpe-nfs", nfsSpec)},
1075+
Containers: containers,
9191076
RestartPolicy: core_v1.RestartPolicyAlways,
9201077
Volumes: volumes,
9211078
HostIPC: false,

0 commit comments

Comments
 (0)