Skip to content

Commit 1ab5ad6

Browse files
Madhu-1Madhu-1
authored
Merge pull request #269 from black-dragon74/csi-addons-podnet-ds
driver: deploy csi-addons sidecar in its own DaemonSet
2 parents 160929b + 49efcbb commit 1ab5ad6

File tree

2 files changed

+195
-74
lines changed

2 files changed

+195
-74
lines changed

‎.github/actions/deploy-charts/action.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ runs:
8383
shell: bash
8484
run: |
8585
DRIVER_NAME="${{ inputs.driverType }}.csi.ceph.com" yq eval '.drivers |= map(.enabled = (.name == strenv(DRIVER_NAME)))' -i ./deploy/charts/ceph-csi-drivers/values.yaml
86+
yq eval '.drivers |= map(.deployCsiAddons = (.name == "rbd.csi.ceph.com"))' -i ./deploy/charts/ceph-csi-drivers/values.yaml
8687
8788
- name: Install csi driver helm chart
8889
shell: bash
@@ -94,9 +95,15 @@ runs:
9495

9596
- name: Check csi pods are running
9697
shell: bash
98+
env:
99+
DRIVER_TYPE: ${{ inputs.driverType }}
97100
run: |
101+
podCount=2
102+
if [ "$DRIVER_TYPE" = "rbd" ]; then
103+
podCount=3
104+
fi
98105
for i in {1..180}; do
99-
if [ $(kubectl get pods -n csi-driver --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]; then
106+
if [ $(kubectl get pods -n csi-driver --field-selector=status.phase=Running --no-headers | wc -l) -eq $podCount ]; then
100107
echo "All CSI pods are running"
101108
kubectl get pods,deployment,daemonset,replicaset -n csi-driver
102109
exit 0

‎internal/controller/driver_controller.go‎

Lines changed: 187 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -229,16 +229,22 @@ func (r *driverReconcile) reconcile() error {
229229
return err
230230
}
231231

232-
// Concurrently reconcile different aspects of the clusters actual state to meet
233-
// the desired state defined on the driver object
234-
errChan := utils.RunConcurrently(
232+
reconcilers := []func() error{
235233
r.reconcileCsiConfigMap,
236234
r.reconcileLogRotateConfigMap,
237235
r.reconcileK8sCsiDriver,
238236
r.reconcileControllerPluginDeployment,
239237
r.reconcileNodePluginDeamonSet,
240238
r.reconcileLivenessService,
241-
)
239+
}
240+
241+
if r.isRdbDriver() {
242+
reconcilers = append(reconcilers, r.reconcileNodePluginDeamonSetForCsiAddons)
243+
}
244+
245+
// Concurrently reconcile different aspects of the clusters actual state to meet
246+
// the desired state defined on the driver object
247+
errChan := utils.RunConcurrently(reconcilers...)
242248

243249
// Check if any reconcilatin error where raised during the concurrent execution
244250
// of the reconciliation steps.
@@ -958,17 +964,177 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
958964
}
959965

960966
func (r *driverReconcile) controllerPluginCsiAddonsContainerPort() corev1.ContainerPort {
961-
962967
// the cephFS and rbd drivers need to use different ports
963968
// to avoid port collisions with host network.
964969
port := utils.ControllerPluginCsiAddonsContainerRbdPort
965970
if r.isCephFsDriver() {
966971
port = utils.ControllerPluginCsiAddonsContainerCephFsPort
967-
968972
}
969973

970974
return port
975+
}
976+
977+
func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
978+
daemonSet := &appsv1.DaemonSet{}
979+
daemonSet.Name = r.generateName("nodeplugin-csi-addons")
980+
daemonSet.Namespace = r.driver.Namespace
981+
982+
log := r.log.WithValues("csiAddonsDaemonSetName", daemonSet.Name)
983+
984+
if !ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
985+
if err := r.Delete(r.ctx, daemonSet); client.IgnoreNotFound(err) != nil {
986+
log.Error(err, "failed to delete csi addons daemonset")
987+
return err
988+
}
989+
return nil
990+
}
991+
992+
log.Info("Reconciling csi addons nodeplugin daemonset")
993+
994+
opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, daemonSet, func() error {
995+
if err := ctrlutil.SetControllerReference(&r.driver, daemonSet, r.Scheme); err != nil {
996+
log.Error(err, "Failed to set owner reference on csi addons nodeplugin daemonset")
997+
998+
return err
999+
}
1000+
1001+
appName := daemonSet.Name
1002+
pluginSpec := cmp.Or(r.driver.Spec.NodePlugin, &csiv1.NodePluginSpec{})
1003+
serviceAccountName := cmp.Or(
1004+
ptr.Deref(pluginSpec.ServiceAccountName, ""),
1005+
fmt.Sprintf("%s%s-nodeplugin-sa", serviceAccountPrefix, r.driverType),
1006+
)
1007+
imagePullPolicy := cmp.Or(pluginSpec.ImagePullPolicy, corev1.PullIfNotPresent)
1008+
logVerbosity := ptr.Deref(r.driver.Spec.Log, csiv1.LogSpec{}).Verbosity
1009+
kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath)
1010+
port := utils.NodePluginCsiAddonsContainerPort
1011+
1012+
logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1.LogSpec{}).Rotation
1013+
logRotationEnabled := logRotationSpec != nil
1014+
1015+
daemonSet.Spec = appsv1.DaemonSetSpec{
1016+
Selector: &metav1.LabelSelector{
1017+
MatchLabels: map[string]string{"app": appName},
1018+
},
1019+
UpdateStrategy: ptr.Deref(pluginSpec.UpdateStrategy, defaultDaemonSetUpdateStrategy),
1020+
Template: corev1.PodTemplateSpec{
1021+
ObjectMeta: metav1.ObjectMeta{
1022+
Labels: utils.Call(func() map[string]string {
1023+
podLabels := map[string]string{}
1024+
maps.Copy(podLabels, pluginSpec.Labels)
1025+
podLabels["app"] = appName
1026+
return podLabels
1027+
}),
1028+
Annotations: maps.Clone(pluginSpec.Annotations),
1029+
},
1030+
Spec: corev1.PodSpec{
1031+
ServiceAccountName: serviceAccountName,
1032+
PriorityClassName: ptr.Deref(pluginSpec.PrioritylClassName, ""),
1033+
// to use e.g. Rook orchestrated cluster, and mons' FQDN is
1034+
// resolved through k8s service, set dns policy to cluster first
1035+
DNSPolicy: corev1.DNSClusterFirstWithHostNet,
1036+
Tolerations: pluginSpec.Tolerations,
1037+
Containers: utils.Call(func() []corev1.Container {
1038+
containers := []corev1.Container{
1039+
{
1040+
Name: "csi-addons",
1041+
Image: r.images["addons"],
1042+
ImagePullPolicy: imagePullPolicy,
1043+
// We need this in order for this container to be able to access
1044+
// the sockets created by the privileged nodeplugin container
1045+
// on systems with enforcing selinux.
1046+
SecurityContext: &corev1.SecurityContext{
1047+
Privileged: ptr.To(true),
1048+
Capabilities: &corev1.Capabilities{
1049+
Drop: []corev1.Capability{"All"},
1050+
},
1051+
},
1052+
Args: utils.DeleteZeroValues(
1053+
[]string{
1054+
utils.CsiAddonsNodeIdContainerArg,
1055+
utils.LogVerbosityContainerArg(logVerbosity),
1056+
utils.CsiAddonsAddressContainerArg,
1057+
utils.ContainerPortArg(port),
1058+
utils.PodContainerArg,
1059+
utils.NamespaceContainerArg,
1060+
utils.PodUidContainerArg,
1061+
utils.StagingPathContainerArg(kubeletDirPath),
1062+
utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
1063+
utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
1064+
utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
1065+
},
1066+
),
1067+
Ports: []corev1.ContainerPort{
1068+
port,
1069+
},
1070+
Env: []corev1.EnvVar{
1071+
utils.NodeIdEnvVar,
1072+
utils.PodNameEnvVar,
1073+
utils.PodNamespaceEnvVar,
1074+
utils.PodUidEnvVar,
1075+
},
1076+
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
1077+
mounts := []corev1.VolumeMount{
1078+
utils.PluginDirVolumeMount,
1079+
}
1080+
if logRotationEnabled {
1081+
mounts = append(mounts, utils.LogsDirVolumeMount)
1082+
}
1083+
return mounts
1084+
}),
1085+
Resources: ptr.Deref(
1086+
pluginSpec.Resources.Addons,
1087+
corev1.ResourceRequirements{},
1088+
),
1089+
},
1090+
}
1091+
// CSI LogRotate Container
1092+
if logRotationEnabled {
1093+
resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})
1094+
containers = append(containers, corev1.Container{
1095+
Name: "log-rotator",
1096+
Image: r.images["plugin"],
1097+
ImagePullPolicy: imagePullPolicy,
1098+
Resources: resources,
1099+
SecurityContext: &corev1.SecurityContext{
1100+
Privileged: ptr.To(true),
1101+
Capabilities: &corev1.Capabilities{
1102+
Drop: []corev1.Capability{"All"},
1103+
},
1104+
},
1105+
Command: []string{"/bin/bash", "-c", logRotateCmd},
1106+
VolumeMounts: []corev1.VolumeMount{
1107+
utils.LogsDirVolumeMount,
1108+
utils.LogRotateDirVolumeMount,
1109+
},
1110+
})
1111+
}
1112+
return containers
1113+
}),
1114+
Volumes: utils.Call(func() []corev1.Volume {
1115+
volumes := []corev1.Volume{
1116+
utils.PluginDirVolume(kubeletDirPath, r.driver.Name),
1117+
}
1118+
1119+
if logRotationEnabled {
1120+
logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath)
1121+
volumes = append(
1122+
volumes,
1123+
utils.LogsDirVolume(logHostPath, daemonSet.Name),
1124+
utils.LogRotateDirVolumeName(r.driver.Name),
1125+
)
1126+
}
1127+
return volumes
1128+
}),
1129+
},
1130+
},
1131+
}
1132+
1133+
return nil
1134+
})
9711135

1136+
logCreateOrUpdateResult(log, "csi addons node plugin daemonset", daemonSet, opResult, err)
1137+
return err
9721138
}
9731139

9741140
func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
@@ -1157,12 +1323,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
11571323
),
11581324
},
11591325
}
1160-
// CSI Addons Sidecar Container
1161-
if r.isRdbDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
1162-
port := utils.NodePluginCsiAddonsContainerPort
1326+
// Liveness Sidecar Container
1327+
if r.driver.Spec.Liveness != nil {
11631328
containers = append(containers, corev1.Container{
1164-
Name: "csi-addons",
1165-
Image: r.images["addons"],
1329+
Name: "liveness-prometheus",
1330+
Image: r.images["plugin"],
11661331
ImagePullPolicy: imagePullPolicy,
11671332
SecurityContext: &corev1.SecurityContext{
11681333
Privileged: ptr.To(true),
@@ -1172,76 +1337,25 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
11721337
},
11731338
Args: utils.DeleteZeroValues(
11741339
[]string{
1175-
utils.CsiAddonsNodeIdContainerArg,
1176-
utils.LogVerbosityContainerArg(logVerbosity),
1177-
utils.CsiAddonsAddressContainerArg,
1178-
utils.ContainerPortArg(port),
1179-
utils.PodContainerArg,
1180-
utils.NamespaceContainerArg,
1181-
utils.PodUidContainerArg,
1182-
utils.StagingPathContainerArg(kubeletDirPath),
1183-
utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
1184-
utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
1185-
utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
1340+
utils.TypeContainerArg("liveness"),
1341+
utils.EndpointContainerArg,
1342+
utils.MetricsPortContainerArg(r.driver.Spec.Liveness.MetricsPort),
1343+
utils.MetricsPathContainerArg,
1344+
utils.PoolTimeContainerArg,
1345+
utils.TimeoutContainerArg(3),
11861346
},
11871347
),
1188-
Ports: []corev1.ContainerPort{
1189-
port,
1190-
},
11911348
Env: []corev1.EnvVar{
1192-
utils.NodeIdEnvVar,
1193-
utils.PodNameEnvVar,
1194-
utils.PodNamespaceEnvVar,
1195-
utils.PodUidEnvVar,
1349+
utils.PodIpEnvVar,
1350+
},
1351+
VolumeMounts: []corev1.VolumeMount{
1352+
utils.PluginDirVolumeMount,
11961353
},
1197-
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
1198-
mounts := []corev1.VolumeMount{
1199-
utils.PluginDirVolumeMount,
1200-
}
1201-
if logRotationEnabled {
1202-
mounts = append(mounts, utils.LogsDirVolumeMount)
1203-
}
1204-
return mounts
1205-
}),
12061354
Resources: ptr.Deref(
1207-
pluginSpec.Resources.Addons,
1355+
pluginSpec.Resources.Liveness,
12081356
corev1.ResourceRequirements{},
12091357
),
12101358
})
1211-
// Liveness Sidecar Container
1212-
if r.driver.Spec.Liveness != nil {
1213-
containers = append(containers, corev1.Container{
1214-
Name: "liveness-prometheus",
1215-
Image: r.images["plugin"],
1216-
ImagePullPolicy: imagePullPolicy,
1217-
SecurityContext: &corev1.SecurityContext{
1218-
Privileged: ptr.To(true),
1219-
Capabilities: &corev1.Capabilities{
1220-
Drop: []corev1.Capability{"All"},
1221-
},
1222-
},
1223-
Args: utils.DeleteZeroValues(
1224-
[]string{
1225-
utils.TypeContainerArg("liveness"),
1226-
utils.EndpointContainerArg,
1227-
utils.MetricsPortContainerArg(r.driver.Spec.Liveness.MetricsPort),
1228-
utils.MetricsPathContainerArg,
1229-
utils.PoolTimeContainerArg,
1230-
utils.TimeoutContainerArg(3),
1231-
},
1232-
),
1233-
Env: []corev1.EnvVar{
1234-
utils.PodIpEnvVar,
1235-
},
1236-
VolumeMounts: []corev1.VolumeMount{
1237-
utils.PluginDirVolumeMount,
1238-
},
1239-
Resources: ptr.Deref(
1240-
pluginSpec.Resources.Liveness,
1241-
corev1.ResourceRequirements{},
1242-
),
1243-
})
1244-
}
12451359
}
12461360
// CSI LogRotate Container
12471361
if logRotationEnabled {

0 commit comments

Comments
 (0)