Add support to setup csi driver on k8s cluster creation

Pearl1594 · Pearl1594 · commit cbdce2d4bfed · 2025-07-28T14:48:19.000-04:00
diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
@@ -166,4 +166,5 @@ enum State {
     Long getEtcdNodeCount();
     Long getCniConfigId();
     String getCniConfigDetails();
+    boolean isCsiEnabled();
 }
diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
@@ -196,6 +196,7 @@ public class ApiConstants {
     public static final String DURATION = "duration";
     public static final String ELIGIBLE = "eligible";
     public static final String EMAIL = "email";
+    public static final String ENABLE_CSI = "enablecsi";
     public static final String END_ASN = "endasn";
     public static final String END_DATE = "enddate";
     public static final String END_IP = "endip";
diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
@@ -203,3 +203,6 @@ SET `sort_key` = CASE
     ELSE `sort_key`
 END;
 -- End: Changes for Guest OS category cleanup
+
+-- Add csi_enabled column to kubernetes_cluster table - Move to 4.22
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes', 'csi_enabled', 'TINYINT(1) unsigned NOT NULL DEFAULT 0 COMMENT "true if kubernetes cluster is using csi, false otherwise" ');
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -1604,6 +1604,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) {
                 if (zone.isSecurityGroupEnabled()) {
                     newCluster.setSecurityGroupId(finalSecurityGroup.getId());
                 }
+                newCluster.setCsiEnabled(cmd.getEnableCsi());
                 kubernetesClusterDao.persist(newCluster);
                 addKubernetesClusterDetails(newCluster, defaultNetwork, cmd);
                 return newCluster;
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java
@@ -145,6 +145,9 @@ public class KubernetesClusterVO implements KubernetesCluster {
     @Column(name = "cni_config_details", updatable = true, length = 4096)
     private String cniConfigDetails;
 
+    @Column(name = "csi_enabled")
+    private boolean csiEnabled;
+
     @Override
     public long getId() {
         return id;
@@ -389,6 +392,14 @@ public void setClusterType(ClusterType clusterType) {
         this.clusterType = clusterType;
     }
 
+    public boolean isCsiEnabled() {
+        return csiEnabled;
+    }
+
+    public void setCsiEnabled(boolean csiEnabled) {
+        this.csiEnabled = csiEnabled;
+    }
+
     public KubernetesClusterVO() {
         this.uuid = UUID.randomUUID().toString();
     }
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
@@ -231,12 +231,14 @@ public class KubernetesClusterActionWorker {
 
     protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
     protected final String deployProviderScriptFilename = "deploy-provider";
+    protected final String deployCsiDriverScriptFilename = "deploy-csi-driver";
     protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
     protected final String validateNodeScript = "validate-cks-node";
     protected final String removeNodeFromClusterScript = "remove-node-from-cluster";
     protected final String scriptPath = "/opt/bin/";
     protected File deploySecretsScriptFile;
     protected File deployProviderScriptFile;
+    protected File deployCsiDriverScriptFile;
     protected File autoscaleScriptFile;
     protected KubernetesClusterManagerImpl manager;
     protected String[] keys;
@@ -713,12 +715,14 @@ protected File retrieveScriptFile(String filename) {
     protected void retrieveScriptFiles() {
         deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
         deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename);
+        deployCsiDriverScriptFile = retrieveScriptFile(deployCsiDriverScriptFilename);
         autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
     }
 
     protected void copyScripts(String nodeAddress, final int sshPort) {
         copyScriptFile(nodeAddress, sshPort, deploySecretsScriptFile, deploySecretsScriptFilename);
         copyScriptFile(nodeAddress, sshPort, deployProviderScriptFile, deployProviderScriptFilename);
+        copyScriptFile(nodeAddress, sshPort, deployCsiDriverScriptFile, deployCsiDriverScriptFilename);
         copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename);
     }
 
@@ -819,6 +823,43 @@ protected boolean deployProvider() {
         }
     }
 
+    protected boolean deployCsiDriver() {
+        File pkFile = getManagementServerSshPublicKeyFile();
+        Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
+        publicIpAddress = publicIpSshPort.first();
+        sshPort = publicIpSshPort.second();
+
+        try {
+            String command = String.format("sudo %s/%s", scriptPath, deployCsiDriverScriptFilename);
+            Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
+                    pkFile, null, command, 10000, 10000, 60000);
+
+            // Maybe the file isn't present. Try and copy it
+            if (!result.first()) {
+                logMessage(Level.INFO, "CSI files missing. Adding them now", null);
+                retrieveScriptFiles();
+                copyScripts(publicIpAddress, sshPort);
+
+                if (!createCloudStackSecret(keys)) {
+                    logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
+                            kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+                }
+
+                // If at first you don't succeed ...
+                result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
+                        pkFile, null, command, 10000, 10000, 60000);
+                if (!result.first()) {
+                    throw new CloudRuntimeException(result.second());
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage());
+            logAndThrow(Level.ERROR, msg);
+            return false;
+        }
+    }
+
     public void setKeys(String[] keys) {
         this.keys = keys;
     }
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
@@ -143,7 +143,7 @@ private boolean isKubernetesVersionSupportsHA() {
 
     private Pair<String, String> getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
                                                                 final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
-                                                                final boolean ejectIso, final boolean externalCni) throws IOException {
+                                                                final boolean ejectIso, final boolean externalCni, final boolean setupCsi) throws IOException {
         String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml");
         final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
         final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
@@ -161,6 +161,7 @@ private Pair<String, String> getKubernetesControlNodeConfig(final String control
         final String certSans = "{{ k8s_control.server_ips }}";
         final String k8sCertificate = "{{ k8s_control.certificate_key }}";
         final String externalCniPlugin = "{{ k8s.external.cni.plugin }}";
+        final String setupCsiDriver = "{{ k8s.setup.csi.driver }}";
 
         final List<String> addresses = new ArrayList<>();
         addresses.add(controlNodeIp);
@@ -212,6 +213,7 @@ private Pair<String, String> getKubernetesControlNodeConfig(final String control
         k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(setupCsiDriver, String.valueOf(setupCsi));
 
         k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
 
@@ -246,7 +248,7 @@ private Pair<UserVm,String> createKubernetesControlNode(final Network network, S
         Long userDataId = kubernetesCluster.getCniConfigId();
         Pair<String, String> k8sControlNodeConfigAndControlIp = new Pair<>(null, null);
         try {
-            k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId));
+            k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId), kubernetesCluster.isCsiEnabled());
         } catch (IOException e) {
             logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
         }
@@ -858,6 +860,9 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon
         }
         taintControlNodes();
         deployProvider();
+        if (kubernetesCluster.isCsiEnabled()) {
+            deployCsiDriver();
+        }
         updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList()));
         stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
         return true;
diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
@@ -207,6 +207,9 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
             since = "4.21.0")
     private Map cniConfigDetails;
 
+    @Parameter(name = ApiConstants.ENABLE_CSI, type = CommandType.BOOLEAN, description = "if true, setups up CloudStack CSI driver", since = "4.21.0")
+    private Boolean enableCsi;
+
     @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, description="the AS Number of the network")
     private Long asNumber;
 
@@ -371,6 +374,10 @@ public Long getCniConfigId() {
         return cniConfigId;
     }
 
+    public boolean getEnableCsi() {
+        return Objects.nonNull(enableCsi) ? enableCsi : Boolean.FALSE;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
@@ -179,6 +179,11 @@ write_files:
           mkdir -p /opt/provider
           cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
         fi
+        if [ -e "${BINARIES_DIR}/snapshot-crds.yaml" ]; then
+          mkdir -p /opt/csi
+          cp "${BINARIES_DIR}/snapshot-crds.yaml" /opt/csi/snapshot-crds.yaml
+          cp "${BINARIES_DIR}/manifest.yaml" /opt/csi/manifest.yaml
+        fi
 
         PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1`
         echo $PAUSE_IMAGE
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver
@@ -0,0 +1,50 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+(/opt/bin/kubectl get pods -A | grep cloud-controller-manager) && exit 0
+
+if [ -e /opt/csi/snapshot-crds.yaml ]; then
+    /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml
+    sleep 5
+    /opt/bin/kubectl apply -f /opt/csi/manifest.yaml
+    exit 0
+else
+    TARGET_DIR="/opt/csi"
+    mkdir -p "$TARGET_DIR"
+#    CSI_URLS=(
+#      "https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml"
+#      "https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml"
+#    )
+    CSI_URLS=(
+       "http://10.0.3.130/cks/csi/snapshot-crds.yaml -O ${working_dir}/snapshot-crds.yaml"
+       "http://10.0.3.130/cks/csi/manifest.yaml -O ${working_dir}/manifest.yaml"
+    )
+    for url in "${URLS[@]}"; do
+        filename=$(basename "$url")
+
+        curl -sSL ${url} -o ${filename}
+        if [ $? -ne 0 ]; then
+          echo "Unable to connect to the internet to download the relevant files to install and setup CloudStack CSI driver"
+          exit 1
+        else
+          /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml
+          /opt/bin/kubectl apply -f /opt/csi/manifest.yaml
+          exit 0
+        fi
+    done
+fi
diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh
@@ -156,6 +156,13 @@ if [ -n "${8}" ]; then
   wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz
 fi
 
+echo "Including CloudStack CSI Driver manifest"
+# TODO: remove the below 2 lines and uncomment the ones below once csi-driver is officially released, this is for testing purposes only
+wget http://10.0.3.130/cks/csi/snapshot-crds.yaml -O ${working_dir}/snapshot-crds.yaml
+wget http://10.0.3.130/cks/csi/manifest.yaml -O ${working_dir}/manifest.yaml
+#wget https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml -O ${working_dir}/snapshot-crds.yaml
+#wget https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml -O ${working_dir}/manifest.yaml
+
 mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}"
 
 rm -rf "${iso_dir}"
diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json
@@ -937,6 +937,7 @@
 "label.elastic": "Elastic",
 "label.email": "Email",
 "label.enable.autoscale.vmgroup": "Enable AutoScaling Group",
+"label.enable.csi": "Enable CloudStack CSI Driver",
 "label.enable.host": "Enable Host",
 "label.enable.network.offering": "Enable Network offering",
 "label.enable.oauth": "Enable OAuth Login",
diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue
@@ -207,6 +207,12 @@
           </template>
           <a-switch v-model:checked="form.advancedmode" />
         </a-form-item>
+        <a-form-item v-if="form.advancedmode" name="enablecsi" ref="enablecsi" :label="$t('label.enable.csi')">
+            <template #label>
+              <tooltip-label :title="$t('label.enable.csi')" :tooltip="apiParams.enablecsi.description"/>
+            </template>
+            <a-switch v-model:checked="form.enablecsi" />
+          </a-form-item>
         <a-form-item v-if="form.advancedmode" name="controlofferingid" ref="controlofferingid">
           <template #label>
             <tooltip-label :title="$t('label.cks.cluster.control.nodes.offeringid')" :tooltip="$t('label.cks.cluster.control.nodes.offeringid')"/>
@@ -902,6 +908,10 @@ export default {
           params.cniconfigurationid = values.cniconfigurationid
         }
 
+        if (values.enablecsi) {
+          params.enablecsi = values.enablecsi
+        }
+
         var idx = 0
         if (this.cniConfigValues) {
           for (const [key, value] of Object.entries(this.cniConfigValues)) {

Original file line number	Diff line number	Diff line change
`@@ -166,4 +166,5 @@ enum State {`
`166`	`166`	`Long getEtcdNodeCount();`
`167`	`167`	`Long getCniConfigId();`
`168`	`168`	`String getCniConfigDetails();`
	`169`	`+ boolean isCsiEnabled();`
`169`	`170`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1604,6 +1604,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) {`
`1604`	`1604`	`if (zone.isSecurityGroupEnabled()) {`
`1605`	`1605`	`newCluster.setSecurityGroupId(finalSecurityGroup.getId());`
`1606`	`1606`	`}`
	`1607`	`+ newCluster.setCsiEnabled(cmd.getEnableCsi());`
`1607`	`1608`	`kubernetesClusterDao.persist(newCluster);`
`1608`	`1609`	`addKubernetesClusterDetails(newCluster, defaultNetwork, cmd);`
`1609`	`1610`	`return newCluster;`