Skip to content

Commit dd981b0

Browse files
committed
delete pvs whose reclaim policy is delete when cluster is destroyed
1 parent 80f551c commit dd981b0

File tree

4 files changed

+206
-0
lines changed

4 files changed

+206
-0
lines changed

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ public class KubernetesClusterActionWorker {
233233
protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
234234
protected final String deployProviderScriptFilename = "deploy-provider";
235235
protected final String deployCsiDriverScriptFilename = "deploy-csi-driver";
236+
protected final String deletePvScriptFilename = "delete-pv-reclaimpolicy-delete";
236237
protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
237238
protected final String validateNodeScript = "validate-cks-node";
238239
protected final String removeNodeFromClusterScript = "remove-node-from-cluster";
@@ -241,6 +242,7 @@ public class KubernetesClusterActionWorker {
241242
protected File deployProviderScriptFile;
242243
protected File deployCsiDriverScriptFile;
243244
protected File autoscaleScriptFile;
245+
protected File deletePvScriptFile;
244246
protected KubernetesClusterManagerImpl manager;
245247
protected String[] keys;
246248

@@ -718,13 +720,15 @@ protected void retrieveScriptFiles() {
718720
deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
719721
deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename);
720722
deployCsiDriverScriptFile = retrieveScriptFile(deployCsiDriverScriptFilename);
723+
deletePvScriptFile = retrieveScriptFile(deletePvScriptFilename);
721724
autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
722725
}
723726

724727
protected void copyScripts(String nodeAddress, final int sshPort) {
725728
copyScriptFile(nodeAddress, sshPort, deploySecretsScriptFile, deploySecretsScriptFilename);
726729
copyScriptFile(nodeAddress, sshPort, deployProviderScriptFile, deployProviderScriptFilename);
727730
copyScriptFile(nodeAddress, sshPort, deployCsiDriverScriptFile, deployCsiDriverScriptFilename);
731+
copyScriptFile(nodeAddress, sshPort, deletePvScriptFile, deletePvScriptFilename);
728732
copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename);
729733
}
730734

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ private boolean destroyClusterVMs() {
106106
ApiCommandResourceType.VirtualMachine);
107107
vmContext.setEventResourceId(vmID);
108108
try {
109+
if (clusterVM.isControlNode() && kubernetesCluster.isCsiEnabled()) {
110+
deletePVsWithReclaimPolicyDelete();
111+
}
109112
UserVm vm = userVmService.destroyVm(vmID, true);
110113
if (!userVmManager.expunge(userVM)) {
111114
logger.warn("Unable to expunge VM {}, destroying Kubernetes cluster will probably fail", vm);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,4 +944,47 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) {
944944
protected List<DedicatedResourceVO> listDedicatedHostsInDomain(Long domainId) {
945945
return dedicatedResourceDao.listByDomainId(domainId);
946946
}
947+
948+
public boolean deletePVsWithReclaimPolicyDelete() {
949+
File pkFile = getManagementServerSshPublicKeyFile();
950+
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
951+
publicIpAddress = publicIpSshPort.first();
952+
sshPort = publicIpSshPort.second();
953+
try {
954+
String command = String.format("sudo %s/%s", scriptPath, deletePvScriptFilename);
955+
logMessage(Level.INFO, "Starting PV deletion script for cluster: " + kubernetesCluster.getName(), null);
956+
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
957+
pkFile, null, command, 10000, 10000, 600000); // 10 minute timeout
958+
if (Boolean.FALSE.equals(result.first())) {
959+
logMessage(Level.INFO, "PV delete script missing. Adding it now", null);
960+
retrieveScriptFiles();
961+
if (deletePvScriptFile != null) {
962+
copyScriptFile(publicIpAddress, sshPort, deletePvScriptFile, deletePvScriptFilename);
963+
logMessage(Level.INFO, "Executing PV deletion script (this may take several minutes)...", null);
964+
result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
965+
pkFile, null, command, 10000, 10000, 600000); // 10 minute timeout
966+
if (Boolean.FALSE.equals(result.first())) {
967+
logMessage(Level.ERROR, "PV deletion script failed: " + result.second(), null);
968+
throw new CloudRuntimeException(result.second());
969+
}
970+
logMessage(Level.INFO, "PV deletion script completed successfully", null);
971+
} else {
972+
logMessage(Level.WARN, "PV delete script file not found in resources, skipping PV deletion", null);
973+
return false;
974+
}
975+
} else {
976+
logMessage(Level.INFO, "PV deletion script completed successfully", null);
977+
}
978+
979+
if (result.second() != null && !result.second().trim().isEmpty()) {
980+
logMessage(Level.INFO, "PV deletion script output: " + result.second(), null);
981+
}
982+
983+
return true;
984+
} catch (Exception e) {
985+
String msg = String.format("Failed to delete PVs with reclaimPolicy=Delete: %s : %s", kubernetesCluster.getName(), e.getMessage());
986+
logMessage(Level.WARN, msg, e);
987+
return false;
988+
}
989+
}
947990
}
Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
#!/bin/bash -e
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
set -e
20+
21+
timestamp() {
22+
date '+%Y-%m-%d %H:%M:%S'
23+
}
24+
25+
echo "$(timestamp) - Starting PV deletion script with reclaimPolicy=Delete"
26+
27+
delete_workloads_using_pvc() {
28+
local namespace=$1
29+
local pvc_name=$2
30+
31+
echo "$(timestamp) - Finding workloads using PVC $pvc_name in namespace $namespace..."
32+
33+
local deleted_count=0
34+
35+
# Find & delete any deployment using the PVC
36+
/opt/bin/kubectl get deployments -n "$namespace" -o json 2>/dev/null | grep -l "$pvc_name" | \
37+
while IFS= read -r deployment; do
38+
if [ -n "$deployment" ]; then
39+
deployment_name=$(echo "$deployment" | cut -d'/' -f2)
40+
echo "$(timestamp) - Deleting Deployment: $deployment_name"
41+
/opt/bin/kubectl delete deployment "$deployment_name" -n "$namespace" --ignore-not-found=true
42+
deleted_count=$((deleted_count + 1))
43+
fi
44+
done
45+
46+
# Find and delete any StatefulSet using the PVC
47+
/opt/bin/kubectl get statefulsets -n "$namespace" -o json 2>/dev/null | grep -l "$pvc_name" | \
48+
while IFS= read -r sts; do
49+
if [ -n "$sts" ]; then
50+
sts_name=$(echo "$sts" | cut -d'/' -f2)
51+
echo "$(timestamp) - Deleting StatefulSet: $sts_name"
52+
/opt/bin/kubectl delete statefulset "$sts_name" -n "$namespace" --ignore-not-found=true
53+
deleted_count=$((deleted_count + 1))
54+
fi
55+
done
56+
57+
# Check standalone ReplicaSets (not owned by Deployments)
58+
/opt/bin/kubectl get replicasets -n "$namespace" --no-headers -o custom-columns=NAME:.metadata.name | \
59+
while read rs_name; do
60+
if [ -n "$rs_name" ]; then
61+
rs_volumes=$(/opt/bin/kubectl get replicaset "$rs_name" -n "$namespace" -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' 2>/dev/null || echo "")
62+
if echo "$rs_volumes" | grep -q "$pvc_name"; then
63+
owner_kind=$(/opt/bin/kubectl get replicaset "$rs_name" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].kind}' 2>/dev/null || echo "")
64+
if [ "$owner_kind" != "Deployment" ]; then
65+
echo "$(timestamp) - Deleting standalone ReplicaSet: $rs_name"
66+
/opt/bin/kubectl delete replicaset "$rs_name" -n "$namespace" --ignore-not-found=true
67+
deleted_count=$((deleted_count + 1))
68+
fi
69+
fi
70+
fi
71+
done
72+
73+
# Find and delete any DaemonSet using the PVC
74+
/opt/bin/kubectl get daemonsets -n "$namespace" -o json 2>/dev/null | grep -l "$pvc_name" | \
75+
while IFS= read -r ds; do
76+
if [ -n "$ds" ]; then
77+
ds_name=$(echo "$ds" | cut -d'/' -f2)
78+
echo "$(timestamp) - Deleting DaemonSet: $ds_name"
79+
/opt/bin/kubectl delete daemonset "$ds_name" -n "$namespace" --ignore-not-found=true
80+
deleted_count=$((deleted_count + 1))
81+
fi
82+
done
83+
84+
# Find and delete any Job using the PVC
85+
/opt/bin/kubectl get jobs -n "$namespace" -o json 2>/dev/null | grep -l "$pvc_name" | \
86+
while IFS= read -r job; do
87+
if [ -n "$job" ]; then
88+
job_name=$(echo "$job" | cut -d'/' -f2)
89+
echo "$(timestamp) - Deleting Job: $job_name"
90+
/opt/bin/kubectl delete job "$job_name" -n "$namespace" --ignore-not-found=true
91+
deleted_count=$((deleted_count + 1))
92+
fi
93+
done
94+
95+
# Find and delete any CronJobs using the PVC
96+
/opt/bin/kubectl get cronjobs -n "$namespace" -o json 2>/dev/null | grep -l "$pvc_name" | \
97+
while IFS= read -r cronjob; do
98+
if [ -n "$cronjob" ]; then
99+
cronjob_name=$(echo "$cronjob" | cut -d'/' -f2)
100+
echo "$(timestamp) - Deleting CronJob: $cronjob_name"
101+
/opt/bin/kubectl delete cronjob "$cronjob_name" -n "$namespace" --ignore-not-found=true
102+
deleted_count=$((deleted_count + 1))
103+
fi
104+
done
105+
106+
# Find and delete any standalone Pods using the PVC
107+
/opt/bin/kubectl get pods -n "$namespace" --no-headers -o custom-columns=NAME:.metadata.name | \
108+
while read pod_name; do
109+
if [ -n "$pod_name" ]; then
110+
pod_volumes=$(/opt/bin/kubectl get pod "$pod_name" -n "$namespace" -o jsonpath='{.spec.volumes[*].persistentVolumeClaim.claimName}' 2>/dev/null || echo "")
111+
if echo "$pod_volumes" | grep -q "$pvc_name"; then
112+
owner_kind=$(/opt/bin/kubectl get pod "$pod_name" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].kind}' 2>/dev/null || echo "")
113+
if [ -z "$owner_kind" ]; then
114+
echo "$(timestamp) - Deleting standalone Pod: $pod_name"
115+
/opt/bin/kubectl delete pod "$pod_name" -n "$namespace" --ignore-not-found=true
116+
deleted_count=$((deleted_count + 1))
117+
fi
118+
fi
119+
fi
120+
done
121+
122+
if [ $deleted_count -eq 0 ]; then
123+
echo "$(timestamp) - No workloads found using PVC $pvc_name"
124+
else
125+
echo "$(timestamp) - Deleted $deleted_count workload(s) using PVC $pvc_name"
126+
fi
127+
128+
echo "$(timestamp) - Waiting for pods to terminate..."
129+
sleep 5
130+
}
131+
132+
total_pvcs=0
133+
processed_pvcs=0
134+
135+
echo "$(timestamp) - Scanning for PVCs with associated PVs having reclaimPolicy=Delete..."
136+
137+
while read namespace pvc_name pv_name; do
138+
if [ -n "$pv_name" ] && [ "$pv_name" != "<none>" ]; then
139+
total_pvcs=$((total_pvcs + 1))
140+
reclaim_policy=$(/opt/bin/kubectl get pv "$pv_name" --no-headers -o custom-columns=RECLAIM:.spec.persistentVolumeReclaimPolicy 2>/dev/null || echo "")
141+
if [ "$reclaim_policy" = "Delete" ]; then
142+
processed_pvcs=$((processed_pvcs + 1))
143+
echo "$(timestamp) - Processing PVC $pvc_name in namespace $namespace (PV: $pv_name has reclaimPolicy=Delete)"
144+
145+
delete_workloads_using_pvc "$namespace" "$pvc_name"
146+
echo "$(timestamp) - Deleting PVC $pvc_name in namespace $namespace"
147+
/opt/bin/kubectl delete pvc "$pvc_name" -n "$namespace" --ignore-not-found=true
148+
149+
echo "$(timestamp) - Completed processing PVC $pvc_name"
150+
echo "---"
151+
fi
152+
fi
153+
done < <(/opt/bin/kubectl get pvc --all-namespaces --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,VOLUME:.spec.volumeName)
154+
155+
echo "$(timestamp) - Script completed successfully!"
156+
echo "$(timestamp) - Summary: Processed $processed_pvcs PVC(s) out of $total_pvcs total PVC(s) found"

0 commit comments

Comments
 (0)