Skip to content

Commit 43eaf98

Browse files
Rade333Rade333
authored
Merge pull request #70 from wunderio/feature/release-cleanup
Removal and cleanup deduplication
2 parents 9e160ce + ed66ab3 commit 43eaf98

File tree

4 files changed

+153
-92
lines changed

4 files changed

+153
-92
lines changed

‎cmd/ciReleaseCleanfailed.go‎

Lines changed: 77 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
package cmd
22

33
import (
4+
"context"
45
"fmt"
6+
"log"
7+
"os"
58

9+
helmclient "github.com/mittwald/go-helm-client"
610
"github.com/spf13/cobra"
11+
"github.com/wunderio/silta-cli/internal/common"
12+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/kubernetes"
14+
"k8s.io/client-go/tools/clientcmd"
715
)
816

917
var ciReleaseCleanfailedCmd = &cobra.Command{
@@ -13,40 +21,75 @@ var ciReleaseCleanfailedCmd = &cobra.Command{
1321
releaseName, _ := cmd.Flags().GetString("release-name")
1422
namespace, _ := cmd.Flags().GetString("namespace")
1523

16-
command := fmt.Sprintf(`
17-
NAMESPACE='%s'
18-
RELEASE_NAME='%s'
19-
20-
failed_revision=$(helm list -n "${NAMESPACE}" --failed --pending --filter="(\s|^)(${RELEASE_NAME})(\s|$)" | tail -1 | cut -f3)
21-
22-
if [[ "${failed_revision}" -eq 1 ]]; then
23-
# Remove any existing post-release hook, since it's technically not part of the release.
24-
kubectl delete job -n "${NAMESPACE}" "${RELEASE_NAME}-post-release" 2> /dev/null || true
25-
26-
echo "Removing failed first release."
27-
helm delete -n "${NAMESPACE}" "${RELEASE_NAME}"
28-
29-
echo "Delete persistent volume claims left over from statefulsets."
30-
kubectl delete pvc -n "${NAMESPACE}" -l release="${RELEASE_NAME}"
31-
kubectl delete pvc -n "${NAMESPACE}" -l app="${RELEASE_NAME}-es"
32-
33-
echo -n "Waiting for volumes to be deleted."
34-
until [[ -z $(kubectl get pv | grep "${NAMESPACE}/${RELEASE_NAME}-") ]]
35-
do
36-
echo -n "."
37-
sleep 5
38-
done
39-
fi
40-
41-
# Workaround for previous Helm release stuck in pending state
42-
pending_release=$(helm list -n "${NAMESPACE}" --pending --filter="(\s|^)(${RELEASE_NAME})(\s|$)"| tail -1 | cut -f1)
43-
44-
if [[ "${pending_release}" == "${RELEASE_NAME}" ]]; then
45-
secret_to_delete=$(kubectl get secret -l owner=helm,status=pending-upgrade,name="${RELEASE_NAME}" -n "${NAMESPACE}" | awk '{print $1}' | grep -v NAME)
46-
kubectl delete secret -n "${NAMESPACE}" "${secret_to_delete}"
47-
fi
48-
`, namespace, releaseName)
49-
pipedExec(command, "", "ERROR: ", debug)
24+
// ----
25+
26+
homeDir, err := os.UserHomeDir()
27+
if err != nil {
28+
log.Fatalf("cannot read user home dir")
29+
}
30+
kubeConfigPath := homeDir + "/.kube/config"
31+
32+
kubeConfig, err := os.ReadFile(kubeConfigPath)
33+
if err != nil {
34+
log.Fatalf("cannot read kubeConfig from path")
35+
}
36+
37+
//k8s go client init logic
38+
config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
39+
if err != nil {
40+
log.Fatalf("cannot read kubeConfig from path: %s", err)
41+
}
42+
clientset, err := kubernetes.NewForConfig(config)
43+
if err != nil {
44+
log.Fatalf("cannot initialize k8s client: %s", err)
45+
}
46+
47+
//Helm client init logic
48+
opt := &helmclient.KubeConfClientOptions{
49+
Options: &helmclient.Options{
50+
Namespace: namespace,
51+
RepositoryCache: "/tmp/.helmcache",
52+
RepositoryConfig: "/tmp/.helmrepo",
53+
Debug: false,
54+
Linting: false, // Change this to false if you don't want linting.
55+
},
56+
KubeContext: "",
57+
KubeConfig: kubeConfig,
58+
}
59+
60+
helmClient, err := helmclient.NewClientFromKubeConf(opt)
61+
if err != nil {
62+
log.Fatalf("Cannot create client from kubeConfig")
63+
}
64+
65+
// Get release info
66+
release, err := helmClient.GetRelease(releaseName)
67+
if err != nil {
68+
return // Release not found or there was an error
69+
}
70+
71+
// Check if there's only one revision and it's failed
72+
if release.Version == 1 && release.Info.Status == "failed" {
73+
74+
fmt.Println("Removing failed first release.")
75+
76+
// Remove release
77+
common.UninstallHelmRelease(clientset, helmClient, releaseName, namespace, true)
78+
}
79+
80+
// Workaround for previous Helm release stuck in pending state
81+
// This is a workaround for a known issue with Helm where a release can get stuck in a pending-upgrade state
82+
// and the secret is not deleted. This is a workaround to delete the secret if it exists.
83+
if release.Info.Status == "pending-upgrade" {
84+
secretName := fmt.Sprintf("%s.%s.v%d", releaseName, namespace, release.Version)
85+
if err == nil {
86+
fmt.Printf("Deleting secret %s\n", secretName)
87+
err := clientset.CoreV1().Secrets(namespace).Delete(context.TODO(), secretName, v1.DeleteOptions{})
88+
if err != nil {
89+
log.Fatalf("Error deleting secret %s: %s", secretName, err)
90+
}
91+
}
92+
}
5093
},
5194
}
5295

‎cmd/ciReleaseDelete.go‎

Lines changed: 4 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
package cmd
22

33
import (
4-
"context"
54
"log"
65
"os"
76

87
helmclient "github.com/mittwald/go-helm-client"
98
"github.com/spf13/cobra"
10-
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"github.com/wunderio/silta-cli/internal/common"
1110
"k8s.io/client-go/kubernetes"
1211
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // gcp auth provider
1312
"k8s.io/client-go/tools/clientcmd"
@@ -60,61 +59,9 @@ var ciReleaseDeleteCmd = &cobra.Command{
6059
log.Fatalf("Cannot create client from kubeConfig")
6160
}
6261

63-
//Uninstall Helm release
64-
uninstallErr := helmClient.UninstallReleaseByName(releaseName)
65-
if uninstallErr != nil {
66-
log.Fatalf("Error removing a release:%s", uninstallErr)
67-
}
68-
69-
//Delete pre-release jobs
70-
selectorLabels := []string{
71-
"release",
72-
"app.kubernetes.io/instance",
73-
}
74-
75-
for _, l := range selectorLabels {
76-
selector := l + "=" + releaseName
77-
list, err := clientset.BatchV1().Jobs(namespace).List(context.TODO(), v1.ListOptions{
78-
LabelSelector: selector,
79-
})
80-
if err != nil {
81-
log.Fatalf("Error getting the list of jobs: %s", err)
82-
}
83-
for _, v := range list.Items {
84-
log.Printf("Deleting job: %s", v.Name)
85-
propagationPolicy := v1.DeletePropagationBackground
86-
clientset.BatchV1().Jobs(namespace).Delete(context.TODO(), v.Name, v1.DeleteOptions{PropagationPolicy: &propagationPolicy})
87-
}
88-
}
89-
90-
//Delete PVCs
91-
if deletePVCs {
92-
93-
PVC_client := clientset.CoreV1().PersistentVolumeClaims(namespace)
94-
95-
selectorLabels := []string{
96-
"app",
97-
"release",
98-
"app.kubernetes.io/instance",
99-
}
100-
101-
for _, l := range selectorLabels {
102-
selector := l + "=" + releaseName
103-
if l == "app" {
104-
selector = l + "=" + releaseName + "-es"
105-
}
106-
list, err := PVC_client.List(context.TODO(), v1.ListOptions{
107-
LabelSelector: selector,
108-
})
109-
if err != nil {
110-
log.Fatalf("Error getting the list of PVCs: %s", err)
111-
}
112-
113-
for _, v := range list.Items {
114-
log.Printf("Deleting PVC: %s", v.Name)
115-
PVC_client.Delete(context.TODO(), v.Name, v1.DeleteOptions{})
116-
}
117-
}
62+
err = common.UninstallHelmRelease(clientset, helmClient, releaseName, namespace, deletePVCs)
63+
if err != nil {
64+
log.Fatalf("Error removing a release: %s", err)
11865
}
11966

12067
},

‎cmd/ciReleaseDeploy.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ var ciReleaseDeployCmd = &cobra.Command{
168168
}
169169

170170
// Create namespace if it doesn't exist
171+
// Describe namespace
171172
_, err = clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, v1meta.GetOptions{})
172173
if err != nil {
173174
_, err = clientset.CoreV1().Namespaces().Create(context.TODO(), &v1core.Namespace{
@@ -176,7 +177,7 @@ var ciReleaseDeployCmd = &cobra.Command{
176177
},
177178
}, v1meta.CreateOptions{})
178179
if err != nil {
179-
log.Fatalf("cannot create namespace: %s", err)
180+
log.Printf("cannot create namespace: %s\n", err)
180181
}
181182
}
182183
}

‎internal/common/ciReleaseFunctions.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package common
2+
3+
import (
4+
"context"
5+
"log"
6+
7+
helmclient "github.com/mittwald/go-helm-client"
8+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/client-go/kubernetes"
10+
)
11+
12+
// UninstallRelease removes a Helm release and related resources
13+
// Note: namespace is inferred from the helmclient.Options struct but set here for kubernetes clientset actions
14+
func UninstallHelmRelease(clientset *kubernetes.Clientset, helmClient helmclient.Client, releaseName string, namespace string, deletePVCs bool) error {
15+
16+
// Uninstall helm release. Namespace and other context is provided via the
17+
// helmclient.Options struct when instantiating a client.
18+
// Do not bail when release removal fails, remove related resources anyway.
19+
err := helmClient.UninstallReleaseByName(releaseName)
20+
if err != nil {
21+
log.Printf("Failed to remove helm release: %s", err)
22+
}
23+
24+
// Delete related jobs
25+
selectorLabels := []string{
26+
"release",
27+
"app.kubernetes.io/instance",
28+
}
29+
30+
for _, l := range selectorLabels {
31+
selector := l + "=" + releaseName
32+
list, _ := clientset.BatchV1().Jobs(namespace).List(context.TODO(), v1.ListOptions{
33+
LabelSelector: selector,
34+
})
35+
for _, v := range list.Items {
36+
log.Printf("Removing job: %s", v.Name)
37+
propagationPolicy := v1.DeletePropagationBackground
38+
clientset.BatchV1().Jobs(namespace).Delete(context.TODO(), v.Name, v1.DeleteOptions{PropagationPolicy: &propagationPolicy})
39+
}
40+
}
41+
42+
if deletePVCs {
43+
44+
// Find and remove related PVC's by release name label
45+
PVC_client := clientset.CoreV1().PersistentVolumeClaims(namespace)
46+
47+
selectorLabels = []string{
48+
"app",
49+
"release",
50+
"app.kubernetes.io/instance",
51+
}
52+
53+
for _, l := range selectorLabels {
54+
selector := l + "=" + releaseName
55+
if l == "app" {
56+
selector = l + "=" + releaseName + "-es"
57+
}
58+
list, _ := PVC_client.List(context.TODO(), v1.ListOptions{
59+
LabelSelector: selector,
60+
})
61+
62+
for _, v := range list.Items {
63+
log.Printf("Removing PVC: %s", v.Name)
64+
PVC_client.Delete(context.TODO(), v.Name, v1.DeleteOptions{})
65+
}
66+
}
67+
}
68+
69+
return nil
70+
}

0 commit comments

Comments
 (0)