feat(marketplace): pre-delete resource check and post-delete complete uninstall

.github/workflows/marketplace-smoke-test.yaml

@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: ["main"]
 
+env:
+  # This version should match the (forked) version of the spin-operator sub-chart
+  SPIN_OPERATOR_VERSION: v0.3.0
+
 jobs:
   helm-install-smoke-test:
     runs-on: ubuntu-22.04
@@ -31,13 +35,13 @@ jobs:
 
       - name: run spin app
         run: |
-          kubectl apply -f https://raw.githubusercontent.com/spinkube/spin-operator/main/config/samples/simple.yaml
+          kubectl apply -f https://raw.githubusercontent.com/spinkube/spin-operator/${{ env.SPIN_OPERATOR_VERSION }}/config/samples/simple.yaml
           kubectl rollout status deployment simple-spinapp --timeout 90s
           kubectl get pods -A
           kubectl port-forward svc/simple-spinapp 8083:80 &
           timeout 15s bash -c 'until curl -f -vvv http://localhost:8083/hello; do sleep 2; done'
 
-      - name: debug
+      - name: debug install
         if: failure()
         run: |
           kubectl get pods -A
@@ -51,3 +55,46 @@ jobs:
 
       - name: Verify curl
         run: curl localhost:8083/hello
+
+      - name: helm delete spinkube
+        run: |
+          if helm delete spinkube --timeout 1m --namespace spinkube; then
+            echo "A Spin App remains on the cluster; the helm release should not have uninstalled successfully."
+            exit 1
+          fi
+
+          kubectl logs -n spinkube -l job-name=spinkube-pre-delete | \
+            grep -q "There are 1 SpinApps still existing on the cluster; aborting helm release deletion."
+
+      - name: Ensure Spin app still reachable
+        run: curl localhost:8083/hello
+
+      - name: Ensure SpinKube resources still exist
+        run: |
+          kubectl get po -n spinkube -o name | grep -q spin-operator
+          kubectl get po -n spinkube -o name | grep -q kwasm-operator
+          kubectl get po -n spinkube -o name | grep -q cert-manager
+          kubectl get crd -o name | grep -q spinapp
+          kubectl get crd -o name | grep -q cert-manager
+
+      - name: Delete Spin App
+        run: kubectl delete spinapp simple-spinapp
+
+      - name: helm delete spinkube
+        run: helm delete spinkube --timeout 1m --namespace spinkube
+
+      - name: Verify all resources are deleted
+        run: |
+          timeout 15s bash -c 'until ! kubectl get crd -o name | grep -q cert-manager; do sleep 2; done'
+          timeout 15s bash -c 'until ! kubectl get crd -o name | grep -q spinapp; do sleep 2; done'
+          timeout 15s bash -c 'until ! kubectl get ns -o name | grep -q spinkube; do sleep 2; done'
+
+      - name: debug uninstall
+        if: failure()
+        run: |
+          kubectl get pods -A
+          kubectl get crd -A
+          kubectl get ns -A
+          kubectl describe pods -A
+          kubectl describe crd -A
+          kubectl describe ns -A

marketplace/charts/spinkube-azure-marketplace/templates/post-delete-job.yaml

@@ -0,0 +1,117 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ .Release.Name }}-post-delete"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}-post-delete
+      containers:
+      - name: post-delete-job
+        image: {{ printf "%s/%s:%s" .Values.global.azure.images.kubectl.registry .Values.global.azure.images.kubectl.image .Values.global.azure.images.kubectl.tag }}
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: RELEASE_NAMESPACE
+          value: {{ .Release.Namespace }}
+        command: ["/bin/sh", "-c"]
+        args:
+          - |-
+              # Delete spin-operator CRDs
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/spinapps.core.spinoperator.dev
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/spinappexecutors.core.spinoperator.dev
+
+              # Delete cert-manager CRDs
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io
+              kubectl delete customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io
+
+              # Delete the namespace
+              # Note: Helm won't delete the release namespace resource when deleting a release.
+              kubectl delete ns "${RELEASE_NAMESPACE}"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-post-delete
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "1"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-post-delete-role
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "1"
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - list
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: '{{ .Release.Name }}-post-delete-rolebinding'
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "2"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: '{{ .Release.Name }}-post-delete-role'
+subjects:
+- kind: ServiceAccount
+  name: '{{ .Release.Name }}-post-delete'
+  namespace: '{{ .Release.Namespace }}'

marketplace/charts/spinkube-azure-marketplace/templates/pre-delete-job.yaml

@@ -0,0 +1,114 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ .Release.Name }}-pre-delete"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}-pre-delete
+      containers:
+      - name: pre-delete-job
+        image: {{ printf "%s/%s:%s" .Values.global.azure.images.kubectl.registry .Values.global.azure.images.kubectl.image .Values.global.azure.images.kubectl.tag }}
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/bash", "-c"]
+        args:
+          - |-
+              # Prevent deletion if one or more SpinApps still exist on the cluster
+              spinapp_count="$(kubectl get spinapps.core.spinoperator.dev --all-namespaces -o name | wc -l)"
+              if [[ $spinapp_count -gt 0 ]]; then
+                echo "There are $spinapp_count SpinApps still existing on the cluster; aborting helm release deletion."
+                echo "Remove these resources before attempting to delete the helm release."
+                exit 1
+              fi
+
+              # Prevent deletion if one or more SpinAppExecutors still exist on the cluster
+              # (Not counting the containerd-shim-spin SpinAppExecutor installed as part of the post-install job)
+              spinappexecutor_count="$(kubectl get spinappexecutors.core.spinoperator.dev --all-namespaces -o name | grep -v containerd-shim-spin | wc -l)"
+              if [[ $spinappexecutor_count -gt 0 ]]; then
+                echo "There are $spinappexecutor_count SpinAppExecutors still existing on the cluster; aborting helm release deletion."
+                echo "Remove these resources before attempting to delete the helm release."
+                exit 1
+              fi
+
+              # Delete the containerd-shim-spin SpinAppExecutor installed as part of the post-install-job
+              kubectl delete --namespace default spinappexecutors.core.spinoperator.dev containerd-shim-spin
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-pre-delete
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "1"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-pre-delete-role
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "1"
+rules:
+- apiGroups:
+  - core.spinoperator.dev
+  resources:
+  - spinapps
+  - spinappexecutors
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: '{{ .Release.Name }}-pre-delete-rolebinding'
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "2"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: '{{ .Release.Name }}-pre-delete-role'
+subjects:
+- kind: ServiceAccount
+  name: '{{ .Release.Name }}-pre-delete'
+  namespace: '{{ .Release.Namespace }}'