feat: Automating legacy konnector agent removal (#1384)

**What problem does this PR solve?**:
This PR adds functionality to automatically detect and uninstall legacy
Helm releases (chart name: `nutanix-k8s-agent`) during cluster upgrades.
This ensures a clean transition from the legacy Helm-based installation
to the new HelmChartProxy-based approach(addon), preventing conflicts
and orphaned resources.


When upgrading clusters that were previously deployed with the legacy
`nutanix-k8s-agent` Helm chart, the old Helm release remains in the
cluster alongside the new HelmChartProxy-managed installation. This can
lead to:

- **Resource Conflicts**: Both the legacy release and new HelmChartProxy
may try to manage the same resources
- **Orphaned Resources**: If the legacy release is manually deleted or
corrupted, its resources remain orphaned
- **Upgrade Failures**: Conflicts during upgrade can cause the upgrade
process to fail
- **Manual Intervention Required**: Currently, operators must manually
identify and uninstall legacy releases before upgrades

Previously, the `BeforeClusterUpgrade` hook had no logic to handle
legacy Helm releases, requiring manual cleanup before upgrades.

**Solution**:
This PR implements automatic legacy Helm release detection and
uninstallation during the `BeforeClusterUpgrade` lifecycle hook:

1. **Automatic Detection**: Scans for Helm release secrets containing
the legacy chart name `nutanix-k8s-agent`
2. **Proper Uninstallation**: Uses Helm's Go client
(`helm.sh/helm/v3/pkg/action`) to properly uninstall releases, ensuring
all associated Kubernetes resources are removed
3. **Best-Effort Operation**: Legacy cleanup errors are logged but don't
block the upgrade process
5. **Edge Case Handling**: Handles multiple legacy releases that may
exist from failed upgrades or manual installations

**Which issue(s) this PR fixes**:
Fixes #
[NCN-110886](https://jira.nutanix.com/browse/NCN-110886)

**How Has This Been Tested?**:
Deployed local kind clusters and tested both new cluster creation and
upgrades
Test logs:
Helm list - 2.16 cluster:
<img width="1357" height="214" alt="Beforeupgrade_helmlist"
src="https://github.com/user-attachments/assets/f710b213-32eb-48f1-a77f-4c81bb7fc636"
/>



After upgrade:
Helm list - 2.17-dev
<img width="1339" height="206" alt="Afterupgrade-helmlist"
src="https://github.com/user-attachments/assets/cf101145-f9a1-4847-939f-1e4d1f3cbcad"
/>


Upgrade logs:
`
I1110 03:56:32.057732 1 handler.go:664] "Uninstalling legacy helm
release" cluster="default/nkp-vijay-test-cluster-36"
releaseName="k8sagent" namespace="ntnx-system"
chartName="nutanix-k8s-agent"
I1110 03:56:32.301845 1 helmaddon.go:147] "Retrieving installation
values template for cluster" cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:33.016166 1 handler.go:740] "uninstall: Deleting k8sagent"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:33.016192 1 handler.go:740] "delete hooks disabled for
k8sagent" cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:34.630251 1 handler.go:740] "uninstall: given cascade value:
, defaulting to delete propagation background"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:34.630517 1 handler.go:740] "Starting delete for
\"nutanix-agent\" Deployment"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:34.886818 1 handler.go:740] "Starting delete for
\"nutanix-agent\" ClusterRoleBinding"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:35.145205 1 handler.go:740] "Starting delete for
\"nutanix-agent\" ClusterRole"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:35.402476 1 handler.go:740] "Starting delete for
\"ntnx-cluster-configmap\" ConfigMap"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:35.402706 1 handler.go:740] "Starting delete for
\"nutanix-agent-config\" ConfigMap"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:35.659980 1 handler.go:740] "Starting delete for
\"nutanix-agent\" ServiceAccount"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:35.917091 1 handler.go:740] "purge requested for k8sagent"
cluster="default/nkp-vijay-test-cluster-36"
I1110 03:56:36.586018 1 handler.go:683] "Successfully uninstalled legacy
helm release" cluster="default/nkp-vijay-test-cluster-36"
releaseName="k8sagent" namespace="ntnx-system"
I1110 03:56:37.623619 1 cm.go:88] "Fetching HelmChart info for
\"konnector-agent\" from configmap default/default-helm-addons-config"
cluster="default/nkp-vijay-test-cluster-36"`

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: vijayaraghavanr31

api/v1alpha1/constants.go

@@ -64,4 +64,8 @@ const (
 	// SkipCiliumKubeProxyReplacementValidation is the key of the annotation on the Cluster
 	// used to skip Cilium kube-proxy replacement validation.
 	SkipCiliumKubeProxyReplacementValidation = APIGroup + "/skip-cilium-kube-proxy-replacement-validation"
+
+	// SkipKonnectorAgentLegacyDeploymentValidation is the key of the annotation on the Cluster
+	// used to skip Konnector Agent legacy deployment validation.
+	SkipKonnectorAgentLegacyDeploymentValidation = APIGroup + "/skip-konnector-agent-legacy-deployment-validation"
 )

go.mod

@@ -33,10 +33,12 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
 	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.16.1
 	k8s.io/api v0.32.9
 	k8s.io/apiextensions-apiserver v0.32.9
 	k8s.io/apimachinery v0.32.9
 	k8s.io/apiserver v0.32.9
+	k8s.io/cli-runtime v0.31.0
 	k8s.io/client-go v0.32.9
 	k8s.io/component-base v0.32.9
 	k8s.io/klog/v2 v2.130.1
@@ -52,12 +54,16 @@ require (
 	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	cel.dev/expr v0.18.0 // indirect
 	dario.cat/mergo v1.0.1 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
+	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/PaesslerAG/gval v1.0.0 // indirect
 	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
@@ -71,20 +77,32 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/containerd/containerd v1.7.12 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v25.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker v28.0.2+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
@@ -98,6 +116,7 @@ require (
 	github.com/go-openapi/validate v0.24.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
@@ -108,38 +127,59 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/gosuri/uitable v0.0.4 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nutanix/ntnx-api-golang-clients/volumes-go-client/v4 v4.0.1-beta.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.2 // indirect
+	github.com/rubenv/sql-migrate v1.7.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -153,6 +193,10 @@ require (
 	github.com/ulikunitz/xz v0.5.15 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel v1.29.0 // indirect
@@ -162,6 +206,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.29.0 // indirect
 	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
@@ -188,8 +233,12 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/cluster-bootstrap v0.32.3 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/kubectl v0.31.0 // indirect
+	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/kind v0.27.0 // indirect
+	sigs.k8s.io/kustomize/api v0.17.2 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 )