fix: adding predelete check in konnector agent delete webhook (#1438)

**What problem does this PR solve?**:
fix: adding predelete check in konnector agent delete webhook
**Which issue(s) this PR fixes**:
Fixes # https://jira.nutanix.com/browse/NCN-111450

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: vijayaraghavanr31

pkg/handlers/lifecycle/konnectoragent/handler.go

@@ -17,11 +17,15 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	prismgoclient "github.com/nutanix-cloud-native/prism-go-client"
+
 	capxv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
 	caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
@@ -31,6 +35,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/addons"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/config"
+	lifecycleutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/utils"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 	handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
 )
@@ -465,6 +470,20 @@ func (n *DefaultKonnectorAgent) BeforeClusterDelete(
 		return
 	}
 
+	// Check cluster is registered in PC
+	clusterRegistered, err := isClusterRegisteredInPC(ctx, n.client, cluster, log)
+	if err != nil {
+		log.Error(err, "Failed to check if cluster is registered in Prism Central, continuing with deletion anyway")
+		// setting response status to success to allow cluster deletion to proceed
+		resp.SetStatus(runtimehooksv1.ResponseStatusSuccess)
+		return
+	}
+	if !clusterRegistered {
+		log.Info("Cluster is not registered in Prism Central, skipping cleanup")
+		resp.SetStatus(runtimehooksv1.ResponseStatusSuccess)
+		return
+	}
+
 	// Check if cleanup is already in progress or completed
 	cleanupStatus, statusMsg, err := n.checkCleanupStatus(ctx, cluster, log)
 	if err != nil {
@@ -651,3 +670,112 @@ func (n *DefaultKonnectorAgent) checkCleanupStatus(
 	log.Info("HelmChartProxy exists, cleanup not started", "name", hcp.Name)
 	return cleanupStatusNotStarted, "HelmChartProxy exists and needs to be deleted", nil
 }
+
+// isClusterRegisteredInPC checks if the cluster is registered in Prism Central by calling
+// the Konnector GetClusterRegistration API using the cluster's kube-system namespace UUID.
+func isClusterRegisteredInPC(
+	ctx context.Context,
+	client ctrlclient.Client,
+	cluster *clusterv1.Cluster,
+	log logr.Logger,
+) (bool, error) {
+	// Get cluster config to extract PC endpoint
+	varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables)
+	clusterConfigVar, err := variables.Get[apivariables.ClusterConfigSpec](
+		varMap,
+		v1alpha1.ClusterConfigVariableName,
+	)
+	if err != nil {
+		return false, fmt.Errorf("failed to read clusterConfig variable: %w", err)
+	}
+
+	if clusterConfigVar.Nutanix == nil || clusterConfigVar.Nutanix.PrismCentralEndpoint.URL == "" {
+		return false, fmt.Errorf("prism central endpoint not configured")
+	}
+
+	prismCentralEndpointSpec := clusterConfigVar.Nutanix.PrismCentralEndpoint
+	host, port, err := prismCentralEndpointSpec.ParseURL()
+	if err != nil {
+		return false, fmt.Errorf("failed to parse prism central endpoint URL: %w", err)
+	}
+
+	// Get konnector agent variable to access its credentials secret
+	k8sAgentVar, err := variables.Get[apivariables.NutanixKonnectorAgent](
+		varMap,
+		v1alpha1.ClusterConfigVariableName,
+		"addons", v1alpha1.KonnectorAgentVariableName,
+	)
+	if err != nil {
+		return false, fmt.Errorf("failed to read konnector agent variable: %w", err)
+	}
+
+	if k8sAgentVar.Credentials == nil || k8sAgentVar.Credentials.SecretRef.Name == "" {
+		return false, fmt.Errorf("konnector agent credentials secret not configured")
+	}
+
+	// Get credentials from konnector agent addon Secret
+	credentialsSecret := &corev1.Secret{}
+	err = client.Get(ctx, types.NamespacedName{
+		Namespace: cluster.Namespace,
+		Name:      k8sAgentVar.Credentials.SecretRef.Name,
+	}, credentialsSecret)
+	if err != nil {
+		return false, fmt.Errorf("failed to get credentials secret: %w", err)
+	}
+
+	usernameData, ok := credentialsSecret.Data["username"]
+	if !ok {
+		return false, fmt.Errorf("credentials secret does not contain 'username' key")
+	}
+	passwordData, ok := credentialsSecret.Data["password"]
+	if !ok {
+		return false, fmt.Errorf("credentials secret does not contain 'password' key")
+	}
+
+	// Create credentials struct
+	credentials := prismgoclient.Credentials{
+		Endpoint: fmt.Sprintf("%s:%d", host, port),
+		URL:      fmt.Sprintf("https://%s:%d", host, port),
+		Username: string(usernameData),
+		Password: string(passwordData),
+		Insecure: prismCentralEndpointSpec.Insecure,
+		Port:     fmt.Sprintf("%d", port),
+	}
+
+	// Get kube-system namespace UUID from the cluster
+	clusterKey := ctrlclient.ObjectKeyFromObject(cluster)
+	remoteClient, err := remote.NewClusterClient(ctx, "", client, clusterKey)
+	if err != nil {
+		return false, fmt.Errorf("failed to create remote cluster client: %w", err)
+	}
+
+	kubeSystemNS := &corev1.Namespace{}
+	err = remoteClient.Get(ctx, types.NamespacedName{Name: "kube-system"}, kubeSystemNS)
+	if err != nil {
+		return false, fmt.Errorf("failed to get kube-system namespace from cluster(%s): %w", cluster.Name, err)
+	}
+
+	clusterUUID := string(kubeSystemNS.UID)
+
+	// Get trust bundle if insecure is false
+	var trustBundle string
+	if !prismCentralEndpointSpec.Insecure {
+		trustBundle = prismCentralEndpointSpec.AdditionalTrustBundle
+	}
+
+	// Create Prism Central Konnector client
+	prismCentralKonnectorClient, err := lifecycleutils.NewPrismCentralKonnectorClient(&credentials, trustBundle)
+	if err != nil {
+		return false, fmt.Errorf("failed to create prism central konnector client: %w", err)
+	}
+
+	// Call GetClusterRegistration API
+	_, err = prismCentralKonnectorClient.GetClusterRegistration(ctx, clusterUUID)
+	if err != nil {
+		return false, fmt.Errorf("failed to get cluster(%s) registration: %w", clusterUUID, err)
+	}
+
+	// If we got here, the cluster is registered
+	log.Info("Cluster is registered in Prism Central", "clusterUUID", clusterUUID)
+	return true, nil
+}

pkg/handlers/lifecycle/konnectoragent/variables_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1165,3 +1166,143 @@ func TestFormatCategoriesFromSlice(t *testing.T) {
 		})
 	}
 }
+
+// Test isClusterRegisteredInPC function
+func TestIsClusterRegisteredInPC_MissingClusterConfig(t *testing.T) {
+	client := fake.NewClientBuilder().WithScheme(testScheme).Build()
+	cluster := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
+		Spec: clusterv1.ClusterSpec{
+			Topology: &clusterv1.Topology{
+				Variables: []clusterv1.ClusterVariable{},
+			},
+		},
+	}
+
+	registered, err := isClusterRegisteredInPC(context.Background(), client, cluster, logr.Discard())
+
+	assert.Error(t, err)
+	assert.False(t, registered)
+	assert.Contains(t, err.Error(), "failed to read clusterConfig variable")
+}
+
+func TestIsClusterRegisteredInPC_MissingCredentialsSecret(t *testing.T) {
+	client := fake.NewClientBuilder().WithScheme(testScheme).Build()
+	cluster := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
+		Spec: clusterv1.ClusterSpec{
+			Topology: &clusterv1.Topology{
+				Variables: []clusterv1.ClusterVariable{{
+					Name: v1alpha1.ClusterConfigVariableName,
+					Value: apiextensionsv1.JSON{Raw: []byte(`{
+						"nutanix": {
+							"prismCentralEndpoint": {
+								"url": "https://prism-central.example.com:9440",
+								"insecure": true
+							}
+						},
+						"addons": {
+							"konnectorAgent": {
+								"credentials": { "secretRef": {"name":"missing-secret"} }
+							}
+						}
+					}`)},
+				}},
+			},
+		},
+	}
+
+	registered, err := isClusterRegisteredInPC(context.Background(), client, cluster, logr.Discard())
+
+	assert.Error(t, err)
+	assert.False(t, registered)
+	assert.Contains(t, err.Error(), "failed to get credentials secret")
+}
+
+func TestIsClusterRegisteredInPC_MissingUsernameInSecret(t *testing.T) {
+	client := fake.NewClientBuilder().WithScheme(testScheme).Build()
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-secret",
+			Namespace: "default",
+		},
+		Data: map[string][]byte{
+			"password": []byte("testpass"),
+		},
+	}
+	require.NoError(t, client.Create(context.Background(), secret))
+
+	cluster := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
+		Spec: clusterv1.ClusterSpec{
+			Topology: &clusterv1.Topology{
+				Variables: []clusterv1.ClusterVariable{{
+					Name: v1alpha1.ClusterConfigVariableName,
+					Value: apiextensionsv1.JSON{Raw: []byte(`{
+						"nutanix": {
+							"prismCentralEndpoint": {
+								"url": "https://prism-central.example.com:9440",
+								"insecure": true
+							}
+						},
+						"addons": {
+							"konnectorAgent": {
+								"credentials": { "secretRef": {"name":"test-secret"} }
+							}
+						}
+					}`)},
+				}},
+			},
+		},
+	}
+
+	registered, err := isClusterRegisteredInPC(context.Background(), client, cluster, logr.Discard())
+
+	assert.Error(t, err)
+	assert.False(t, registered)
+	assert.Contains(t, err.Error(), "credentials secret does not contain 'username' key")
+}
+
+func TestIsClusterRegisteredInPC_MissingPasswordInSecret(t *testing.T) {
+	client := fake.NewClientBuilder().WithScheme(testScheme).Build()
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-secret",
+			Namespace: "default",
+		},
+		Data: map[string][]byte{
+			"username": []byte("testuser"),
+		},
+	}
+	require.NoError(t, client.Create(context.Background(), secret))
+
+	cluster := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
+		Spec: clusterv1.ClusterSpec{
+			Topology: &clusterv1.Topology{
+				Variables: []clusterv1.ClusterVariable{{
+					Name: v1alpha1.ClusterConfigVariableName,
+					Value: apiextensionsv1.JSON{Raw: []byte(`{
+						"nutanix": {
+							"prismCentralEndpoint": {
+								"url": "https://prism-central.example.com:9440",
+								"insecure": true
+							}
+						},
+						"addons": {
+							"konnectorAgent": {
+								"credentials": { "secretRef": {"name":"test-secret"} }
+							}
+						}
+					}`)},
+				}},
+			},
+		},
+	}
+
+	registered, err := isClusterRegisteredInPC(context.Background(), client, cluster, logr.Discard())
+
+	assert.Error(t, err)
+	assert.False(t, registered)
+	assert.Contains(t, err.Error(), "credentials secret does not contain 'password' key")
+}

pkg/handlers/lifecycle/utils/konnector_client.go

@@ -0,0 +1,65 @@
+// Copyright 2023 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package utils provides utility functions for lifecycle handlers.
+package utils
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+
+	prismgoclient "github.com/nutanix-cloud-native/prism-go-client"
+	konnectorprismgoclient "github.com/nutanix-cloud-native/prism-go-client/karbon"
+)
+
+// NewPrismCentralKonnectorClient creates a new Prism Konnector client that is used to call the Konnector APIs.
+func NewPrismCentralKonnectorClient(credentials *prismgoclient.Credentials, additionalTrustBundle string,
+	clientOpts ...konnectorprismgoclient.ClientOption,
+) (*PrismCentralKonnectorClient, error) {
+	if credentials == nil {
+		return nil, fmt.Errorf(
+			" Prism Central credentials cannot be nil, needed to create Prism Central Konnector client",
+		)
+	}
+
+	if additionalTrustBundle != "" {
+		certBytes, err := base64.StdEncoding.DecodeString(additionalTrustBundle)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode base64 certificate: %w", err)
+		}
+		clientOpts = append(clientOpts, konnectorprismgoclient.WithPEMEncodedCertBundle(certBytes))
+		// Set insecure to false if trust bundle is provided.
+		credentials.Insecure = false
+	}
+
+	prismCentralKonnectorClient, err := konnectorprismgoclient.NewKarbonAPIClient(*credentials, clientOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create prism konnector client: %w", err)
+	}
+
+	return &PrismCentralKonnectorClient{prismCentralKonnectorClient: prismCentralKonnectorClient}, nil
+}
+
+// PrismCentralKonnectorClient wraps the Prism Central Konnector client.
+type PrismCentralKonnectorClient struct {
+	prismCentralKonnectorClient *konnectorprismgoclient.Client
+}
+
+// GetClusterRegistration retrieves the cluster registration from Prism Central.
+func (pc *PrismCentralKonnectorClient) GetClusterRegistration(
+	ctx context.Context,
+	clusterUUID string,
+) (*konnectorprismgoclient.K8sClusterRegistration, error) {
+	if pc == nil || pc.prismCentralKonnectorClient == nil {
+		return nil, fmt.Errorf("could not connect to API server on PC: client is nil")
+	}
+	clusterRegistration, err := pc.prismCentralKonnectorClient.ClusterRegistrationOperations.GetK8sRegistration(
+		ctx,
+		clusterUUID,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get Kubernetes cluster(%s) registration: %w", clusterUUID, err)
+	}
+	return clusterRegistration, nil
+}