Reconciler to monitor the changes in AWS FSxN resources

Utkarshjh · web-flow · commit 774190bf4e04 · 2024-12-14T18:35:56.000+05:30
diff --git a/helm/trident-operator/templates/clusterrole.yaml b/helm/trident-operator/templates/clusterrole.yaml
@@ -144,6 +144,7 @@ rules:
       - volumesnapshotclasses
     verbs:
       - create
+      - delete
   - apiGroups:
       - snapshot.storage.k8s.io
     resources:
diff --git a/mocks/mock_operator/mock_controllers/mock_configurator/mock_clients/mock_api.go b/mocks/mock_operator/mock_controllers/mock_configurator/mock_clients/mock_api.go
diff --git a/mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go b/mocks/mock_storage_drivers/mock_ontap/mock_awsapi.go
diff --git a/operator/clients/trident_crd.go b/operator/clients/trident_crd.go
@@ -41,3 +41,7 @@ func (tc *TridentCRDClient) PatchTridentBackendConfig(name, namespace string, pa
 func (tc *TridentCRDClient) DeleteTridentBackendConfig(name, namespace string) error {
 	return tc.client.TridentV1().TridentBackendConfigs(namespace).Delete(ctx, name, deleteOpts)
 }
+
+func (tc *TridentCRDClient) ListTridentBackend(namespace string) (*tridentV1.TridentBackendList, error) {
+	return tc.client.TridentV1().TridentBackends(namespace).List(ctx, listOpts)
+}
diff --git a/operator/clients/types.go b/operator/clients/types.go
@@ -38,6 +38,7 @@ type OperatorCRDClientInterface interface {
 type TridentCRDClientInterface interface {
 	CheckTridentBackendConfigExists(name, namespace string) (bool, error)
 	GetTridentBackendConfig(name, namespace string) (*tridentV1.TridentBackendConfig, error)
+	ListTridentBackend(namespace string) (*tridentV1.TridentBackendList, error)
 	PatchTridentBackendConfig(name, namespace string, patchBytes []byte, patchType types.PatchType) error
 	DeleteTridentBackendConfig(name, namespace string) error
 }
diff --git a/operator/controllers/configurator/clients/configurator.go b/operator/controllers/configurator/clients/configurator.go
@@ -65,7 +65,7 @@ func (c *ConfiguratorClient) CreateOrPatchObject(objType ObjectType, objName, ob
 		Log().Errorf("Patch failed for %s %s: %v", objName, objType, err)
 	}
 
-	if err := c.deleteObject(objType, objName, objNamespace); err != nil {
+	if err := c.DeleteObject(objType, objName, objNamespace); err != nil {
 		return err
 	}
 
@@ -166,7 +166,8 @@ func (c *ConfiguratorClient) patchObject(objType ObjectType, objName, objNamespa
 	return fmt.Errorf("unsupported object %s of type %s", objName, objType)
 }
 
-func (c *ConfiguratorClient) deleteObject(objType ObjectType, objName, objNamespace string) error {
+// DeleteObject deletes the object of the given type and name.
+func (c *ConfiguratorClient) DeleteObject(objType ObjectType, objName, objNamespace string) error {
 	switch objType {
 	case OCRD:
 		return c.kClient.DeleteCRD(objName)
@@ -181,6 +182,19 @@ func (c *ConfiguratorClient) deleteObject(objType ObjectType, objName, objNamesp
 	return fmt.Errorf("unsupported object %s of type %s", objName, objType)
 }
 
+// ListObjects lists the objects of the given type.
+func (c *ConfiguratorClient) ListObjects(objType ObjectType, objNamespace string) (interface{}, error) {
+	switch objType {
+	case OCRD:
+	case OBackend:
+		return c.tClient.ListTridentBackend(objNamespace)
+	case OStorageClass:
+	case OSnapshotClass:
+	}
+
+	return nil, fmt.Errorf("unsupported object type %s", objType)
+}
+
 func (c *ConfiguratorClient) GetControllingTorcCR() (*operatorV1.TridentOrchestrator, error) {
 	return c.oClient.GetControllingTorcCR()
 }
diff --git a/operator/controllers/configurator/clients/types.go b/operator/controllers/configurator/clients/types.go
@@ -47,6 +47,8 @@ type ConfiguratorClientInterface interface {
 	GetControllingTorcCR() (*operatorV1.TridentOrchestrator, error)
 	GetTconfCR(name string) (*operatorV1.TridentConfigurator, error)
 	GetANFSecrets(secretName string) (string, string, error)
+	DeleteObject(objType ObjectType, objName, objNamespace string) error
+	ListObjects(objType ObjectType, objNamespace string) (interface{}, error)
 }
 
 type ExtendedK8sClientInterface interface {
diff --git a/operator/controllers/configurator/controller.go b/operator/controllers/configurator/controller.go
@@ -394,20 +394,20 @@ func (c *Controller) reconcile(keyItem string) error {
 			return err
 		}
 	case config.OntapNASStorageDriverName, config.OntapSANStorageDriverName:
-		isAwsFsxn, err := tconfCR.IsAwsFsxnTconf()
+		isAwsFSxN, err := tconfCR.IsAWSFSxNTconf()
 		if err != nil {
 			Log().Error("Failed to check if tconf is for AWS FSxN: ", err)
 			return err
 		}
-		if isAwsFsxn {
+		if isAwsFSxN {
 			Log().Debugf("Tconf indicates auto-backend config is for AWS FSxN")
-			fsxn, err := storage_drivers.NewFsxnInstance(torcCR, tconfCR, c.Clients)
+			fsxn, err := storage_drivers.NewFSxNInstance(torcCR, tconfCR, c.Clients)
 			if err != nil {
 				Log().Info("Failed to create FsxN backend instance: ", err)
 				return err
 			}
 			if err := c.ProcessBackend(fsxn, tconfCR); err != nil {
-				Log().Error("Failed to process FsxN backend: ", err)
+				Log().Error("Failed to process FSxN backend: ", err)
 				return err
 			}
 		}
diff --git a/operator/controllers/configurator/storage_drivers/anf.go b/operator/controllers/configurator/storage_drivers/anf.go
@@ -222,6 +222,18 @@ func (a *ANF) GetCloudProvider() string {
 	return "None"
 }
 
+func (a *ANF) DeleteBackend(map[string]interface{}) error {
+	return nil
+}
+
+func (a *ANF) DeleteStorageClass(map[string]interface{}) error {
+	return nil
+}
+
+func (a *ANF) DeleteSnapshotClass() error {
+	return nil
+}
+
 func (a *ANF) populateAndValidateAZResources() error {
 	a.FilteredCapacityPoolMap = a.AZClient.FilteredCapacityPoolMap(context.TODO(), a.ResourceGroups,
 		a.NetappAccounts, a.CapacityPools)
diff --git a/operator/controllers/configurator/storage_drivers/fsx.go b/operator/controllers/configurator/storage_drivers/fsx.go
@@ -17,9 +17,11 @@ import (
 	. "github.com/netapp/trident/logging"
 	confClients "github.com/netapp/trident/operator/controllers/configurator/clients"
 	operatorV1 "github.com/netapp/trident/operator/crd/apis/netapp/v1"
+	tridentV1 "github.com/netapp/trident/persistent_store/crd/apis/netapp/v1"
 	sa "github.com/netapp/trident/storage_attribute"
 	"github.com/netapp/trident/storage_drivers/ontap/awsapi"
 	"github.com/netapp/trident/utils"
+	"github.com/netapp/trident/utils/errors"
 )
 
 const (
@@ -60,8 +62,8 @@ type SVM struct {
 	ManagementLIF string   `json:"managementLIF"`
 }
 
-// NewFsxnInstance creates a new instance of the AWS struct and populates it with the provided CRs and client
-func NewFsxnInstance(
+// NewFSxNInstance creates a new instance of the AWS struct and populates it with the provided CRs and client
+func NewFSxNInstance(
 	torcCR *operatorV1.TridentOrchestrator, configuratorCR *operatorV1.TridentConfigurator,
 	client confClients.ConfiguratorClientInterface,
 ) (*AWS, error) {
@@ -70,7 +72,7 @@ func NewFsxnInstance(
 	}
 
 	if configuratorCR == nil {
-		return nil, fmt.Errorf("empty AWSFsxN configurator CR")
+		return nil, fmt.Errorf("empty AWS FSxN configurator CR")
 	}
 
 	if client == nil {
@@ -117,33 +119,39 @@ func (aws *AWS) Validate() error {
 	aws.AwsClient = api
 
 	for key, fsxnInstance := range aws.SVMs {
-		if err := aws.processFsxnInstance(context.Background(), key, fsxnInstance); err != nil {
+		if err := aws.processFSxNInstance(context.Background(), key, fsxnInstance); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
-// processFsxnInstance processes the auto-backend configuration for the FSxN instance.
+// processFSxNInstance processes the auto-backend configuration for the FSxN instance.
 // It creates the SVM if it does not exist and creates the secret for the SVM.
-func (aws *AWS) processFsxnInstance(ctx context.Context, key int, svm SVM) error {
+func (aws *AWS) processFSxNInstance(ctx context.Context, key int, svm SVM) error {
 	var (
-		svmName    string
-		secretName string
-		secretARN  string
-		svmExists  bool
+		svmName       string
+		secretName    string
+		secretARN     string
+		svmExists     bool
+		ErrStatusCode = "StatusCode: 400"
 	)
 	svmName = svm.SvmName
 	if svmName == "" {
 		svmName = fmt.Sprintf(SvmNamePattern, svm.FsxnID)
+		svm.SvmName = svmName
 	}
 	_, err := aws.AwsClient.GetFilesystemByID(ctx, svm.FsxnID)
-	if err != nil {
+	if errors.IsNotFoundError(err) || (err != nil && strings.Contains(err.Error(), ErrStatusCode)) {
+		err = aws.cleanUpFSxNRelatedObjects(svm, svm.Protocols)
+		if err != nil {
+			Log().Error(err)
+		}
 		return fmt.Errorf("error occurred while getting fsxn id: %v : %v", svm.FsxnID, err)
 	}
 	Log().Debugf("Filesystem ID: %s exists", svm.FsxnID)
-	secretName = fmt.Sprintf(TridentSecretPattern, strings.TrimPrefix(svmName, "trident-"))
+	secretName = getAWSSecretName(svmName)
 	// Get the secret if it already exists or create a new one
 	secret, _ := aws.AwsClient.GetSecret(ctx, secretName)
 	if secret != nil {
@@ -245,7 +253,7 @@ func (aws *AWS) Create() ([]string, error) {
 	)
 	for _, svm := range aws.SVMs {
 		for _, protocol := range svm.Protocols {
-			backendName = getFsxnBackendName(svm.FsxnID, protocol)
+			backendName = getFSxNBackendName(svm.FsxnID, protocol)
 			backendYAML := getFsxnTBCYaml(svm, aws.TridentNamespace, backendName, protocol)
 			if err := aws.ConfClient.CreateOrPatchObject(confClients.OBackend, backendName,
 				aws.TridentNamespace, backendYAML); err != nil {
@@ -262,7 +270,7 @@ func (aws *AWS) CreateStorageClass() error {
 	var driver string
 	for _, svm := range aws.SVMs {
 		for _, protocol := range svm.Protocols {
-			name := getFsxnStorageClassName(svm.FsxnID, protocol)
+			name := getFSxNStorageClassName(svm.FsxnID, protocol)
 			if protocol == sa.NFS {
 				driver = config.OntapNASStorageDriverName
 			} else if protocol == sa.ISCSI {
@@ -290,12 +298,95 @@ func (aws *AWS) GetCloudProvider() string {
 	return k8sclient.CloudProviderAWS
 }
 
-// getFsxnTBCYaml returns the FsxN Trident backend config name
-func getFsxnBackendName(fsxnId, protocolType string) string {
+// DeleteBackend deletes the backend if the FSxN instance is deleted
+func (aws *AWS) DeleteBackend(request map[string]interface{}) error {
+	protocols := request["protocols"].([]string)
+	fsxnId := request["FSxNID"].(string)
+	for _, protocol := range protocols {
+		backendName := getFSxNBackendName(fsxnId, protocol)
+		if err := aws.ConfClient.DeleteObject(confClients.OBackend, backendName, aws.TridentNamespace); err != nil {
+			return fmt.Errorf("error occurred while deleting backend: %w", err)
+		}
+	}
+	return nil
+}
+
+// DeleteStorageClass deletes the storage class if the FSxN instance is deleted
+func (aws *AWS) DeleteStorageClass(request map[string]interface{}) error {
+	protocols := request["protocols"].([]string)
+	fsxnId := request["FSxNID"].(string)
+	for _, protocol := range protocols {
+		name := getFSxNStorageClassName(fsxnId, protocol)
+		if err := aws.ConfClient.DeleteObject(confClients.OStorageClass, name, aws.TridentNamespace); err != nil {
+			return fmt.Errorf("error occurred while deleting storage class: %w", err)
+		}
+	}
+	return nil
+}
+
+// DeleteSnapshotClass deletes the snapshot class if the FSxN instance is deleted
+func (aws *AWS) DeleteSnapshotClass() error {
+	tbcList, err := aws.ConfClient.ListObjects(confClients.OBackend, "")
+	if err != nil {
+		return fmt.Errorf("error occurred while listing TBC objects: %w", err)
+	}
+	if len(tbcList.(*tridentV1.TridentBackendList).Items) == 0 {
+		if err := aws.ConfClient.DeleteObject(confClients.OSnapshotClass, NetAppSnapshotClassName, ""); err != nil {
+			return fmt.Errorf("error occurred while deleting snapshot class: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// cleanUpFSxNRelatedObjects cleans up the FSxN instance related objects like storage class, backend, and AWS secret
+func (aws *AWS) cleanUpFSxNRelatedObjects(svm SVM, protocols []string) (err error) {
+	cleanupRequest := map[string]interface{}{
+		"FSxNID":    svm.FsxnID,
+		"protocols": protocols,
+	}
+
+	// Delete the storage class
+	if deleteErr := aws.DeleteStorageClass(cleanupRequest); err != nil {
+		err = fmt.Errorf("failed to delete VolumeSnapshotClass: %w", deleteErr)
+	}
+
+	// Delete the backend
+	if deleteErr := aws.DeleteBackend(cleanupRequest); err != nil {
+		err = fmt.Errorf("failed to delete backend: %w", deleteErr)
+	}
+
+	// Delete the AWS secret
+	if deleteErr := deleteSecret(context.Background(), aws, getAWSSecretName(svm.SvmName)); err != nil {
+		err = fmt.Errorf("failed to delete AWS secret: %w", deleteErr)
+	}
+
+	// Delete the VolumeSnapshotClass if no backend is present
+	if deleteErr := aws.DeleteSnapshotClass(); err != nil {
+		err = fmt.Errorf("failed to delete VolumeSnapshotClass: %w", deleteErr)
+	}
+
+	return
+}
+
+func deleteSecret(ctx context.Context, aws *AWS, secretName string) error {
+	if err := aws.AwsClient.DeleteSecret(ctx, secretName); err != nil {
+		return fmt.Errorf("error occurred while deleting secret: %w", err)
+	}
+	return nil
+}
+
+// getFSxNBackendName returns the FsxN Trident backend config name
+func getFSxNBackendName(fsxnId, protocolType string) string {
 	return fmt.Sprintf(BackendNamePattern, fsxnId, protocolType)
 }
 
-// getFsxnStorageClassName returns the storage class name for the FSxN backend
-func getFsxnStorageClassName(fsxnId, protocolType string) string {
+// getFSxNStorageClassName returns the storage class name for the FSxN backend
+func getFSxNStorageClassName(fsxnId, protocolType string) string {
 	return fmt.Sprintf(StorageClassNamePattern, fsxnId, protocolType)
 }
+
+// getAWSSecretName returns the AWS secret name for the FSxN backend
+func getAWSSecretName(svmName string) string {
+	return fmt.Sprintf(TridentSecretPattern, strings.TrimPrefix(svmName, "trident-"))
+}
diff --git a/operator/controllers/configurator/storage_drivers/types.go b/operator/controllers/configurator/storage_drivers/types.go
@@ -8,4 +8,7 @@ type Backend interface {
 	CreateStorageClass() error
 	CreateSnapshotClass() error
 	GetCloudProvider() string
+	DeleteBackend(map[string]interface{}) error
+	DeleteStorageClass(map[string]interface{}) error
+	DeleteSnapshotClass() error
 }
diff --git a/operator/crd/apis/netapp/v1/tridentconfigurator.go b/operator/crd/apis/netapp/v1/tridentconfigurator.go
@@ -58,7 +58,7 @@ func (tc *TridentConfigurator) Validate() error {
 	return nil
 }
 
-func (tc *TridentConfigurator) IsAwsFsxnTconf() (bool, error) {
+func (tc *TridentConfigurator) IsAWSFSxNTconf() (bool, error) {
 	var tConfSpec map[string]interface{}
 	if err := json.Unmarshal(tc.Spec.Raw, &tConfSpec); err != nil {
 		return false, err
diff --git a/storage_drivers/ontap/awsapi/aws.go b/storage_drivers/ontap/awsapi/aws.go
@@ -186,6 +186,25 @@ func (d *Client) GetSecret(ctx context.Context, secretARN string) (*Secret, erro
 	return secret, nil
 }
 
+// DeleteSecret deletes a secret by its ARN/name.
+func (d *Client) DeleteSecret(ctx context.Context, secretARN string) error {
+	logFields := LogFields{
+		"API":       "DeleteSecret",
+		"secretARN": secretARN,
+	}
+	deleteSecretInput := &secretsmanager.DeleteSecretInput{
+		SecretId:                   utils.Ptr(secretARN),
+		ForceDeleteWithoutRecovery: utils.Ptr(true),
+	}
+	if _, err := d.secretsClient.DeleteSecret(ctx, deleteSecretInput); err != nil {
+		logFields["requestID"] = GetRequestIDFromError(err)
+		Logc(ctx).WithFields(logFields).WithError(err).Error("Could not delete secret.")
+		return fmt.Errorf("error deleting secret; %w", err)
+	}
+	Logc(ctx).WithFields(logFields).Info("Secret deleted.")
+	return nil
+}
+
 // ParseVolumeARN parses the AWS-style ARN for a volume.
 func ParseVolumeARN(volumeARN string) (region, accountID, filesystemID, volumeID string, err error) {
 	match := volumeARNRegex.FindStringSubmatch(volumeARN)
diff --git a/storage_drivers/ontap/awsapi/types.go b/storage_drivers/ontap/awsapi/types.go
@@ -14,6 +14,7 @@ import (
 type AWSAPI interface {
 	CreateSecret(ctx context.Context, request *SecretCreateRequest) (*Secret, error)
 	GetSecret(ctx context.Context, secretARN string) (*Secret, error)
+	DeleteSecret(ctx context.Context, secretARN string) error
 
 	GetFilesystems(ctx context.Context) (*[]*Filesystem, error)
 	GetFilesystemByID(ctx context.Context, ID string) (*Filesystem, error)

Original file line number	Diff line number	Diff line change
`@@ -41,3 +41,7 @@ func (tc *TridentCRDClient) PatchTridentBackendConfig(name, namespace string, pa`
`41`	`41`	`func (tc *TridentCRDClient) DeleteTridentBackendConfig(name, namespace string) error {`
`42`	`42`	`return tc.client.TridentV1().TridentBackendConfigs(namespace).Delete(ctx, name, deleteOpts)`
`43`	`43`	`}`
	`44`	`+`
	`45`	`+func (tc TridentCRDClient) ListTridentBackend(namespace string) (tridentV1.TridentBackendList, error) {`
	`46`	`+ return tc.client.TridentV1().TridentBackends(namespace).List(ctx, listOpts)`
	`47`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ type OperatorCRDClientInterface interface {`
`38`	`38`	`type TridentCRDClientInterface interface {`
`39`	`39`	`CheckTridentBackendConfigExists(name, namespace string) (bool, error)`
`40`	`40`	`GetTridentBackendConfig(name, namespace string) (*tridentV1.TridentBackendConfig, error)`
	`41`	`+ ListTridentBackend(namespace string) (*tridentV1.TridentBackendList, error)`
`41`	`42`	`PatchTridentBackendConfig(name, namespace string, patchBytes []byte, patchType types.PatchType) error`
`42`	`43`	`DeleteTridentBackendConfig(name, namespace string) error`
`43`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,8 @@ type ConfiguratorClientInterface interface {`
`47`	`47`	`GetControllingTorcCR() (*operatorV1.TridentOrchestrator, error)`
`48`	`48`	`GetTconfCR(name string) (*operatorV1.TridentConfigurator, error)`
`49`	`49`	`GetANFSecrets(secretName string) (string, string, error)`
	`50`	`+ DeleteObject(objType ObjectType, objName, objNamespace string) error`
	`51`	`+ ListObjects(objType ObjectType, objNamespace string) (interface{}, error)`
`50`	`52`	`}`
`51`	`53`
`52`	`54`	`type ExtendedK8sClientInterface interface {`
Original file line number	Diff line number	Diff line change
`@@ -394,20 +394,20 @@ func (c *Controller) reconcile(keyItem string) error {`
`394`	`394`	`return err`
`395`	`395`	`}`
`396`	`396`	`case config.OntapNASStorageDriverName, config.OntapSANStorageDriverName:`
`397`		`- isAwsFsxn, err := tconfCR.IsAwsFsxnTconf()`
	`397`	`+ isAwsFSxN, err := tconfCR.IsAWSFSxNTconf()`
`398`	`398`	`if err != nil {`
`399`	`399`	`Log().Error("Failed to check if tconf is for AWS FSxN: ", err)`
`400`	`400`	`return err`
`401`	`401`	`}`
`402`		`- if isAwsFsxn {`
	`402`	`+ if isAwsFSxN {`
`403`	`403`	`Log().Debugf("Tconf indicates auto-backend config is for AWS FSxN")`
`404`		`- fsxn, err := storage_drivers.NewFsxnInstance(torcCR, tconfCR, c.Clients)`
	`404`	`+ fsxn, err := storage_drivers.NewFSxNInstance(torcCR, tconfCR, c.Clients)`
`405`	`405`	`if err != nil {`
`406`	`406`	`Log().Info("Failed to create FsxN backend instance: ", err)`
`407`	`407`	`return err`
`408`	`408`	`}`
`409`	`409`	`if err := c.ProcessBackend(fsxn, tconfCR); err != nil {`
`410`		`- Log().Error("Failed to process FsxN backend: ", err)`
	`410`	`+ Log().Error("Failed to process FSxN backend: ", err)`
`411`	`411`	`return err`
`412`	`412`	`}`
`413`	`413`	`}`