Merge pull request #37 from aws/feature/destroy_vm_async

Destroy VM asynchronously

Author: jweite-amazon

controllers/cloudstackmachine_controller.go

@@ -55,6 +55,7 @@ type CloudStackMachineReconciler struct {
 }
 
 const requeueTimeout = 5 * time.Second
+const destoryVMRequeueInterval = 10 * time.Second
 
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get;update;patch
@@ -225,6 +226,10 @@ func (r *CloudStackMachineReconciler) reconcileDelete(
 
 	log.Info("Deleting instance", "instance-id", *csMachine.Spec.InstanceID)
 	if err := r.CS.DestroyVMInstance(csMachine); err != nil {
+		if err.Error() == "VM deletion in progress" {
+			log.Info(err.Error())
+			return ctrl.Result{RequeueAfter: destoryVMRequeueInterval}, nil
+		}
 		return ctrl.Result{}, err
 	}
 	controllerutil.RemoveFinalizer(csMachine, infrav1.MachineFinalizer)

pkg/cloud/client.go

@@ -41,9 +41,8 @@ type Client interface {
 }
 
 type client struct {
-	cs *cloudstack.CloudStackClient
-	// This is a placeholder for sending non-blocking requests.
-	// csA *cloudstack.CloudStackClient
+	cs      *cloudstack.CloudStackClient
+	csAsync *cloudstack.CloudStackClient
 }
 
 // cloud-config ini structure.
@@ -65,11 +64,9 @@ func NewClient(ccPath string) (Client, error) {
 		return nil, errors.Wrapf(err, "error encountered while parsing [Global] section from config at path: %s", ccPath)
 	}
 
-	// This is a placeholder for sending non-blocking requests.
-	// c.csA = cloudstack.NewClient(apiUrl, apiKey, secretKey, false)
-	// TODO: attempt a less clunky client liveliness check (not just listing zones).
 	c.cs = cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
-	_, err := c.cs.Zone.ListZones(c.cs.Zone.NewListZonesParams())
+	c.csAsync = cloudstack.NewClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
+	_, err := c.cs.APIDiscovery.ListApis(c.cs.APIDiscovery.NewListApisParams())
 	if err != nil && strings.Contains(strings.ToLower(err.Error()), "i/o timeout") {
 		return c, errors.Wrap(err, "Timeout while checking CloudStack API Client connectivity.")
 	}

pkg/cloud/instance.go

@@ -213,12 +213,26 @@ func (c *client) GetOrCreateVMInstance(
 // DestroyVMInstance Destroy a VM instance. Assumes machine has been fetched prior and has an instance ID.
 func (c *client) DestroyVMInstance(csMachine *infrav1.CloudStackMachine) error {
 
+	if err := c.ResolveVMInstanceDetails(csMachine); err == nil && csMachine.Status.InstanceState != "Running" {
+		if csMachine.Status.InstanceState == "Stopping" ||
+			csMachine.Status.InstanceState == "Stopped" {
+			return errors.New("VM deletion in progress")
+		} else if csMachine.Status.InstanceState == "Expunging" ||
+			csMachine.Status.InstanceState == "Expunged" {
+			// VM is stopped and getting expunged.  So the desired state is getting satisfied.  Let's move on.
+			return nil
+		}
+	} else if err != nil && strings.Contains(strings.ToLower(err.Error()), "no match found") {
+		// VM doesn't exist.  So the desired state is in effect.  Our work is done here.
+		return nil
+	}
+
 	p := c.cs.VirtualMachine.NewDestroyVirtualMachineParams(*csMachine.Spec.InstanceID)
 	p.SetExpunge(true)
-	_, err := c.cs.VirtualMachine.DestroyVirtualMachine(p)
-	if err != nil && strings.Contains(strings.ToLower(err.Error()), "unable to find UUID for id ") {
+	_, err := c.csAsync.VirtualMachine.DestroyVirtualMachine(p)
+	if err != nil && strings.Contains(err.Error(), "Unable to find UUID for id") {
 		// VM doesn't exist.  So the desired state is in effect.  Our work is done here.
 		return nil
 	}
-	return err
+	return errors.New("VM deletion in progress")
 }