vmm: Suspend the VM before destroying it

Otherwise we don't do anything to kick vcpu threads out of a sleep
state when destroying a VM.  For instance, suppose a guest executes hlt
on amd64 or wfi on arm64 with interrupts disabled.  Then,
bhyvectl --destroy will hang until the vcpu thread somehow comes out of
vm_handle_hlt()/vm_handle_wfi() since destroy_dev() is waiting for vCPU
threads to drain.

Note that on amd64, if hw.vmm.halt_detection is set to 1 (the default),
the guest will automatically exit in this case since it's treated as a
shutdown.  But, the above should not hang if halt_detection is set to 0.

Here, vm_suspend() wakes up vcpu threads, and a subsequent attempt to
run the vCPU will result in an error which gets propagated to userspace,
allowing destroy_dev() to proceed.

Add a new suspend code for this purpose.  Modify bhyve to exit with
status 4 ("exited due to an error") when it's received, since that's
what'll happen generally when the VM is destroyed asynchronously.

Reported by:	def
MFC after:	2 weeks
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D51761

Author: markjdb

sys/amd64/include/vmm.h

@@ -46,6 +46,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
 	VM_SUSPEND_TRIPLEFAULT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 

sys/arm64/include/vmm.h

@@ -42,6 +42,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_RESET,
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 

sys/arm64/vmm/vmm.c

@@ -1342,8 +1342,14 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 static int
 vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 {
+	struct vm *vm;
+
+	vm = vcpu->vm;
 	vcpu_lock(vcpu);
 	while (1) {
+		if (vm->suspend)
+			break;
+
 		if (vgic_has_pending_irq(vcpu->cookie))
 			break;
 

sys/dev/vmm/vmm_dev.c

@@ -901,6 +901,7 @@ vmmdev_lookup_and_destroy(const char *name, struct ucred *cred)
 	sc->cdev = NULL;
 	sx_xunlock(&vmmdev_mtx);
 
+	vm_suspend(sc->vm, VM_SUSPEND_DESTROY);
 	destroy_dev(cdev);
 	vmmdev_destroy(sc);
 

sys/riscv/include/vmm.h

@@ -49,6 +49,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_RESET,
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 

sys/riscv/vmm/vmm.c

@@ -1036,10 +1036,14 @@ vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
 static int
 vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 {
+	struct vm *vm;
 
+	vm = vcpu->vm;
 	vcpu_lock(vcpu);
-
 	while (1) {
+		if (vm->suspend)
+			break;
+
 		if (aplic_check_pending(vcpu->cookie))
 			break;
 

usr.sbin/bhyve/aarch64/vmexit.c

@@ -122,6 +122,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 		exit(1);
 	case VM_SUSPEND_HALT:
 		exit(2);
+	case VM_SUSPEND_DESTROY:
+		exit(4);
 	default:
 		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
 		exit(100);

usr.sbin/bhyve/amd64/vmexit.c

@@ -418,6 +418,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 		exit(2);
 	case VM_SUSPEND_TRIPLEFAULT:
 		exit(3);
+	case VM_SUSPEND_DESTROY:
+		exit(4);
 	default:
 		EPRINTLN("vmexit_suspend: invalid reason %d", how);
 		exit(100);

usr.sbin/bhyve/bhyve.8

@@ -1126,7 +1126,7 @@ powered off
 .It 2
 halted
 .It 3
-triple fault
+triple fault (amd64 only)
 .It 4
 exited due to an error
 .El

usr.sbin/bhyve/bhyverun.c

@@ -561,10 +561,8 @@ fbsdrun_start_thread(void *param)
 #endif
 
 	vm_loop(vi->ctx, vi->vcpu);
-
-	/* not reached */
-	exit(1);
-	return (NULL);
+	/* We get here if the VM was destroyed asynchronously. */
+	exit(4);
 }
 
 void