vmm: Move vm_maxcpu handling into MI code

No functional change intended.

Reviewed by:	corvink
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Sponsored by:	Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D53477

Author: markjdb

sys/amd64/include/vmm.h

@@ -206,8 +206,6 @@ struct vmm_ops {
 extern const struct vmm_ops vmm_ops_intel;
 extern const struct vmm_ops vmm_ops_amd;
 
-extern u_int vm_maxcpu;			/* maximum virtual cpus */
-
 int vm_create(const char *name, struct vm **retvm);
 struct vcpu *vm_alloc_vcpu(struct vm *vm, int vcpuid);
 void vm_disable_vcpu_creation(struct vm *vm);

sys/amd64/vmm/intel/vmx.c

@@ -27,7 +27,6 @@
  * SUCH DAMAGE.
  */
 
-#include <sys/cdefs.h>
 #include "opt_bhyve_snapshot.h"
 
 #include <sys/param.h>
@@ -58,6 +57,7 @@
 #include <machine/vmm_instruction_emul.h>
 #include <machine/vmm_snapshot.h>
 
+#include <dev/vmm/vmm_dev.h>
 #include <dev/vmm/vmm_ktr.h>
 #include <dev/vmm/vmm_mem.h>
 

sys/amd64/vmm/vmm.c

@@ -267,10 +267,6 @@ static int trap_wbinvd;
 SYSCTL_INT(_hw_vmm, OID_AUTO, trap_wbinvd, CTLFLAG_RDTUN, &trap_wbinvd, 0,
     "WBINVD triggers a VM-exit");
 
-u_int vm_maxcpu;
-SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
-    &vm_maxcpu, 0, "Maximum number of vCPUs");
-
 static void vcpu_notify_event_locked(struct vcpu *vcpu);
 
 /* global statistics */
@@ -296,14 +292,6 @@ VMM_STAT(VMEXIT_USERSPACE, "number of vm exits handled in userspace");
 VMM_STAT(VMEXIT_RENDEZVOUS, "number of times rendezvous pending at exit");
 VMM_STAT(VMEXIT_EXCEPTION, "number of vm exits due to exceptions");
 
-/*
- * Upper limit on vm_maxcpu.  Limited by use of uint16_t types for CPU
- * counts as well as range of vpid values for VT-x and by the capacity
- * of cpuset_t masks.  The call to new_unrhdr() in vpid_init() in
- * vmx.c requires 'vm_maxcpu + 1 <= 0xffff', hence the '- 1' below.
- */
-#define	VM_MAXCPU	MIN(0xffff - 1, CPU_SETSIZE)
-
 #ifdef KTR
 static const char *
 vcpu_state2str(enum vcpu_state state)
@@ -405,16 +393,6 @@ vmm_modinit(void)
 	if (!vmm_is_hw_supported())
 		return (ENXIO);
 
-	vm_maxcpu = mp_ncpus;
-	TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu);
-
-	if (vm_maxcpu > VM_MAXCPU) {
-		printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU);
-		vm_maxcpu = VM_MAXCPU;
-	}
-	if (vm_maxcpu == 0)
-		vm_maxcpu = 1;
-
 	vmm_host_state_init();
 
 	vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :

sys/arm64/vmm/vmm.c

@@ -205,10 +205,6 @@ static const struct vmm_regs vmm_arch_regs_masks = {
 /* Host registers masked by vmm_arch_regs_masks. */
 static struct vmm_regs vmm_arch_regs;
 
-u_int vm_maxcpu;
-SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
-    &vm_maxcpu, 0, "Maximum number of vCPUs");
-
 static void vcpu_notify_event_locked(struct vcpu *vcpu);
 
 /* global statistics */
@@ -228,12 +224,6 @@ VMM_STAT(VMEXIT_SS, "number of vmexits for a single-step exception");
 VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception");
 VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception");
 
-/*
- * Upper limit on vm_maxcpu. We could increase this to 28 bits, but this
- * is a safe value for now.
- */
-#define	VM_MAXCPU	MIN(0xffff - 1, CPU_SETSIZE)
-
 static int
 vmm_regs_init(struct vmm_regs *regs, const struct vmm_regs *masks)
 {
@@ -329,16 +319,6 @@ vmm_modinit(void)
 	if (error != 0)
 		return (error);
 
-	vm_maxcpu = mp_ncpus;
-	TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu);
-
-	if (vm_maxcpu > VM_MAXCPU) {
-		printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU);
-		vm_maxcpu = VM_MAXCPU;
-	}
-	if (vm_maxcpu == 0)
-		vm_maxcpu = 1;
-
 	error = vmm_regs_init(&vmm_arch_regs, &vmm_arch_regs_masks);
 	if (error != 0)
 		return (error);

sys/dev/vmm/vmm_dev.c

@@ -18,6 +18,7 @@
 #include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/queue.h>
+#include <sys/smp.h>
 #include <sys/sx.h>
 #include <sys/sysctl.h>
 #include <sys/ucred.h>
@@ -91,6 +92,10 @@ static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev");
 
 SYSCTL_DECL(_hw_vmm);
 
+u_int vm_maxcpu;
+SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+    &vm_maxcpu, 0, "Maximum number of vCPUs");
+
 static void devmem_destroy(void *arg);
 static int devmem_create_cdev(struct vmmdev_softc *sc, int id, char *devmem);
 
@@ -1158,6 +1163,16 @@ vmm_handler(module_t mod, int what, void *arg)
 		error = vmmdev_init();
 		if (error != 0)
 			break;
+
+		vm_maxcpu = mp_ncpus;
+		TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu);
+		if (vm_maxcpu > VM_MAXCPU) {
+			printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU);
+			vm_maxcpu = VM_MAXCPU;
+		}
+		if (vm_maxcpu == 0)
+			vm_maxcpu = 1;
+
 		error = vmm_modinit();
 		if (error == 0)
 			vmm_initialized = true;

sys/dev/vmm/vmm_dev.h

@@ -57,6 +57,17 @@ struct vmmdev_ioctl {
 extern const struct vmmdev_ioctl vmmdev_machdep_ioctls[];
 extern const size_t vmmdev_machdep_ioctl_count;
 
+/*
+ * Upper limit on vm_maxcpu.  Limited by use of uint16_t types for CPU counts as
+ * well as range of vpid values for VT-x on amd64 and by the capacity of
+ * cpuset_t masks.  The call to new_unrhdr() in vpid_init() in vmx.c requires
+ * 'vm_maxcpu + 1 <= 0xffff', hence the '- 1' below.
+ */
+#define	VM_MAXCPU	MIN(0xffff - 1, CPU_SETSIZE)
+
+/* Maximum number of vCPUs in a single VM. */
+extern u_int vm_maxcpu;
+
 #endif /* _KERNEL */
 
 struct vmmctl_vm_create {

sys/riscv/vmm/vmm.c

@@ -143,23 +143,13 @@ static int vmm_ipinum;
 SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
     "IPI vector used for vcpu notifications");
 
-u_int vm_maxcpu;
-SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
-    &vm_maxcpu, 0, "Maximum number of vCPUs");
-
 static void vcpu_notify_event_locked(struct vcpu *vcpu);
 
 /* global statistics */
 VMM_STAT(VMEXIT_COUNT, "total number of vm exits");
 VMM_STAT(VMEXIT_IRQ, "number of vmexits for an irq");
 VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception");
 
-/*
- * Upper limit on vm_maxcpu. We could increase this to 28 bits, but this
- * is a safe value for now.
- */
-#define	VM_MAXCPU	MIN(0xffff - 1, CPU_SETSIZE)
-
 static void
 vcpu_cleanup(struct vcpu *vcpu, bool destroy)
 {
@@ -210,18 +200,6 @@ vm_exitinfo(struct vcpu *vcpu)
 int
 vmm_modinit(void)
 {
-	vm_maxcpu = mp_ncpus;
-
-	TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu);
-
-	if (vm_maxcpu > VM_MAXCPU) {
-		printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU);
-		vm_maxcpu = VM_MAXCPU;
-	}
-
-	if (vm_maxcpu == 0)
-		vm_maxcpu = 1;
-
 	return (vmmops_modinit());
 }