Skip to content

Commit 0a51759

Browse files
Zuulopenstack-gerrit
Zuul
authored and
openstack-gerrit
committed
Merge "libvirt: apply mixed instance CPU policy"
2 parents aaef6ce + 6aa1931 commit 0a51759

File tree

2 files changed

+339
-5
lines changed

2 files changed

+339
-5
lines changed

‎nova/tests/unit/virt/libvirt/test_driver.py‎

Lines changed: 329 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3823,6 +3823,335 @@ def test_get_guest_config_numa_host_instance_topo_cpu_pinning(self):
38233823
self.assertEqual([instance_cell.id], memnode.nodeset)
38243824
self.assertEqual("strict", memnode.mode)
38253825

3826+
def test_get_guest_config_numa_host_instance_cpu_mixed(self):
3827+
"""Test to create mixed instance libvirt configuration which has a
3828+
default emulator thread policy and verify the NUMA topology related
3829+
settings.
3830+
"""
3831+
self.flags(cpu_shared_set='2-5,8-29',
3832+
cpu_dedicated_set='6,7,30,31',
3833+
group='compute')
3834+
3835+
instance_topology = objects.InstanceNUMATopology(cells=[
3836+
objects.InstanceNUMACell(
3837+
id=3, cpuset=set([0, 1]), pcpuset=set([2, 3]), memory=1024,
3838+
cpu_pinning={2: 30, 3: 31}
3839+
),
3840+
objects.InstanceNUMACell(
3841+
id=0, cpuset=set([4, 5, 6]), pcpuset=set([7]), memory=1024,
3842+
cpu_pinning={7: 6}
3843+
),
3844+
])
3845+
instance_ref = objects.Instance(**self.test_instance)
3846+
instance_ref.numa_topology = instance_topology
3847+
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
3848+
flavor = objects.Flavor(memory_mb=2048, vcpus=8, root_gb=496,
3849+
ephemeral_gb=8128, swap=33550336, name='fake',
3850+
extra_specs={})
3851+
instance_ref.flavor = flavor
3852+
3853+
caps = vconfig.LibvirtConfigCaps()
3854+
caps.host = vconfig.LibvirtConfigCapsHost()
3855+
caps.host.cpu = vconfig.LibvirtConfigCPU()
3856+
caps.host.cpu.arch = fields.Architecture.X86_64
3857+
caps.host.topology = fakelibvirt.NUMATopology(
3858+
cpu_nodes=4, cpu_sockets=1, cpu_cores=4, cpu_threads=2)
3859+
conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
3860+
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
3861+
instance_ref,
3862+
image_meta)
3863+
3864+
with test.nested(
3865+
mock.patch.object(
3866+
objects.InstanceNUMATopology, "get_by_instance_uuid",
3867+
return_value=instance_topology),
3868+
mock.patch.object(host.Host, 'has_min_version', return_value=True),
3869+
mock.patch.object(host.Host, "get_capabilities",
3870+
return_value=caps),
3871+
mock.patch.object(host.Host, 'get_online_cpus',
3872+
return_value=set(range(32)))
3873+
):
3874+
cfg = conn._get_guest_config(instance_ref, [],
3875+
image_meta, disk_info)
3876+
self.assertIsNone(cfg.cpuset)
3877+
3878+
# NOTE(huaqiang): Within a mixed instance, it is expected that the
3879+
# pinned and unpinned CPUs, which belong to the same instance NUMA
3880+
# cell, are scheduled on a same host NUMA cell, the instance pinned
3881+
# CPU is 1:1 scheduled to a dedicated host CPU, each unpinned CPU
3882+
# floats over the shared CPU list of the same host NUMA cell.
3883+
# The host NUMA cell's dedicated CPU list and shared CPU list are
3884+
# calculated from a combination of '[compute]cpu_dedicated_set',
3885+
# '[compute]cpu_shared_set', the host NUMA topology and the online
3886+
# CPUs.
3887+
#
3888+
# The first instance NUMA cell is fit into the fourth host NUMA
3889+
# cell due to having the same 'id' 3. Instance CPU 0 and 1 are
3890+
# unpinned CPUs, check each of them floats on the host NUMA cell's
3891+
# sharing CPUs, which are CPU 24-29.
3892+
self.assertEqual(0, cfg.cputune.vcpupin[0].id)
3893+
self.assertEqual(set(range(24, 30)), cfg.cputune.vcpupin[0].cpuset)
3894+
self.assertEqual(1, cfg.cputune.vcpupin[1].id)
3895+
self.assertEqual(set(range(24, 30)), cfg.cputune.vcpupin[1].cpuset)
3896+
# Check each of the instance NUMA cell's pinned CPUs is pinned to a
3897+
# dedicated CPU from the fourth host NUMA cell.
3898+
self.assertEqual(2, cfg.cputune.vcpupin[2].id)
3899+
self.assertEqual(set([30]), cfg.cputune.vcpupin[2].cpuset)
3900+
self.assertEqual(3, cfg.cputune.vcpupin[3].id)
3901+
self.assertEqual(set([31]), cfg.cputune.vcpupin[3].cpuset)
3902+
3903+
# Instance CPU 4-7 belong to the second instance NUMA cell, which
3904+
# is fit into host NUMA cell 0. CPU 4-6 are unpinned CPUs, each of
3905+
# them floats on the host NUMA cell's sharing CPU set, CPU 2-5.
3906+
self.assertEqual(4, cfg.cputune.vcpupin[4].id)
3907+
self.assertEqual(set(range(2, 6)), cfg.cputune.vcpupin[4].cpuset)
3908+
self.assertEqual(5, cfg.cputune.vcpupin[5].id)
3909+
self.assertEqual(set(range(2, 6)), cfg.cputune.vcpupin[5].cpuset)
3910+
self.assertEqual(6, cfg.cputune.vcpupin[6].id)
3911+
self.assertEqual(set(range(2, 6)), cfg.cputune.vcpupin[6].cpuset)
3912+
# Instance CPU 7 is pinned to the host NUMA cell's dedicated CPU 6.
3913+
self.assertEqual(set([6]), cfg.cputune.vcpupin[7].cpuset)
3914+
self.assertIsNotNone(cfg.cpu.numa)
3915+
3916+
# Check emulator thread is pinned to union of
3917+
# cfg.cputune.vcpupin[*].cpuset
3918+
self.assertIsInstance(cfg.cputune.emulatorpin,
3919+
vconfig.LibvirtConfigGuestCPUTuneEmulatorPin)
3920+
self.assertEqual(
3921+
set([2, 3, 4, 5, 6, 24, 25, 26, 27, 28, 29, 30, 31]),
3922+
cfg.cputune.emulatorpin.cpuset)
3923+
3924+
for i, (instance_cell, numa_cfg_cell) in enumerate(
3925+
zip(instance_topology.cells, cfg.cpu.numa.cells)
3926+
):
3927+
self.assertEqual(i, numa_cfg_cell.id)
3928+
self.assertEqual(instance_cell.total_cpus, numa_cfg_cell.cpus)
3929+
self.assertEqual(instance_cell.memory * units.Ki,
3930+
numa_cfg_cell.memory)
3931+
self.assertIsNone(numa_cfg_cell.memAccess)
3932+
3933+
allnodes = set([cell.id for cell in instance_topology.cells])
3934+
self.assertEqual(allnodes, set(cfg.numatune.memory.nodeset))
3935+
self.assertEqual("strict", cfg.numatune.memory.mode)
3936+
3937+
for i, (instance_cell, memnode) in enumerate(
3938+
zip(instance_topology.cells, cfg.numatune.memnodes)
3939+
):
3940+
self.assertEqual(i, memnode.cellid)
3941+
self.assertEqual([instance_cell.id], memnode.nodeset)
3942+
3943+
def test_get_guest_config_numa_host_instance_cpu_mixed_isolated_emu(self):
3944+
"""Test to create mixed instance libvirt configuration which has an
3945+
ISOLATED emulator thread policy and verify the NUMA topology related
3946+
settings.
3947+
"""
3948+
self.flags(cpu_shared_set='2-5,8-29',
3949+
cpu_dedicated_set='6,7,30,31',
3950+
group='compute')
3951+
instance_topology = objects.InstanceNUMATopology(
3952+
emulator_threads_policy=fields.CPUEmulatorThreadsPolicy.ISOLATE,
3953+
cells=[objects.InstanceNUMACell(
3954+
id=0, cpuset=set([0, 1]), pcpuset=set([2]), memory=1024,
3955+
cpu_pinning={2: 6},
3956+
cpuset_reserved=set([7]))])
3957+
instance_ref = objects.Instance(**self.test_instance)
3958+
instance_ref.numa_topology = instance_topology
3959+
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
3960+
flavor = objects.Flavor(memory_mb=2048, vcpus=8, root_gb=496,
3961+
ephemeral_gb=8128, swap=33550336, name='fake',
3962+
extra_specs={})
3963+
instance_ref.flavor = flavor
3964+
3965+
caps = vconfig.LibvirtConfigCaps()
3966+
caps.host = vconfig.LibvirtConfigCapsHost()
3967+
caps.host.cpu = vconfig.LibvirtConfigCPU()
3968+
caps.host.cpu.arch = fields.Architecture.X86_64
3969+
caps.host.topology = fakelibvirt.NUMATopology(
3970+
cpu_nodes=4, cpu_sockets=1, cpu_cores=4, cpu_threads=2)
3971+
3972+
conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
3973+
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
3974+
instance_ref,
3975+
image_meta)
3976+
3977+
with test.nested(
3978+
mock.patch.object(
3979+
objects.InstanceNUMATopology, "get_by_instance_uuid",
3980+
return_value=instance_topology),
3981+
mock.patch.object(host.Host, 'has_min_version',
3982+
return_value=True),
3983+
mock.patch.object(host.Host, "get_capabilities",
3984+
return_value=caps),
3985+
mock.patch.object(host.Host, 'get_online_cpus',
3986+
return_value=set(range(32))),
3987+
):
3988+
cfg = conn._get_guest_config(instance_ref, [],
3989+
image_meta, disk_info)
3990+
self.assertIsNone(cfg.cpuset)
3991+
# NOTE(huaqiang): The instance NUMA cell is fit into the first host
3992+
# NUMA cell, which is matched by the 'id' fields of two objects.
3993+
# CPU 2-5 are the first host NUMA cell's floating CPU set.
3994+
# Check any instance unpinned CPU is floating on this CPU set.
3995+
self.assertEqual(0, cfg.cputune.vcpupin[0].id)
3996+
self.assertEqual(set([2, 3, 4, 5]), cfg.cputune.vcpupin[0].cpuset)
3997+
self.assertEqual(1, cfg.cputune.vcpupin[1].id)
3998+
self.assertEqual(set([2, 3, 4, 5]), cfg.cputune.vcpupin[1].cpuset)
3999+
# Check instance CPU 2, a pinned CPU, is pinned to a dedicated CPU
4000+
# of host's first NUMA cell.
4001+
self.assertEqual(2, cfg.cputune.vcpupin[2].id)
4002+
self.assertEqual(set([6]), cfg.cputune.vcpupin[2].cpuset)
4003+
self.assertIsNotNone(cfg.cpu.numa)
4004+
4005+
# With an ISOLATE policy, emulator thread will be pinned to the
4006+
# reserved host CPU.
4007+
self.assertIsInstance(cfg.cputune.emulatorpin,
4008+
vconfig.LibvirtConfigGuestCPUTuneEmulatorPin)
4009+
self.assertEqual(set([7]), cfg.cputune.emulatorpin.cpuset)
4010+
4011+
for i, (instance_cell, numa_cfg_cell) in enumerate(
4012+
zip(instance_topology.cells, cfg.cpu.numa.cells)
4013+
):
4014+
self.assertEqual(i, numa_cfg_cell.id)
4015+
self.assertEqual(instance_cell.total_cpus, numa_cfg_cell.cpus)
4016+
self.assertEqual(instance_cell.memory * units.Ki,
4017+
numa_cfg_cell.memory)
4018+
self.assertIsNone(numa_cfg_cell.memAccess)
4019+
4020+
allnodes = set([cell.id for cell in instance_topology.cells])
4021+
self.assertEqual(allnodes, set(cfg.numatune.memory.nodeset))
4022+
self.assertEqual("strict", cfg.numatune.memory.mode)
4023+
4024+
for i, (instance_cell, memnode) in enumerate(
4025+
zip(instance_topology.cells, cfg.numatune.memnodes)
4026+
):
4027+
self.assertEqual(i, memnode.cellid)
4028+
self.assertEqual([instance_cell.id], memnode.nodeset)
4029+
4030+
def test_get_guest_config_numa_host_instance_cpu_mixed_realtime(self):
4031+
"""Test of creating mixed instance libvirt configuration. which is
4032+
created through 'hw:cpu_realtime_mask' and 'hw:cpu_realtime' extra
4033+
specs, verifying the NUMA topology and real-time related settings.
4034+
"""
4035+
self.flags(cpu_shared_set='2-5,8-29',
4036+
cpu_dedicated_set='6,7,30,31',
4037+
group='compute')
4038+
4039+
instance_topology = objects.InstanceNUMATopology(
4040+
cells=[
4041+
objects.InstanceNUMACell(
4042+
id=0, cpuset=set([2]), pcpuset=set([0, 1]),
4043+
cpu_pinning={0: 6, 1: 7},
4044+
memory=1024, pagesize=2048),
4045+
objects.InstanceNUMACell(
4046+
id=3, cpuset=set([3]), pcpuset=set([4, 5]),
4047+
cpu_pinning={4: 30, 5: 31},
4048+
memory=1024, pagesize=2048)])
4049+
instance_ref = objects.Instance(**self.test_instance)
4050+
instance_ref.numa_topology = instance_topology
4051+
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
4052+
# NOTE(huaqiang): libvirt driver takes the real-time CPU list from the
4053+
# flavor extra spec 'hw:cpu_realtime_mask'. For a mixed instance with
4054+
# real-time CPUs, the dedicated CPU and the real-time CPU are the
4055+
# same CPU set, this is checked in API layer.
4056+
flavor = objects.Flavor(
4057+
vcpus=6, memory_mb=2048, root_gb=496,
4058+
ephemeral_gb=8128, swap=33550336, name='fake',
4059+
extra_specs={
4060+
"hw:numa_nodes": "2",
4061+
"hw:cpu_realtime": "yes",
4062+
"hw:cpu_policy": "mixed",
4063+
"hw:cpu_realtime_mask": "^2-3"
4064+
})
4065+
instance_ref.flavor = flavor
4066+
4067+
caps = vconfig.LibvirtConfigCaps()
4068+
caps.host = vconfig.LibvirtConfigCapsHost()
4069+
caps.host.cpu = vconfig.LibvirtConfigCPU()
4070+
caps.host.cpu.arch = fields.Architecture.X86_64
4071+
caps.host.topology = fakelibvirt.NUMATopology(
4072+
cpu_nodes=4, cpu_sockets=1, cpu_cores=4, cpu_threads=2)
4073+
for i, cell in enumerate(caps.host.topology.cells):
4074+
cell.mempages = fakelibvirt.create_mempages(
4075+
[(4, 1024 * i), (2048, i)])
4076+
4077+
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
4078+
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
4079+
instance_ref,
4080+
image_meta)
4081+
4082+
with test.nested(
4083+
mock.patch.object(
4084+
objects.InstanceNUMATopology, "get_by_instance_uuid",
4085+
return_value=instance_topology),
4086+
mock.patch.object(host.Host, 'has_min_version', return_value=True),
4087+
mock.patch.object(host.Host, "get_capabilities",
4088+
return_value=caps),
4089+
mock.patch.object(host.Host, 'get_online_cpus',
4090+
return_value=set(range(32))),
4091+
):
4092+
cfg = drvr._get_guest_config(instance_ref, [],
4093+
image_meta, disk_info)
4094+
4095+
for instance_cell, numa_cfg_cell, index in zip(
4096+
instance_topology.cells,
4097+
cfg.cpu.numa.cells,
4098+
range(len(instance_topology.cells))
4099+
):
4100+
self.assertEqual(index, numa_cfg_cell.id)
4101+
self.assertEqual(instance_cell.total_cpus, numa_cfg_cell.cpus)
4102+
self.assertEqual(instance_cell.memory * units.Ki,
4103+
numa_cfg_cell.memory)
4104+
self.assertEqual("shared", numa_cfg_cell.memAccess)
4105+
4106+
allnodes = [cell.id for cell in instance_topology.cells]
4107+
self.assertEqual(allnodes, cfg.numatune.memory.nodeset)
4108+
self.assertEqual("strict", cfg.numatune.memory.mode)
4109+
4110+
for instance_cell, memnode, index in zip(
4111+
instance_topology.cells,
4112+
cfg.numatune.memnodes,
4113+
range(len(instance_topology.cells))
4114+
):
4115+
self.assertEqual(index, memnode.cellid)
4116+
self.assertEqual([instance_cell.id], memnode.nodeset)
4117+
self.assertEqual("strict", memnode.mode)
4118+
4119+
# NOTE(huaqiang): Instance first NUMA cell is fit to the first host
4120+
# NUMA cell. In this host NUMA cell, CPU 2-5 are the sharing CPU
4121+
# set, CPU 6, 7 are dedicated CPUs.
4122+
#
4123+
# Check instance CPU 0, 1 are 1:1 pinned on host NUMA cell's
4124+
# dedicated CPUs.
4125+
self.assertEqual(set([6]), cfg.cputune.vcpupin[0].cpuset)
4126+
self.assertEqual(set([7]), cfg.cputune.vcpupin[1].cpuset)
4127+
# Check CPU 2, an unpinned CPU, is floating on this host NUMA
4128+
# cell's sharing CPU set.
4129+
self.assertEqual(set([2, 3, 4, 5]), cfg.cputune.vcpupin[2].cpuset)
4130+
4131+
# The second instance NUMA cell is fit to the fourth host NUMA
4132+
# cell due to a same 'id'. Host CPU 24-29 are sharing CPU set, host
4133+
# CPU 30, 31 are dedicated CPU to be pinning.
4134+
#
4135+
# Check CPU 3 is floating on the sharing CPU set.
4136+
self.assertEqual(set([24, 25, 26, 27, 28, 29]),
4137+
cfg.cputune.vcpupin[3].cpuset)
4138+
# Check CPU 4, 5 are pinned on host dedicated CPUs.
4139+
self.assertEqual(set([30]), cfg.cputune.vcpupin[4].cpuset)
4140+
self.assertEqual(set([31]), cfg.cputune.vcpupin[5].cpuset)
4141+
4142+
# Check the real-time host CPUs are excluded from the host CPU
4143+
# list the emulator is floating on.
4144+
self.assertEqual(set([2, 3, 4, 5, 24, 25, 26, 27, 28, 29]),
4145+
cfg.cputune.emulatorpin.cpuset)
4146+
4147+
# Check the real-time scheduler is set, and all real-time CPUs are
4148+
# in the vcpusched[0].vcpus list. In nova, the real-time scheduler
4149+
# is always set to 'fifo', and there is always only one element in
4150+
# cfg.cputune.vcpusched.
4151+
self.assertEqual(1, len(cfg.cputune.vcpusched))
4152+
self.assertEqual("fifo", cfg.cputune.vcpusched[0].scheduler)
4153+
self.assertEqual(set([0, 1, 4, 5]), cfg.cputune.vcpusched[0].vcpus)
4154+
38264155
def test_get_guest_config_numa_host_mempages_shared(self):
38274156
self.flags(cpu_shared_set='2-5', cpu_dedicated_set=None,
38284157
group='compute')

‎nova/virt/libvirt/driver.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4975,19 +4975,24 @@ def _get_cell_pairs(self, guest_cpu_numa_config, host_topology):
49754975
cell_pairs.append((guest_config_cell, host_cell))
49764976
return cell_pairs
49774977

4978-
def _get_pin_cpuset(self, vcpu, object_numa_cell, host_cell):
4978+
def _get_pin_cpuset(self, vcpu, inst_cell, host_cell):
49794979
"""Returns the config object of LibvirtConfigGuestCPUTuneVCPUPin.
4980+
49804981
Prepares vcpupin config for the guest with the following caveats:
49814982
4982-
a) If there is pinning information in the cell, we pin vcpus to
4983-
individual CPUs
4983+
a) If the specified instance vCPU is intended to be pinned, we pin
4984+
it to the previously selected host CPU.
49844985
b) Otherwise we float over the whole host NUMA node
49854986
"""
49864987
pin_cpuset = vconfig.LibvirtConfigGuestCPUTuneVCPUPin()
49874988
pin_cpuset.id = vcpu
49884989

4989-
if object_numa_cell.cpu_pinning:
4990-
pin_cpuset.cpuset = set([object_numa_cell.cpu_pinning[vcpu]])
4990+
# 'InstanceNUMACell.cpu_pinning' tracks the CPU pinning pair for guest
4991+
# CPU and host CPU. If the guest CPU is in the keys of 'cpu_pinning',
4992+
# fetch the host CPU from it and pin on it, otherwise, let the guest
4993+
# CPU be floating on the sharing CPU set belonging to this NUMA cell.
4994+
if inst_cell.cpu_pinning and vcpu in inst_cell.cpu_pinning:
4995+
pin_cpuset.cpuset = set([inst_cell.cpu_pinning[vcpu]])
49914996
else:
49924997
pin_cpuset.cpuset = host_cell.cpuset
49934998

0 commit comments

Comments
 (0)