Skip to content

Commit eb0216e

Browse files
klueskaklueska
committed
Update semantics to set Preferred field in TopologyHint generation
We now only set Preferred to true if resources can be allocated with a size equal to the minimimum _possible_ mask when all resources are available.
1 parent e0e8b3e commit eb0216e

File tree

4 files changed

+123
-48
lines changed

4 files changed

+123
-48
lines changed

‎pkg/kubelet/cm/cpumanager/topology_hints.go‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,26 @@ func (m *manager) GetTopologyHints(pod v1.Pod, container v1.Container) map[strin
7171
// bits set as the narrowest matching NUMANodeAffinity with 'Preferred: true', and
7272
// marking all others with 'Preferred: false'.
7373
func (m *manager) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
74-
// Initialize minAffinity to a full affinity mask.
75-
minAffinity, _ := socketmask.NewSocketMask()
76-
minAffinity.Fill()
74+
// Initialize minAffinitySize to include all NUMA Nodes.
75+
minAffinitySize := m.topology.CPUDetails.NUMANodes().Size()
76+
// Initialize minSocketsOnMinAffinity to include all Sockets.
77+
minSocketsOnMinAffinity := m.topology.CPUDetails.Sockets().Size()
7778

7879
// Iterate through all combinations of socketMasks and build hints from them.
7980
hints := []topologymanager.TopologyHint{}
8081
socketmask.IterateSocketMasks(m.topology.CPUDetails.NUMANodes().ToSlice(), func(mask socketmask.SocketMask) {
81-
// Check to see if we have enough CPUs available on the current
82+
// First, update minAffinitySize and minSocketsOnMinAffinity for the
83+
// current request size.
84+
cpusInMask := m.topology.CPUDetails.CPUsInNUMANodes(mask.GetSockets()...).Size()
85+
socketsInMask := m.topology.CPUDetails.SocketsInNUMANodes(mask.GetSockets()...).Size()
86+
if cpusInMask >= request && mask.Count() < minAffinitySize {
87+
minAffinitySize = mask.Count()
88+
if socketsInMask < minSocketsOnMinAffinity {
89+
minSocketsOnMinAffinity = socketsInMask
90+
}
91+
}
92+
93+
// Then check to see if we have enough CPUs available on the current
8294
// SocketMask to satisfy the CPU request.
8395
numMatching := 0
8496
for _, c := range availableCPUs.ToSlice() {
@@ -99,20 +111,19 @@ func (m *manager) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, request
99111
NUMANodeAffinity: mask,
100112
Preferred: false,
101113
})
102-
103-
// Update minAffinity if relevant
104-
if mask.IsNarrowerThan(minAffinity) {
105-
minAffinity = mask
106-
}
107114
})
108115

109116
// Loop back through all hints and update the 'Preferred' field based on
110117
// counting the number of bits sets in the affinity mask and comparing it
111-
// to the minAffinity. Only those with an equal number of bits set will be
112-
// considered preferred.
118+
// to the minAffinitySize. Only those with an equal number of bits set (and
119+
// with a minimal set of sockets) will be considered preferred.
113120
for i := range hints {
114-
if hints[i].NUMANodeAffinity.Count() == minAffinity.Count() {
115-
hints[i].Preferred = true
121+
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
122+
nodes := hints[i].NUMANodeAffinity.GetSockets()
123+
numSockets := m.topology.CPUDetails.SocketsInNUMANodes(nodes...).Size()
124+
if numSockets == minSocketsOnMinAffinity {
125+
hints[i].Preferred = true
126+
}
116127
}
117128
}
118129

‎pkg/kubelet/cm/cpumanager/topology_hints_test.go‎

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -75,28 +75,18 @@ func TestGetTopologyHints(t *testing.T) {
7575
1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
7676
}
7777

78-
topology, _ := topology.Discover(&machineInfo, numaNodeInfo)
79-
80-
m := manager{
81-
policy: &staticPolicy{
82-
topology: topology,
83-
},
84-
state: &mockState{
85-
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
86-
},
87-
topology: topology,
88-
}
89-
9078
tcases := []struct {
9179
name string
9280
pod v1.Pod
9381
container v1.Container
82+
defaultCPUSet cpuset.CPUSet
9483
expectedHints []topologymanager.TopologyHint
9584
}{
9685
{
97-
name: "Request 2 CPUs; 4 available on Socket 0, 6 available on Socket 1",
98-
pod: *testPod1,
99-
container: *testContainer1,
86+
name: "Request 2 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
87+
pod: *testPod1,
88+
container: *testContainer1,
89+
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
10090
expectedHints: []topologymanager.TopologyHint{
10191
{
10292
NUMANodeAffinity: firstSocketMask,
@@ -113,9 +103,10 @@ func TestGetTopologyHints(t *testing.T) {
113103
},
114104
},
115105
{
116-
name: "Request 5 CPUs; 4 available on Socket 0, 6 available on Socket 1",
117-
pod: *testPod2,
118-
container: *testContainer2,
106+
name: "Request 5 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
107+
pod: *testPod2,
108+
container: *testContainer2,
109+
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
119110
expectedHints: []topologymanager.TopologyHint{
120111
{
121112
NUMANodeAffinity: secondSocketMask,
@@ -128,9 +119,10 @@ func TestGetTopologyHints(t *testing.T) {
128119
},
129120
},
130121
{
131-
name: "Request 7 CPUs, 4 available on Socket 0, 6 available on Socket 1",
132-
pod: *testPod3,
133-
container: *testContainer3,
122+
name: "Request 7 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
123+
pod: *testPod3,
124+
container: *testContainer3,
125+
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
134126
expectedHints: []topologymanager.TopologyHint{
135127
{
136128
NUMANodeAffinity: crossSocketMask,
@@ -139,13 +131,38 @@ func TestGetTopologyHints(t *testing.T) {
139131
},
140132
},
141133
{
142-
name: "Request 11 CPUs, 4 available on Socket 0, 6 available on Socket 1",
134+
name: "Request 11 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
143135
pod: *testPod4,
144136
container: *testContainer4,
137+
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
145138
expectedHints: nil,
146139
},
140+
{
141+
name: "Request 2 CPUs, 1 available on NUMA 0, 1 available on NUMA 1",
142+
pod: *testPod1,
143+
container: *testContainer1,
144+
defaultCPUSet: cpuset.NewCPUSet(0, 3),
145+
expectedHints: []topologymanager.TopologyHint{
146+
{
147+
NUMANodeAffinity: crossSocketMask,
148+
Preferred: false,
149+
},
150+
},
151+
},
147152
}
148153
for _, tc := range tcases {
154+
topology, _ := topology.Discover(&machineInfo, numaNodeInfo)
155+
156+
m := manager{
157+
policy: &staticPolicy{
158+
topology: topology,
159+
},
160+
state: &mockState{
161+
defaultCPUSet: tc.defaultCPUSet,
162+
},
163+
topology: topology,
164+
}
165+
149166
hints := m.GetTopologyHints(tc.pod, tc.container)[string(v1.ResourceCPU)]
150167
if len(tc.expectedHints) == 0 && len(hints) == 0 {
151168
continue

‎pkg/kubelet/cm/devicemanager/topology_hints.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,30 @@ func (m *ManagerImpl) getAvailableDevices(resource string) sets.String {
7373
}
7474

7575
func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.String, request int) []topologymanager.TopologyHint {
76-
// Initialize minAffinity to a full affinity mask.
77-
minAffinity, _ := socketmask.NewSocketMask(m.numaNodes...)
76+
// Initialize minAffinitySize to include all NUMA Nodes
77+
minAffinitySize := len(m.numaNodes)
7878

7979
// Iterate through all combinations of NUMA Nodes and build hints from them.
8080
hints := []topologymanager.TopologyHint{}
8181
socketmask.IterateSocketMasks(m.numaNodes, func(mask socketmask.SocketMask) {
82-
// Check to see if we have enough devices available on the current
82+
// First, update minAffinitySize for the current request size.
83+
devicesInMask := 0
84+
for _, device := range m.allDevices[resource] {
85+
if device.Topology == nil {
86+
continue
87+
}
88+
for _, node := range device.Topology.Nodes {
89+
if mask.IsSet(int(node.ID)) {
90+
devicesInMask++
91+
break
92+
}
93+
}
94+
}
95+
if devicesInMask >= request && mask.Count() < minAffinitySize {
96+
minAffinitySize = mask.Count()
97+
}
98+
99+
// Then check to see if we have enough devices available on the current
83100
// NUMA Node combination to satisfy the device request.
84101
numMatching := 0
85102
for d := range devices {
@@ -106,19 +123,14 @@ func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.
106123
NUMANodeAffinity: mask,
107124
Preferred: false,
108125
})
109-
110-
// Update minAffinity if relevant
111-
if mask.IsNarrowerThan(minAffinity) {
112-
minAffinity = mask
113-
}
114126
})
115127

116128
// Loop back through all hints and update the 'Preferred' field based on
117129
// counting the number of bits sets in the affinity mask and comparing it
118130
// to the minAffinity. Only those with an equal number of bits set will be
119131
// considered preferred.
120132
for i := range hints {
121-
if hints[i].NUMANodeAffinity.Count() == minAffinity.Count() {
133+
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
122134
hints[i].Preferred = true
123135
}
124136
}

‎pkg/kubelet/cm/devicemanager/topology_hints_test.go‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,11 @@ func makeSocketMask(sockets ...int) socketmask.SocketMask {
5858

5959
func TestGetTopologyHints(t *testing.T) {
6060
tcases := []struct {
61-
description string
62-
request map[string]string
63-
devices map[string][]pluginapi.Device
64-
expectedHints map[string][]topologymanager.TopologyHint
61+
description string
62+
request map[string]string
63+
devices map[string][]pluginapi.Device
64+
allocatedDevices map[string][]string
65+
expectedHints map[string][]topologymanager.TopologyHint
6566
}{
6667
{
6768
description: "Single Request, no alignment",
@@ -180,6 +181,31 @@ func TestGetTopologyHints(t *testing.T) {
180181
},
181182
},
182183
},
184+
{
185+
description: "Request for 2, optimal on 1 NUMA node, forced cross-NUMA",
186+
request: map[string]string{
187+
"testdevice": "2",
188+
},
189+
devices: map[string][]pluginapi.Device{
190+
"testdevice": {
191+
makeNUMADevice("Dev1", 0),
192+
makeNUMADevice("Dev2", 1),
193+
makeNUMADevice("Dev3", 0),
194+
makeNUMADevice("Dev4", 1),
195+
},
196+
},
197+
allocatedDevices: map[string][]string{
198+
"testdevice": {"Dev1", "Dev2"},
199+
},
200+
expectedHints: map[string][]topologymanager.TopologyHint{
201+
"testdevice": {
202+
{
203+
NUMANodeAffinity: makeSocketMask(0, 1),
204+
Preferred: false,
205+
},
206+
},
207+
},
208+
},
183209
{
184210
description: "2 device types, mixed configuration",
185211
request: map[string]string{
@@ -254,6 +280,14 @@ func TestGetTopologyHints(t *testing.T) {
254280
}
255281
}
256282

283+
for r := range tc.allocatedDevices {
284+
m.allocatedDevices[r] = sets.NewString()
285+
286+
for _, d := range tc.allocatedDevices[r] {
287+
m.allocatedDevices[r].Insert(d)
288+
}
289+
}
290+
257291
hints := m.GetTopologyHints(*pod, pod.Spec.Containers[0])
258292

259293
for r := range tc.expectedHints {
@@ -276,6 +310,7 @@ func TestTopologyAlignedAllocation(t *testing.T) {
276310
resource string
277311
request int
278312
devices []pluginapi.Device
313+
allocatedDevices []string
279314
hint topologymanager.TopologyHint
280315
expectedAllocation int
281316
expectedAlignment map[int]int

0 commit comments

Comments
 (0)