Skip to content

Commit 36c9cdc

Browse files
PiotrProkopPiotrProkop
committed
Add Costs field to NUMANode struct, which represents distance between NUMA domains
Signed-off-by: PiotrProkop <[email protected]>
1 parent 9701eb8 commit 36c9cdc

File tree

3 files changed

+252
-9
lines changed

3 files changed

+252
-9
lines changed

‎pkg/noderesourcetopology/plugin.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ const (
4141
type NUMANode struct {
4242
NUMAID int
4343
Resources v1.ResourceList
44+
Costs map[int]int
45+
}
46+
47+
func (n *NUMANode) WithCosts(costs map[int]int) *NUMANode {
48+
n.Costs = costs
49+
return n
4450
}
4551

4652
type NUMANodeList []NUMANode

‎pkg/noderesourcetopology/pluginhelpers.go

Lines changed: 75 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package noderesourcetopology
1919
import (
2020
"context"
2121
"fmt"
22+
"strconv"
23+
"strings"
2224
"time"
2325

2426
corev1 "k8s.io/api/core/v1"
@@ -35,6 +37,10 @@ import (
3537
"sigs.k8s.io/scheduler-plugins/pkg/noderesourcetopology/stringify"
3638
)
3739

40+
const (
41+
maxNUMAId = 64
42+
)
43+
3844
func initNodeTopologyInformer(tcfg *apiconfig.NodeResourceTopologyMatchArgs, handle framework.Handle) (nrtcache.Interface, error) {
3945
topoClient, err := topoclientset.NewForConfig(handle.KubeConfig())
4046
if err != nil {
@@ -85,32 +91,92 @@ func initNodeTopologyInformer(tcfg *apiconfig.NodeResourceTopologyMatchArgs, han
8591
}
8692

8793
func createNUMANodeList(zones topologyv1alpha2.ZoneList) NUMANodeList {
88-
nodes := make(NUMANodeList, 0, len(zones))
89-
for _, zone := range zones {
94+
numaIDToZoneIDx := make([]int, maxNUMAId)
95+
nodes := NUMANodeList{}
96+
// filter non Node zones and create idToIdx lookup array
97+
for i, zone := range zones {
9098
if zone.Type != "Node" {
9199
continue
92100
}
93-
var numaID int
94-
_, err := fmt.Sscanf(zone.Name, "node-%d", &numaID)
101+
102+
numaID, err := getID(zone.Name)
95103
if err != nil {
96-
klog.ErrorS(nil, "Invalid zone format", "zone", zone.Name)
97-
continue
98-
}
99-
if numaID > 63 || numaID < 0 {
100-
klog.ErrorS(nil, "Invalid NUMA id range", "numaID", numaID)
104+
klog.Error(err)
101105
continue
102106
}
107+
108+
numaIDToZoneIDx[numaID] = i
109+
103110
resources := extractResources(zone)
104111
klog.V(6).InfoS("extracted NUMA resources", stringify.ResourceListToLoggable(zone.Name, resources)...)
105112
nodes = append(nodes, NUMANode{NUMAID: numaID, Resources: resources})
106113
}
114+
115+
// iterate over nodes and fill them with Costs
116+
for i, node := range nodes {
117+
nodes[i] = *node.WithCosts(extractCosts(zones[numaIDToZoneIDx[node.NUMAID]].Costs))
118+
}
119+
107120
return nodes
108121
}
109122

123+
func getID(name string) (int, error) {
124+
splitted := strings.Split(name, "-")
125+
if len(splitted) != 2 {
126+
return -1, fmt.Errorf("invalid zone format zone: %s", name)
127+
}
128+
129+
if splitted[0] != "node" {
130+
return -1, fmt.Errorf("invalid zone format zone: %s", name)
131+
}
132+
133+
numaID, err := strconv.Atoi(splitted[1])
134+
if err != nil {
135+
return -1, fmt.Errorf("invalid zone format zone: %s : %v", name, err)
136+
}
137+
138+
if numaID > maxNUMAId-1 || numaID < 0 {
139+
return -1, fmt.Errorf("invalid NUMA id range numaID: %d", numaID)
140+
}
141+
142+
return numaID, nil
143+
}
144+
145+
func extractCosts(costs topologyv1alpha2.CostList) map[int]int {
146+
nodeCosts := make(map[int]int)
147+
148+
// return early if CostList is missing
149+
if len(costs) == 0 {
150+
return nodeCosts
151+
}
152+
153+
for _, cost := range costs {
154+
numaID, err := getID(cost.Name)
155+
if err != nil {
156+
continue
157+
}
158+
nodeCosts[numaID] = int(cost.Value)
159+
}
160+
161+
return nodeCosts
162+
}
163+
110164
func extractResources(zone topologyv1alpha2.Zone) corev1.ResourceList {
111165
res := make(corev1.ResourceList)
112166
for _, resInfo := range zone.Resources {
113167
res[corev1.ResourceName(resInfo.Name)] = resInfo.Available.DeepCopy()
114168
}
115169
return res
116170
}
171+
172+
func onlyNonNUMAResources(numaNodes NUMANodeList, resources corev1.ResourceList) bool {
173+
for resourceName := range resources {
174+
for _, node := range numaNodes {
175+
if _, ok := node.Resources[resourceName]; ok {
176+
return false
177+
}
178+
}
179+
}
180+
181+
return true
182+
}

‎pkg/noderesourcetopology/pluginhelpers_test.go

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package noderesourcetopology
18+
19+
import (
20+
"fmt"
21+
"strings"
22+
"testing"
23+
24+
corev1 "k8s.io/api/core/v1"
25+
"k8s.io/apimachinery/pkg/api/resource"
26+
)
27+
28+
func TestGetID(t *testing.T) {
29+
testCases := []struct {
30+
description string
31+
name string
32+
expectedID int
33+
expectedErr error
34+
}{
35+
{
36+
description: "id equals 1",
37+
name: "node-1",
38+
expectedID: 1,
39+
},
40+
{
41+
description: "id equals 10",
42+
name: "node-10",
43+
expectedID: 10,
44+
},
45+
{
46+
description: "invalid format of name, node name without hyphen",
47+
name: "node0",
48+
expectedErr: fmt.Errorf("invalid zone format"),
49+
},
50+
{
51+
description: "invalid format of name, zone instead of node",
52+
name: "zone-10",
53+
expectedErr: fmt.Errorf("invalid zone format"),
54+
},
55+
{
56+
description: "invalid format of name, suffix is not an integer",
57+
name: "node-a10a",
58+
expectedErr: fmt.Errorf("invalid zone format"),
59+
},
60+
{
61+
description: "invalid format of name, suffix is not an integer",
62+
name: "node-10a",
63+
expectedErr: fmt.Errorf("invalid zone format"),
64+
},
65+
{
66+
description: "invalid numaID range",
67+
name: "node-10123412415115114",
68+
expectedErr: fmt.Errorf("invalid NUMA id range"),
69+
},
70+
}
71+
72+
for _, testCase := range testCases {
73+
t.Run(testCase.description, func(t *testing.T) {
74+
id, err := getID(testCase.name)
75+
if testCase.expectedErr == nil {
76+
if err != nil {
77+
t.Fatalf("expected err to be nil not %v", err)
78+
}
79+
80+
if id != testCase.expectedID {
81+
t.Fatalf("expected id to equal %d not %d", testCase.expectedID, id)
82+
}
83+
} else {
84+
fmt.Println(id)
85+
if !strings.Contains(err.Error(), testCase.expectedErr.Error()) {
86+
t.Fatalf("expected err: %v to contain %s", err, testCase.expectedErr)
87+
}
88+
}
89+
})
90+
}
91+
}
92+
93+
func TestOnlyNonNUMAResources(t *testing.T) {
94+
numaNodes := NUMANodeList{
95+
{
96+
NUMAID: 0,
97+
Resources: corev1.ResourceList{
98+
corev1.ResourceCPU: *resource.NewQuantity(8, resource.DecimalSI),
99+
corev1.ResourceMemory: resource.MustParse("10Gi"),
100+
"gpu": resource.MustParse("1"),
101+
},
102+
},
103+
{
104+
NUMAID: 1,
105+
Resources: corev1.ResourceList{
106+
corev1.ResourceCPU: *resource.NewQuantity(8, resource.DecimalSI),
107+
corev1.ResourceMemory: resource.MustParse("10Gi"),
108+
"nic": resource.MustParse("1"),
109+
},
110+
},
111+
}
112+
testCases := []struct {
113+
description string
114+
resources corev1.ResourceList
115+
expected bool
116+
}{
117+
{
118+
description: "all resources missing in NUMANodeList",
119+
resources: corev1.ResourceList{
120+
"resource1": resource.MustParse("1"),
121+
"resource2": resource.MustParse("1"),
122+
},
123+
expected: true,
124+
},
125+
{
126+
description: "resource is present in both NUMA nodes",
127+
resources: corev1.ResourceList{
128+
corev1.ResourceCPU: resource.MustParse("1"),
129+
},
130+
expected: false,
131+
},
132+
{
133+
description: "more than resource is present in both NUMA nodes",
134+
resources: corev1.ResourceList{
135+
corev1.ResourceCPU: resource.MustParse("1"),
136+
corev1.ResourceMemory: resource.MustParse("1"),
137+
},
138+
expected: false,
139+
},
140+
{
141+
description: "resource is present only in NUMA node 0",
142+
resources: corev1.ResourceList{
143+
"gpu": resource.MustParse("1"),
144+
},
145+
expected: false,
146+
},
147+
{
148+
description: "resource is present only in NUMA node 1",
149+
resources: corev1.ResourceList{
150+
"nic": resource.MustParse("1"),
151+
},
152+
expected: false,
153+
},
154+
{
155+
description: "two distinct resources from different NUMA nodes",
156+
resources: corev1.ResourceList{
157+
"nic": resource.MustParse("1"),
158+
"gpu": resource.MustParse("1"),
159+
},
160+
expected: false,
161+
},
162+
}
163+
for _, testCase := range testCases {
164+
t.Run(testCase.description, func(t *testing.T) {
165+
result := onlyNonNUMAResources(numaNodes, testCase.resources)
166+
if result != testCase.expected {
167+
t.Fatalf("expected %t to equal %t", result, testCase.expected)
168+
}
169+
})
170+
}
171+
}

0 commit comments

Comments
 (0)