Skip to content

Commit 7094685

Browse files
fmuyassarovfmuyassarov
authored
Merge pull request #545 from klihub/devel/pick-resources-by-hints
topology-aware: try picking resources by hints first
2 parents ffbbf5c + 7ea0bb5 commit 7094685

File tree

9 files changed

+327
-76
lines changed

9 files changed

+327
-76
lines changed

‎cmd/plugins/topology-aware/policy/libmem.go‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,12 @@
1414

1515
package topologyaware
1616

17-
import libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
17+
import (
18+
"fmt"
19+
20+
"github.com/containers/nri-plugins/pkg/agent/podresapi"
21+
libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
22+
)
1823

1924
func (p *policy) getMemOffer(pool Node, req Request) (*libmem.Offer, error) {
2025
var (
@@ -45,6 +50,61 @@ func (p *policy) getMemOffer(pool Node, req Request) (*libmem.Offer, error) {
4550
return o, err
4651
}
4752

53+
func (p *policy) getMemOfferByHints(pool Node, req Request) (*libmem.Offer, error) {
54+
ctr := req.GetContainer()
55+
56+
memType := req.MemoryType()
57+
if memType == memoryPreserve {
58+
return nil, fmt.Errorf("%s by hints: memoryPreserve requested", pool.Name())
59+
}
60+
61+
zone := libmem.NodeMask(0)
62+
from := libmem.NewNodeMask(pool.GetMemset(memType).Members()...)
63+
mtyp := libmem.TypeMask(memType)
64+
65+
for provider, hint := range ctr.GetTopologyHints() {
66+
if !podresapi.IsPodResourceHint(provider) {
67+
continue
68+
}
69+
70+
m, err := libmem.ParseNodeMask(hint.NUMAs)
71+
if err != nil {
72+
return nil, err
73+
}
74+
75+
if !from.And(m).Contains(m.Slice()...) {
76+
return nil, fmt.Errorf("%s by hints: %s of wrong type (%s)", pool.Name(), m, mtyp)
77+
}
78+
79+
zone = zone.Set(m.Slice()...)
80+
}
81+
82+
if zone == libmem.NodeMask(0) {
83+
return nil, fmt.Errorf("%s by hints: no pod resource API hints", pool.Name())
84+
}
85+
86+
if zoneType := p.memAllocator.ZoneType(zone); zoneType != mtyp {
87+
return nil, fmt.Errorf("%s by hints: no type %s", pool.Name(), mtyp.Clear(zoneType.Slice()...))
88+
}
89+
90+
o, err := p.memAllocator.GetOffer(
91+
libmem.ContainerWithTypes(
92+
ctr.GetID(),
93+
ctr.PrettyName(),
94+
string(ctr.GetQOSClass()),
95+
req.MemAmountToAllocate(),
96+
zone,
97+
mtyp,
98+
),
99+
)
100+
101+
if err != nil {
102+
return nil, err
103+
}
104+
105+
return o, nil
106+
}
107+
48108
func (p *policy) restoreMemOffer(g Grant) (*libmem.Offer, error) {
49109
var (
50110
ctr = g.GetContainer()

‎cmd/plugins/topology-aware/policy/pod-preferences.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ const (
3939
keyCpuPriorityPreference = "prefer-cpu-priority"
4040
// annotation key for hiding hyperthreads from allocated CPU sets
4141
keyHideHyperthreads = "hide-hyperthreads"
42+
// annotation key for picking individual resources by topology hints
43+
keyPickResourcesByHints = "pick-resources-by-hints"
4244

4345
// effective annotation key for isolated CPU preference
4446
preferIsolatedCPUsKey = "prefer-isolated-cpus" + "." + kubernetes.ResmgrKeyNamespace
@@ -54,6 +56,8 @@ const (
5456
preferCpuPriorityKey = keyCpuPriorityPreference + "." + kubernetes.ResmgrKeyNamespace
5557
// effective annotation key for hiding hyperthreads
5658
hideHyperthreadsKey = keyHideHyperthreads + "." + kubernetes.ResmgrKeyNamespace
59+
// effective annotation key for picking resources by topology hints
60+
pickResourcesByHints = keyPickResourcesByHints + "." + kubernetes.ResmgrKeyNamespace
5761
)
5862

5963
type prefKind int
@@ -449,6 +453,25 @@ func memoryAllocationPreference(pod cache.Pod, c cache.Container) (int64, int64,
449453
return req, lim, mtype
450454
}
451455

456+
func pickByHintsPreference(pod cache.Pod, container cache.Container) bool {
457+
value, ok := pod.GetEffectiveAnnotation(pickResourcesByHints, container.GetName())
458+
if !ok {
459+
return false
460+
}
461+
462+
pick, err := strconv.ParseBool(value)
463+
if err != nil {
464+
log.Error("failed to parse pick resources by hints preference %s = '%s': %v",
465+
pickResourcesByHints, value, err)
466+
return false
467+
}
468+
469+
log.Debug("%s: effective pick resources by hints preference %v",
470+
container.PrettyName(), pick)
471+
472+
return pick
473+
}
474+
452475
// String stringifies a cpuClass.
453476
func (t cpuClass) String() string {
454477
if cpuClassName, ok := cpuClassNames[t]; ok {

‎cmd/plugins/topology-aware/policy/pools.go‎

Lines changed: 51 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -670,10 +670,11 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco
670670
//
671671
// - insufficient isolated, reserved or shared capacity loses
672672
// - if we have affinity, the higher affinity score wins
673-
// - if only one node matches the memory type request, it wins
674673
// - if we have topology hints
675674
// * better hint score wins
676675
// * for a tie, prefer the lower node then the smaller id
676+
// - if we have a better matching or tighter fitting memory offer, it wins
677+
// - if only one node matches the memory type request, it wins
677678
// - for low-prio and high-prio CPU preference, if only one node has such CPUs, it wins
678679
// - if a node is lower in the tree it wins
679680
// - for reserved allocations
@@ -722,6 +723,55 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco
722723

723724
log.Debug(" - affinity is a TIE")
724725

726+
// better topology hint score wins
727+
hScores1 := score1.HintScores()
728+
if len(hScores1) > 0 {
729+
hScores2 := score2.HintScores()
730+
hs1, nz1 := combineHintScores(hScores1)
731+
hs2, nz2 := combineHintScores(hScores2)
732+
733+
if hs1 > hs2 {
734+
log.Debug(" => %s WINS on hints", node1.Name())
735+
return true
736+
}
737+
if hs2 > hs1 {
738+
log.Debug(" => %s WINS on hints", node2.Name())
739+
return false
740+
}
741+
742+
log.Debug(" - hints are a TIE")
743+
744+
if hs1 == 0 {
745+
if nz1 > nz2 {
746+
log.Debug(" => %s WINS on non-zero hints", node1.Name())
747+
return true
748+
}
749+
if nz2 > nz1 {
750+
log.Debug(" => %s WINS on non-zero hints", node2.Name())
751+
return false
752+
}
753+
754+
log.Debug(" - non-zero hints are a TIE")
755+
}
756+
757+
// for a tie, prefer lower nodes and smaller ids
758+
if hs1 == hs2 && nz1 == nz2 && (hs1 != 0 || nz1 != 0) {
759+
if depth1 > depth2 {
760+
log.Debug(" => %s WINS as it is lower", node1.Name())
761+
return true
762+
}
763+
if depth1 < depth2 {
764+
log.Debug(" => %s WINS as it is lower", node2.Name())
765+
return false
766+
}
767+
768+
log.Debug(" => %s WINS based on equal hint socres, lower id",
769+
map[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])
770+
771+
return id1 < id2
772+
}
773+
}
774+
725775
// better matching or tighter memory offer wins
726776
switch {
727777
case o1 != nil && o2 == nil:
@@ -789,55 +839,6 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco
789839
log.Debug(" - memory type is a TIE")
790840
}
791841

792-
// better topology hint score wins
793-
hScores1 := score1.HintScores()
794-
if len(hScores1) > 0 {
795-
hScores2 := score2.HintScores()
796-
hs1, nz1 := combineHintScores(hScores1)
797-
hs2, nz2 := combineHintScores(hScores2)
798-
799-
if hs1 > hs2 {
800-
log.Debug(" => %s WINS on hints", node1.Name())
801-
return true
802-
}
803-
if hs2 > hs1 {
804-
log.Debug(" => %s WINS on hints", node2.Name())
805-
return false
806-
}
807-
808-
log.Debug(" - hints are a TIE")
809-
810-
if hs1 == 0 {
811-
if nz1 > nz2 {
812-
log.Debug(" => %s WINS on non-zero hints", node1.Name())
813-
return true
814-
}
815-
if nz2 > nz1 {
816-
log.Debug(" => %s WINS on non-zero hints", node2.Name())
817-
return false
818-
}
819-
820-
log.Debug(" - non-zero hints are a TIE")
821-
}
822-
823-
// for a tie, prefer lower nodes and smaller ids
824-
if hs1 == hs2 && nz1 == nz2 && (hs1 != 0 || nz1 != 0) {
825-
if depth1 > depth2 {
826-
log.Debug(" => %s WINS as it is lower", node1.Name())
827-
return true
828-
}
829-
if depth1 < depth2 {
830-
log.Debug(" => %s WINS as it is lower", node2.Name())
831-
return false
832-
}
833-
834-
log.Debug(" => %s WINS based on equal hint socres, lower id",
835-
map[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])
836-
837-
return id1 < id2
838-
}
839-
}
840-
841842
// for low-prio and high-prio CPU preference, the only fulfilling node wins
842843
log.Debug(" - preferred CPU priority is %s", request.CPUPrio())
843844
switch request.CPUPrio() {

0 commit comments

Comments
 (0)