WIP: nrt: make it possible to consume MaxAllowdNUMANodes

TODO

Signed-off-by: Francesco Romani <[email protected]>

Author: ffromani

pkg/noderesourcetopology/filter.go

@@ -35,10 +35,6 @@ import (
 	"sigs.k8s.io/scheduler-plugins/pkg/util"
 )
 
-// The maximum number of NUMA nodes that Topology Manager allows is 8
-// https://kubernetes.io/docs/tasks/administer-cluster/topology-manager/#known-limitations
-const highestNUMAID = 8
-
 type PolicyHandler func(pod *v1.Pod, zoneMap topologyv1alpha2.ZoneList) *framework.Status
 
 func singleNUMAContainerLevelHandler(lh logr.Logger, pod *v1.Pod, info *filterInfo) *framework.Status {
@@ -85,7 +81,7 @@ func singleNUMAContainerLevelHandler(lh logr.Logger, pod *v1.Pod, info *filterIn
 // resourcesAvailableInAnyNUMANodes checks for sufficient resource and return the NUMAID that would be selected by Kubelet.
 // this function requires NUMANodeList with properly populated NUMANode, NUMAID should be in range 0-63
 func resourcesAvailableInAnyNUMANodes(lh logr.Logger, info *filterInfo, resources v1.ResourceList) (int, bool, string) {
-	numaID := highestNUMAID
+	numaID := info.topologyManager.MaxNUMANodes // highest NUMA ID
 	bitmask := bm.NewEmptyBitMask()
 	// set all bits, each bit is a NUMA node, if resources couldn't be aligned
 	// on the NUMA node, bit should be unset

pkg/noderesourcetopology/least_numa.go

@@ -67,7 +67,7 @@ func leastNUMAContainerScopeScore(lh logr.Logger, pod *v1.Pod, info *scoreInfo)
 		return framework.MaxNodeScore, nil
 	}
 
-	return normalizeScore(maxNUMANodesCount, allContainersMinAvgDistance), nil
+	return normalizeScore(maxNUMANodesCount, allContainersMinAvgDistance, info.topologyManager.MaxNUMANodes), nil
 }
 
 func leastNUMAPodScopeScore(lh logr.Logger, pod *v1.Pod, info *scoreInfo) (int64, *framework.Status) {
@@ -85,11 +85,11 @@ func leastNUMAPodScopeScore(lh logr.Logger, pod *v1.Pod, info *scoreInfo) (int64
 		return framework.MinNodeScore, nil
 	}
 
-	return normalizeScore(numaNodes.Count(), isMinAvgDistance), nil
+	return normalizeScore(numaNodes.Count(), isMinAvgDistance, info.topologyManager.MaxNUMANodes), nil
 }
 
-func normalizeScore(numaNodesCount int, isMinAvgDistance bool) int64 {
-	numaNodeScore := framework.MaxNodeScore / highestNUMAID
+func normalizeScore(numaNodesCount int, isMinAvgDistance bool, highestNUMAID int) int64 {
+	numaNodeScore := framework.MaxNodeScore / int64(highestNUMAID)
 	score := framework.MaxNodeScore - int64(numaNodesCount)*numaNodeScore
 	if isMinAvgDistance {
 		// if distance between NUMA domains is optimal add half of numaNodeScore to make this node more favorable

pkg/noderesourcetopology/least_numa_test.go

@@ -25,6 +25,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
+	"sigs.k8s.io/scheduler-plugins/pkg/noderesourcetopology/nodeconfig"
 )
 
 const (
@@ -746,7 +747,7 @@ func TestNormalizeScore(t *testing.T) {
 
 	for _, tc := range tcases {
 		t.Run(tc.description, func(t *testing.T) {
-			normalizedScore := normalizeScore(tc.score, tc.optimalDistance)
+			normalizedScore := normalizeScore(tc.score, tc.optimalDistance, nodeconfig.DefaultMaxNUMANodes)
 			if normalizedScore != tc.expectedScore {
 				t.Errorf("Expected normalizedScore to be %d not %d", tc.expectedScore, normalizedScore)
 			}

pkg/noderesourcetopology/nodeconfig/topologymanager.go

@@ -18,6 +18,7 @@ package nodeconfig
 
 import (
 	"fmt"
+	"strconv"
 
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 
@@ -26,11 +27,14 @@ import (
 )
 
 const (
-	AttributeScope  = "topologyManagerScope"
-	AttributePolicy = "topologyManagerPolicy"
+	DefaultMaxNUMANodes = 8 // legacy setting and default value for TopologyManager. NOTE: kube doesn't expose this constant
 )
 
-// TODO: handle topologyManagerPolicyOptions added in k8s 1.26
+const (
+	AttributeScope        = "topologyManagerScope"
+	AttributePolicy       = "topologyManagerPolicy"
+	AttributeMaxNUMANodes = "topologyManagerMaxNUMANodes"
+)
 
 func IsValidScope(scope string) bool {
 	if scope == kubeletconfig.ContainerTopologyManagerScope || scope == kubeletconfig.PodTopologyManagerScope {
@@ -48,14 +52,16 @@ func IsValidPolicy(policy string) bool {
 }
 
 type TopologyManager struct {
-	Scope  string
-	Policy string
+	Scope        string
+	Policy       string
+	MaxNUMANodes int
 }
 
 func TopologyManagerDefaults() TopologyManager {
 	return TopologyManager{
-		Scope:  kubeletconfig.ContainerTopologyManagerScope,
-		Policy: kubeletconfig.NoneTopologyManagerPolicy,
+		Scope:        kubeletconfig.ContainerTopologyManagerScope,
+		Policy:       kubeletconfig.NoneTopologyManagerPolicy,
+		MaxNUMANodes: DefaultMaxNUMANodes,
 	}
 }
 
@@ -70,7 +76,7 @@ func TopologyManagerFromNodeResourceTopology(lh logr.Logger, nodeTopology *topol
 }
 
 func (conf TopologyManager) String() string {
-	return fmt.Sprintf("policy=%q scope=%q", conf.Policy, conf.Scope)
+	return fmt.Sprintf("policy=%q scope=%q maxNUMANodes=%d", conf.Policy, conf.Scope, conf.MaxNUMANodes)
 }
 
 func (conf TopologyManager) Equal(other TopologyManager) bool {
@@ -93,7 +99,13 @@ func (conf *TopologyManager) updateFromAttributes(attrs topologyv1alpha2.Attribu
 			conf.Policy = attr.Value
 			continue
 		}
-		// TODO: handle topologyManagerPolicyOptions added in k8s 1.26
+		if attr.Name == AttributeMaxNUMANodes {
+			// the value is used in a division
+			if val, err := strconv.Atoi(attr.Value); err == nil && val > 1 {
+				conf.MaxNUMANodes = val
+				continue
+			}
+		}
 	}
 }
 

pkg/noderesourcetopology/nodeconfig/topologymanager_test.go

@@ -427,8 +427,9 @@ func TestConfigFromNRT(t *testing.T) {
 				},
 			},
 			expected: TopologyManager{
-				Policy: kubeletconfig.BestEffortTopologyManagerPolicy,
-				Scope:  kubeletconfig.PodTopologyManagerScope,
+				Policy:       kubeletconfig.BestEffortTopologyManagerPolicy,
+				Scope:        kubeletconfig.PodTopologyManagerScope,
+				MaxNUMANodes: DefaultMaxNUMANodes,
 			},
 		},
 		{
@@ -440,8 +441,9 @@ func TestConfigFromNRT(t *testing.T) {
 				},
 			},
 			expected: TopologyManager{
-				Policy: kubeletconfig.RestrictedTopologyManagerPolicy,
-				Scope:  kubeletconfig.ContainerTopologyManagerScope,
+				Policy:       kubeletconfig.RestrictedTopologyManagerPolicy,
+				Scope:        kubeletconfig.ContainerTopologyManagerScope,
+				MaxNUMANodes: DefaultMaxNUMANodes,
 			},
 		},
 		{
@@ -455,8 +457,9 @@ func TestConfigFromNRT(t *testing.T) {
 				},
 			},
 			expected: TopologyManager{
-				Policy: kubeletconfig.RestrictedTopologyManagerPolicy,
-				Scope:  kubeletconfig.ContainerTopologyManagerScope,
+				Policy:       kubeletconfig.RestrictedTopologyManagerPolicy,
+				Scope:        kubeletconfig.ContainerTopologyManagerScope,
+				MaxNUMANodes: DefaultMaxNUMANodes,
 			},
 		},
 		{
@@ -473,8 +476,9 @@ func TestConfigFromNRT(t *testing.T) {
 				},
 			},
 			expected: TopologyManager{
-				Policy: kubeletconfig.BestEffortTopologyManagerPolicy,
-				Scope:  kubeletconfig.ContainerTopologyManagerScope,
+				Policy:       kubeletconfig.BestEffortTopologyManagerPolicy,
+				Scope:        kubeletconfig.ContainerTopologyManagerScope,
+				MaxNUMANodes: DefaultMaxNUMANodes,
 			},
 		},
 		{
@@ -495,8 +499,9 @@ func TestConfigFromNRT(t *testing.T) {
 				},
 			},
 			expected: TopologyManager{
-				Policy: kubeletconfig.RestrictedTopologyManagerPolicy,
-				Scope:  kubeletconfig.ContainerTopologyManagerScope,
+				Policy:       kubeletconfig.RestrictedTopologyManagerPolicy,
+				Scope:        kubeletconfig.ContainerTopologyManagerScope,
+				MaxNUMANodes: DefaultMaxNUMANodes,
 			},
 		},
 	}